That's quite a bit of dirty, and a fair amount of slab, relative to the rest. I'm not used to seeing writeback above a couple hundred pages, but I can't say that I know what, if anything, that means (it might just be high from having lots of dirty pages, and mean nothing on its own).Active:132283
inactive:1508049
dirty:161830
writeback:2625
unstable:0
free:10126
slab:90087
mapped:168019
pagetables:12444
bounce:0
vm.min_free_kbytes = 262144
vm.dirty_ratio = 10
vm.dirty_background_ratio = 5
vm.extfrag_threshold = 100
[root@hal9000 envirostate]# dmesg
e1000e: eth0 NIC Link is Down
CIFS VFS: Server borg.loc has not responded in 300 seconds. Reconnecting...
INFO: task bdi-default:29 blocked for more than 120 seconds.
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
bdi-default D 0000000000000002 0 29 2 0x00000000
ffff88013cdf1a40 0000000000000046 0000000000000000 ffff880139d7d980
ffff88013cdf1a50 ffffffffa02800ac ffff88013cdf1a18 ffff88013b7b1d70
ffff88013cdeb0b8 ffff88013cdf1fd8 000000000000f4e8 ffff88013cdeb0b8
Call Trace:
[<ffffffffa02800ac>] ? smb_sendv+0xfc/0x380 [cifs]
[<ffffffff81110930>] ? sync_page+0x0/0x50
[<ffffffff814f40d3>] io_schedule+0x73/0xc0
[<ffffffff8111096d>] sync_page+0x3d/0x50
[<ffffffff814f493a>] __wait_on_bit_lock+0x5a/0xc0
[<ffffffff81110907>] __lock_page+0x67/0x70
[<ffffffff81090a50>] ? wake_bit_function+0x0/0x50
[<ffffffffa0276dbf>] cifs_writepages+0x63f/0x670 [cifs]
[<ffffffff81126171>] do_writepages+0x21/0x40
[<ffffffff811a034d>] writeback_single_inode+0xdd/0x2c0
[<ffffffff811a078e>] writeback_sb_inodes+0xce/0x180
[<ffffffff811a08eb>] writeback_inodes_wb+0xab/0x1b0
[<ffffffff810096f0>] ? __switch_to+0xd0/0x320
[<ffffffff811a0c8b>] wb_writeback+0x29b/0x3f0
[<ffffffff814f3911>] ? thread_return+0x4e/0x79d
[<ffffffff811a0f79>] wb_do_writeback+0x199/0x240
[<ffffffff8107c020>] ? process_timeout+0x0/0x10
[<ffffffff811348ba>] bdi_forker_task+0x6a/0x310
[<ffffffff81134850>] ? bdi_forker_task+0x0/0x310
[<ffffffff810906a6>] kthread+0x96/0xa0
[<ffffffff8100c14a>] child_rip+0xa/0x20
[<ffffffff81090610>] ? kthread+0x0/0xa0
[<ffffffff8100c140>] ? child_rip+0x0/0x20
INFO: task FahCore_a3.exe:14391 blocked for more than 120 seconds.
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
FahCore_a3.ex D 0000000000000001 0 14391 1972 0x00000080
ffff880138eedb58 0000000000000082 0000000000000000 ffff880037930a38
ffffffff81facfa0 ffffea0003fee0e0 ffff880138eedae8 ffffffff812763d6
ffff880037930638 ffff880138eedfd8 000000000000f4e8 ffff880037930638
Call Trace:
[<ffffffff812763d6>] ? __prop_inc_single+0x46/0x60
[<ffffffff81110930>] ? sync_page+0x0/0x50
[<ffffffff814f40d3>] io_schedule+0x73/0xc0
[<ffffffff8111096d>] sync_page+0x3d/0x50
[<ffffffff814f4a8f>] __wait_on_bit+0x5f/0x90
[<ffffffff81110b23>] wait_on_page_bit+0x73/0x80
[<ffffffff81090a50>] ? wake_bit_function+0x0/0x50
[<ffffffffa0276a5d>] cifs_writepages+0x2dd/0x670 [cifs]
[<ffffffff81126171>] do_writepages+0x21/0x40
[<ffffffff8111108b>] __filemap_fdatawrite_range+0x5b/0x60
[<ffffffff8111158f>] filemap_fdatawrite+0x1f/0x30
[<ffffffffa025e11c>] cifs_file_aio_write+0x3c/0x50 [cifs]
[<ffffffff811760fa>] do_sync_write+0xfa/0x140
[<ffffffff81090a10>] ? autoremove_wake_function+0x0/0x40
[<ffffffff81060fa0>] ? pick_next_task_fair+0xd0/0x130
[<ffffffff8120c296>] ? security_file_permission+0x16/0x20
[<ffffffff811763f8>] vfs_write+0xb8/0x1a0
[<ffffffff81176e01>] sys_write+0x51/0x90
[<ffffffff8100b0f2>] system_call_fastpath+0x16/0x1b
INFO: task bdi-default:29 blocked for more than 120 seconds.
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
bdi-default D 0000000000000002 0 29 2 0x00000000
ffff88013cdf1a40 0000000000000046 0000000000000000 ffff880139d7d980
ffff88013cdf1a50 ffffffffa02800ac ffff88013cdf1a18 ffff88013b7b1d70
ffff88013cdeb0b8 ffff88013cdf1fd8 000000000000f4e8 ffff88013cdeb0b8
Call Trace:
[<ffffffffa02800ac>] ? smb_sendv+0xfc/0x380 [cifs]
[<ffffffff81110930>] ? sync_page+0x0/0x50
[<ffffffff814f40d3>] io_schedule+0x73/0xc0
[<ffffffff8111096d>] sync_page+0x3d/0x50
[<ffffffff814f493a>] __wait_on_bit_lock+0x5a/0xc0
[<ffffffff81110907>] __lock_page+0x67/0x70
[<ffffffff81090a50>] ? wake_bit_function+0x0/0x50
[<ffffffffa0276dbf>] cifs_writepages+0x63f/0x670 [cifs]
[<ffffffff81126171>] do_writepages+0x21/0x40
[<ffffffff811a034d>] writeback_single_inode+0xdd/0x2c0
[<ffffffff811a078e>] writeback_sb_inodes+0xce/0x180
[<ffffffff811a08eb>] writeback_inodes_wb+0xab/0x1b0
[<ffffffff810096f0>] ? __switch_to+0xd0/0x320
[<ffffffff811a0c8b>] wb_writeback+0x29b/0x3f0
[<ffffffff814f3911>] ? thread_return+0x4e/0x79d
[<ffffffff811a0f79>] wb_do_writeback+0x199/0x240
[<ffffffff8107c020>] ? process_timeout+0x0/0x10
[<ffffffff811348ba>] bdi_forker_task+0x6a/0x310
[<ffffffff81134850>] ? bdi_forker_task+0x0/0x310
[<ffffffff810906a6>] kthread+0x96/0xa0
[<ffffffff8100c14a>] child_rip+0xa/0x20
[<ffffffff81090610>] ? kthread+0x0/0xa0
[<ffffffff8100c140>] ? child_rip+0x0/0x20
INFO: task FahCore_a3.exe:14391 blocked for more than 120 seconds.
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
FahCore_a3.ex D 0000000000000001 0 14391 1972 0x00000080
ffff880138eedb58 0000000000000082 0000000000000000 ffff880037930a38
ffffffff81facfa0 ffffea0003fee0e0 ffff880138eedae8 ffffffff812763d6
ffff880037930638 ffff880138eedfd8 000000000000f4e8 ffff880037930638
Call Trace:
[<ffffffff812763d6>] ? __prop_inc_single+0x46/0x60
[<ffffffff81110930>] ? sync_page+0x0/0x50
[<ffffffff814f40d3>] io_schedule+0x73/0xc0
[<ffffffff8111096d>] sync_page+0x3d/0x50
[<ffffffff814f4a8f>] __wait_on_bit+0x5f/0x90
[<ffffffff81110b23>] wait_on_page_bit+0x73/0x80
[<ffffffff81090a50>] ? wake_bit_function+0x0/0x50
[<ffffffffa0276a5d>] cifs_writepages+0x2dd/0x670 [cifs]
[<ffffffff81126171>] do_writepages+0x21/0x40
[<ffffffff8111108b>] __filemap_fdatawrite_range+0x5b/0x60
[<ffffffff8111158f>] filemap_fdatawrite+0x1f/0x30
[<ffffffffa025e11c>] cifs_file_aio_write+0x3c/0x50 [cifs]
[<ffffffff811760fa>] do_sync_write+0xfa/0x140
[<ffffffff81090a10>] ? autoremove_wake_function+0x0/0x40
[<ffffffff81060fa0>] ? pick_next_task_fair+0xd0/0x130
[<ffffffff8120c296>] ? security_file_permission+0x16/0x20
[<ffffffff811763f8>] vfs_write+0xb8/0x1a0
[<ffffffff81176e01>] sys_write+0x51/0x90
[<ffffffff8100b0f2>] system_call_fastpath+0x16/0x1b
INFO: task kslowd000:14390 blocked for more than 120 seconds.
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
kslowd000 D 0000000000000001 0 14390 2 0x00000080
ffff8801394f7d00 0000000000000046 0000000000000000 0000000000000082
ffff8801394f7cd0 ffffffff8105e5dc 0000000000015f80 0000000000000002
ffff88013b7e4638 ffff8801394f7fd8 000000000000f4e8 ffff88013b7e4638
Call Trace:
[<ffffffff8105e5dc>] ? try_to_wake_up+0x24c/0x3e0
[<ffffffff81110930>] ? sync_page+0x0/0x50
[<ffffffff814f40d3>] io_schedule+0x73/0xc0
[<ffffffff8111096d>] sync_page+0x3d/0x50
[<ffffffff814f493a>] __wait_on_bit_lock+0x5a/0xc0
[<ffffffff81110907>] __lock_page+0x67/0x70
[<ffffffff81090a50>] ? wake_bit_function+0x0/0x50
[<ffffffffa026994e>] cifs_writev_complete+0x1ee/0x210 [cifs]
[<ffffffff81104673>] slow_work_execute+0x233/0x310
[<ffffffff811048a7>] slow_work_thread+0x157/0x360
[<ffffffff81090a10>] ? autoremove_wake_function+0x0/0x40
[<ffffffff81104750>] ? slow_work_thread+0x0/0x360
[<ffffffff810906a6>] kthread+0x96/0xa0
[<ffffffff8100c14a>] child_rip+0xa/0x20
[<ffffffff81090610>] ? kthread+0x0/0xa0
[<ffffffff8100c140>] ? child_rip+0x0/0x20
INFO: task bdi-default:29 blocked for more than 120 seconds.
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
bdi-default D 0000000000000002 0 29 2 0x00000000
ffff88013cdf1a40 0000000000000046 0000000000000000 ffff880139d7d980
ffff88013cdf1a50 ffffffffa02800ac ffff88013cdf1a18 ffff88013b7b1d70
ffff88013cdeb0b8 ffff88013cdf1fd8 000000000000f4e8 ffff88013cdeb0b8
Call Trace:
[<ffffffffa02800ac>] ? smb_sendv+0xfc/0x380 [cifs]
[<ffffffff81110930>] ? sync_page+0x0/0x50
[<ffffffff814f40d3>] io_schedule+0x73/0xc0
[<ffffffff8111096d>] sync_page+0x3d/0x50
[<ffffffff814f493a>] __wait_on_bit_lock+0x5a/0xc0
[<ffffffff81110907>] __lock_page+0x67/0x70
[<ffffffff81090a50>] ? wake_bit_function+0x0/0x50
[<ffffffffa0276dbf>] cifs_writepages+0x63f/0x670 [cifs]
[<ffffffff81126171>] do_writepages+0x21/0x40
[<ffffffff811a034d>] writeback_single_inode+0xdd/0x2c0
[<ffffffff811a078e>] writeback_sb_inodes+0xce/0x180
[<ffffffff811a08eb>] writeback_inodes_wb+0xab/0x1b0
[<ffffffff810096f0>] ? __switch_to+0xd0/0x320
[<ffffffff811a0c8b>] wb_writeback+0x29b/0x3f0
[<ffffffff814f3911>] ? thread_return+0x4e/0x79d
[<ffffffff811a0f79>] wb_do_writeback+0x199/0x240
[<ffffffff8107c020>] ? process_timeout+0x0/0x10
[<ffffffff811348ba>] bdi_forker_task+0x6a/0x310
[<ffffffff81134850>] ? bdi_forker_task+0x0/0x310
[<ffffffff810906a6>] kthread+0x96/0xa0
[<ffffffff8100c14a>] child_rip+0xa/0x20
[<ffffffff81090610>] ? kthread+0x0/0xa0
[<ffffffff8100c140>] ? child_rip+0x0/0x20
INFO: task FahCore_a3.exe:14391 blocked for more than 120 seconds.
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
FahCore_a3.ex D 0000000000000001 0 14391 1972 0x00000080
ffff880138eedb58 0000000000000082 0000000000000000 ffff880037930a38
ffffffff81facfa0 ffffea0003fee0e0 ffff880138eedae8 ffffffff812763d6
ffff880037930638 ffff880138eedfd8 000000000000f4e8 ffff880037930638
Call Trace:
[<ffffffff812763d6>] ? __prop_inc_single+0x46/0x60
[<ffffffff81110930>] ? sync_page+0x0/0x50
[<ffffffff814f40d3>] io_schedule+0x73/0xc0
[<ffffffff8111096d>] sync_page+0x3d/0x50
[<ffffffff814f4a8f>] __wait_on_bit+0x5f/0x90
[<ffffffff81110b23>] wait_on_page_bit+0x73/0x80
[<ffffffff81090a50>] ? wake_bit_function+0x0/0x50
[<ffffffffa0276a5d>] cifs_writepages+0x2dd/0x670 [cifs]
[<ffffffff81126171>] do_writepages+0x21/0x40
[<ffffffff8111108b>] __filemap_fdatawrite_range+0x5b/0x60
[<ffffffff8111158f>] filemap_fdatawrite+0x1f/0x30
[<ffffffffa025e11c>] cifs_file_aio_write+0x3c/0x50 [cifs]
[<ffffffff811760fa>] do_sync_write+0xfa/0x140
[<ffffffff81090a10>] ? autoremove_wake_function+0x0/0x40
[<ffffffff81060fa0>] ? pick_next_task_fair+0xd0/0x130
[<ffffffff8120c296>] ? security_file_permission+0x16/0x20
[<ffffffff811763f8>] vfs_write+0xb8/0x1a0
[<ffffffff81176e01>] sys_write+0x51/0x90
[<ffffffff8100b0f2>] system_call_fastpath+0x16/0x1b
INFO: task kslowd000:14390 blocked for more than 120 seconds.
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
kslowd000 D 0000000000000001 0 14390 2 0x00000080
ffff8801394f7d00 0000000000000046 0000000000000000 0000000000000082
ffff8801394f7cd0 ffffffff8105e5dc 0000000000015f80 0000000000000002
ffff88013b7e4638 ffff8801394f7fd8 000000000000f4e8 ffff88013b7e4638
Call Trace:
[<ffffffff8105e5dc>] ? try_to_wake_up+0x24c/0x3e0
[<ffffffff81110930>] ? sync_page+0x0/0x50
[<ffffffff814f40d3>] io_schedule+0x73/0xc0
[<ffffffff8111096d>] sync_page+0x3d/0x50
[<ffffffff814f493a>] __wait_on_bit_lock+0x5a/0xc0
[<ffffffff81110907>] __lock_page+0x67/0x70
[<ffffffff81090a50>] ? wake_bit_function+0x0/0x50
[<ffffffffa026994e>] cifs_writev_complete+0x1ee/0x210 [cifs]
[<ffffffff81104673>] slow_work_execute+0x233/0x310
[<ffffffff811048a7>] slow_work_thread+0x157/0x360
[<ffffffff81090a10>] ? autoremove_wake_function+0x0/0x40
[<ffffffff81104750>] ? slow_work_thread+0x0/0x360
[<ffffffff810906a6>] kthread+0x96/0xa0
[<ffffffff8100c14a>] child_rip+0xa/0x20
[<ffffffff81090610>] ? kthread+0x0/0xa0
[<ffffffff8100c140>] ? child_rip+0x0/0x20
INFO: task bdi-default:29 blocked for more than 120 seconds.
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
bdi-default D 0000000000000002 0 29 2 0x00000000
ffff88013cdf1a40 0000000000000046 0000000000000000 ffff880139d7d980
ffff88013cdf1a50 ffffffffa02800ac ffff88013cdf1a18 ffff88013b7b1d70
ffff88013cdeb0b8 ffff88013cdf1fd8 000000000000f4e8 ffff88013cdeb0b8
Call Trace:
[<ffffffffa02800ac>] ? smb_sendv+0xfc/0x380 [cifs]
[<ffffffff81110930>] ? sync_page+0x0/0x50
[<ffffffff814f40d3>] io_schedule+0x73/0xc0
[<ffffffff8111096d>] sync_page+0x3d/0x50
[<ffffffff814f493a>] __wait_on_bit_lock+0x5a/0xc0
[<ffffffff81110907>] __lock_page+0x67/0x70
[<ffffffff81090a50>] ? wake_bit_function+0x0/0x50
[<ffffffffa0276dbf>] cifs_writepages+0x63f/0x670 [cifs]
[<ffffffff81126171>] do_writepages+0x21/0x40
[<ffffffff811a034d>] writeback_single_inode+0xdd/0x2c0
[<ffffffff811a078e>] writeback_sb_inodes+0xce/0x180
[<ffffffff811a08eb>] writeback_inodes_wb+0xab/0x1b0
[<ffffffff810096f0>] ? __switch_to+0xd0/0x320
[<ffffffff811a0c8b>] wb_writeback+0x29b/0x3f0
[<ffffffff814f3911>] ? thread_return+0x4e/0x79d
[<ffffffff811a0f79>] wb_do_writeback+0x199/0x240
[<ffffffff8107c020>] ? process_timeout+0x0/0x10
[<ffffffff811348ba>] bdi_forker_task+0x6a/0x310
[<ffffffff81134850>] ? bdi_forker_task+0x0/0x310
[<ffffffff810906a6>] kthread+0x96/0xa0
[<ffffffff8100c14a>] child_rip+0xa/0x20
[<ffffffff81090610>] ? kthread+0x0/0xa0
[<ffffffff8100c140>] ? child_rip+0x0/0x20
INFO: task FahCore_a3.exe:14391 blocked for more than 120 seconds.
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
FahCore_a3.ex D 0000000000000001 0 14391 1972 0x00000080
ffff880138eedb58 0000000000000082 0000000000000000 ffff880037930a38
ffffffff81facfa0 ffffea0003fee0e0 ffff880138eedae8 ffffffff812763d6
ffff880037930638 ffff880138eedfd8 000000000000f4e8 ffff880037930638
Call Trace:
[<ffffffff812763d6>] ? __prop_inc_single+0x46/0x60
[<ffffffff81110930>] ? sync_page+0x0/0x50
[<ffffffff814f40d3>] io_schedule+0x73/0xc0
[<ffffffff8111096d>] sync_page+0x3d/0x50
[<ffffffff814f4a8f>] __wait_on_bit+0x5f/0x90
[<ffffffff81110b23>] wait_on_page_bit+0x73/0x80
[<ffffffff81090a50>] ? wake_bit_function+0x0/0x50
[<ffffffffa0276a5d>] cifs_writepages+0x2dd/0x670 [cifs]
[<ffffffff81126171>] do_writepages+0x21/0x40
[<ffffffff8111108b>] __filemap_fdatawrite_range+0x5b/0x60
[<ffffffff8111158f>] filemap_fdatawrite+0x1f/0x30
[<ffffffffa025e11c>] cifs_file_aio_write+0x3c/0x50 [cifs]
[<ffffffff811760fa>] do_sync_write+0xfa/0x140
[<ffffffff81090a10>] ? autoremove_wake_function+0x0/0x40
[<ffffffff81060fa0>] ? pick_next_task_fair+0xd0/0x130
[<ffffffff8120c296>] ? security_file_permission+0x16/0x20
[<ffffffff811763f8>] vfs_write+0xb8/0x1a0
[<ffffffff81176e01>] sys_write+0x51/0x90
[<ffffffff8100b0f2>] system_call_fastpath+0x16/0x1b
e1000e: eth0 NIC Link is Up 1000 Mbps Full Duplex, Flow Control: Rx/Tx
[root@hal9000 envirostate]#
[root@hal9000 envirostate]#
Ok this really sucks. One of my other servers started doing it too now. WTF?
No VM software of any kind on that machine, so guess I can rule out VirtualBox being the cause of these crashes. My server monitoring app crashed in this process. This server runs my hvac as well so this is quite serious. This one is running CentOS (version 5 I think, but not entirely sure how to check)
Edit: seems this may have happened during my switch cut over. Still, a simple network connectivity issue should not cause an entire system to crash.
ata5: exception Emask 0x50 SAct 0x0 SErr 0x4090800 action 0xe frozen
ata5: irq_stat 0x00400040, connection status changed
ata5: SError: { HostInt PHYRdyChg 10B8B DevExch }
ata5: hard resetting link
ata5: SATA link down (SStatus 0 SControl 300)
ata5: hard resetting link
ata5: SATA link down (SStatus 0 SControl 300)
ata5: limiting SATA link speed to 1.5 Gbps
ata5: hard resetting link
ata5: SATA link down (SStatus 0 SControl 310)
ata5.00: disabled
ata5: EH complete
ata5.00: detaching (SCSI 9:0:0:0)
sd 9:0:0:0: [sdo] Synchronizing SCSI cache
sd 9:0:0:0: [sdo] Result: hostbyte=DID_BAD_TARGET driverbyte=DRIVER_OK
sd 9:0:0:0: [sdo] Stopping disk
sd 9:0:0:0: [sdo] START_STOP FAILED
sd 9:0:0:0: [sdo] Result: hostbyte=DID_BAD_TARGET driverbyte=DRIVER_OK
md: data-check of RAID array md1
md: minimum _guaranteed_ speed: 200000 KB/sec/disk.
md: using maximum available idle IO bandwidth (but not more than 75000000 KB/sec) for data-check.
md: using 128k window, over a total of 976759936k.
md: data-check of RAID array md0
md: minimum _guaranteed_ speed: 200000 KB/sec/disk.
md: using maximum available idle IO bandwidth (but not more than 75000000 KB/sec) for data-check.
md: using 128k window, over a total of 5860270080k.
md: md1: data-check done.
md: md0: data-check done.
swapper: page allocation failure. order:1, mode:0x20
Pid: 0, comm: swapper Not tainted 2.6.32-358.el6.x86_64 #1
Call Trace:
<IRQ> [<ffffffff8112c127>] ? __alloc_pages_nodemask+0x757/0x8d0
[<ffffffff811669d2>] ? kmem_getpages+0x62/0x170
[<ffffffff811675ea>] ? fallback_alloc+0x1ba/0x270
[<ffffffff8116703f>] ? cache_grow+0x2cf/0x320
[<ffffffff81167369>] ? ____cache_alloc_node+0x99/0x160
[<ffffffff811682eb>] ? kmem_cache_alloc+0x11b/0x190
[<ffffffff81439b38>] ? sk_prot_alloc+0x48/0x1c0
[<ffffffff8143ac12>] ? sk_clone+0x22/0x2e0
[<ffffffff81489ae6>] ? inet_csk_clone+0x16/0xd0
[<ffffffff814a29f3>] ? tcp_create_openreq_child+0x23/0x450
[<ffffffff814a01ed>] ? tcp_v4_syn_recv_sock+0x4d/0x310
[<ffffffff814a2796>] ? tcp_check_req+0x226/0x460
[<ffffffff814981b6>] ? tcp_rcv_state_process+0x126/0xa10
[<ffffffff8149fc8b>] ? tcp_v4_do_rcv+0x35b/0x430
[<ffffffff814a5d1f>] ? __udp_queue_rcv_skb+0x4f/0x170
[<ffffffff814a149e>] ? tcp_v4_rcv+0x4fe/0x8d0
[<ffffffff8143cfd7>] ? __kfree_skb+0x47/0xa0
[<ffffffff8147f28d>] ? ip_local_deliver_finish+0xdd/0x2d0
[<ffffffff8147f518>] ? ip_local_deliver+0x98/0xa0
[<ffffffff8147e9dd>] ? ip_rcv_finish+0x12d/0x440
[<ffffffff8147ef65>] ? ip_rcv+0x275/0x350
[<ffffffff8144819b>] ? __netif_receive_skb+0x4ab/0x750
[<ffffffff8144a578>] ? netif_receive_skb+0x58/0x60
[<ffffffff8144a680>] ? napi_skb_finish+0x50/0x70
[<ffffffff8144cc29>] ? napi_gro_receive+0x39/0x50
[<ffffffffa01d07cb>] ? e1000_receive_skb+0x5b/0x90 [e1000e]
[<ffffffffa01d33a0>] ? e1000_clean_rx_irq+0x380/0x580 [e1000e]
[<ffffffffa01daffd>] ? e1000e_poll+0x8d/0x380 [e1000e]
[<ffffffff8109ace2>] ? enqueue_hrtimer+0x82/0xd0
[<ffffffff8144cd43>] ? net_rx_action+0x103/0x2f0
[<ffffffff81076fb1>] ? __do_softirq+0xc1/0x1e0
[<ffffffff810e1640>] ? handle_IRQ_event+0x60/0x170
[<ffffffff8100c1cc>] ? call_softirq+0x1c/0x30
[<ffffffff8100de05>] ? do_softirq+0x65/0xa0
[<ffffffff81076d95>] ? irq_exit+0x85/0x90
[<ffffffff81516c95>] ? do_IRQ+0x75/0xf0
[<ffffffff8100b9d3>] ? ret_from_intr+0x0/0x11
<EOI> [<ffffffff812d37ae>] ? intel_idle+0xde/0x170
[<ffffffff812d3791>] ? intel_idle+0xc1/0x170
[<ffffffff81414ef7>] ? cpuidle_idle_call+0xa7/0x140
[<ffffffff81009fc6>] ? cpu_idle+0xb6/0x110
[<ffffffff814f2f2a>] ? rest_init+0x7a/0x80
[<ffffffff81c27f7b>] ? start_kernel+0x424/0x430
[<ffffffff81c2733a>] ? x86_64_start_reservations+0x125/0x129
[<ffffffff81c27438>] ? x86_64_start_kernel+0xfa/0x109
swapper: page allocation failure. order:1, mode:0x20
Pid: 0, comm: swapper Not tainted 2.6.32-358.el6.x86_64 #1
Call Trace:
<IRQ> [<ffffffff8112c127>] ? __alloc_pages_nodemask+0x757/0x8d0
[<ffffffff811669d2>] ? kmem_getpages+0x62/0x170
[<ffffffff811675ea>] ? fallback_alloc+0x1ba/0x270
[<ffffffff8116703f>] ? cache_grow+0x2cf/0x320
[<ffffffff81167369>] ? ____cache_alloc_node+0x99/0x160
[<ffffffff811682eb>] ? kmem_cache_alloc+0x11b/0x190
[<ffffffff81439b38>] ? sk_prot_alloc+0x48/0x1c0
[<ffffffff8143ac12>] ? sk_clone+0x22/0x2e0
[<ffffffff81489ae6>] ? inet_csk_clone+0x16/0xd0
[<ffffffff814a29f3>] ? tcp_create_openreq_child+0x23/0x450
[<ffffffff814a01ed>] ? tcp_v4_syn_recv_sock+0x4d/0x310
[<ffffffff814a2796>] ? tcp_check_req+0x226/0x460
[<ffffffff8149fc8b>] ? tcp_v4_do_rcv+0x35b/0x430
[<ffffffff814a5d1f>] ? __udp_queue_rcv_skb+0x4f/0x170
[<ffffffff814a149e>] ? tcp_v4_rcv+0x4fe/0x8d0
[<ffffffff8143cfd7>] ? __kfree_skb+0x47/0xa0
[<ffffffff8147f28d>] ? ip_local_deliver_finish+0xdd/0x2d0
[<ffffffff8147f518>] ? ip_local_deliver+0x98/0xa0
[<ffffffff8147e9dd>] ? ip_rcv_finish+0x12d/0x440
[<ffffffff8147ef65>] ? ip_rcv+0x275/0x350
[<ffffffff8144819b>] ? __netif_receive_skb+0x4ab/0x750
[<ffffffff8144a578>] ? netif_receive_skb+0x58/0x60
[<ffffffff8144a680>] ? napi_skb_finish+0x50/0x70
[<ffffffff8144cc29>] ? napi_gro_receive+0x39/0x50
[<ffffffffa01d07cb>] ? e1000_receive_skb+0x5b/0x90 [e1000e]
[<ffffffffa01d33a0>] ? e1000_clean_rx_irq+0x380/0x580 [e1000e]
[<ffffffffa01daffd>] ? e1000e_poll+0x8d/0x380 [e1000e]
[<ffffffff8109ace2>] ? enqueue_hrtimer+0x82/0xd0
[<ffffffff8144cd43>] ? net_rx_action+0x103/0x2f0
[<ffffffff81076fb1>] ? __do_softirq+0xc1/0x1e0
[<ffffffff810e1640>] ? handle_IRQ_event+0x60/0x170
[<ffffffff8100c1cc>] ? call_softirq+0x1c/0x30
[<ffffffff8100de05>] ? do_softirq+0x65/0xa0
[<ffffffff81076d95>] ? irq_exit+0x85/0x90
[<ffffffff81516c95>] ? do_IRQ+0x75/0xf0
[<ffffffff8100b9d3>] ? ret_from_intr+0x0/0x11
<EOI> [<ffffffff812d37ae>] ? intel_idle+0xde/0x170
[<ffffffff812d3791>] ? intel_idle+0xc1/0x170
[<ffffffff81414ef7>] ? cpuidle_idle_call+0xa7/0x140
[<ffffffff81009fc6>] ? cpu_idle+0xb6/0x110
[<ffffffff814f2f2a>] ? rest_init+0x7a/0x80
[<ffffffff81c27f7b>] ? start_kernel+0x424/0x430
[<ffffffff81c2733a>] ? x86_64_start_reservations+0x125/0x129
[<ffffffff81c27438>] ? x86_64_start_kernel+0xfa/0x109
swapper: page allocation failure. order:1, mode:0x20
Pid: 0, comm: swapper Not tainted 2.6.32-358.el6.x86_64 #1
Call Trace:
<IRQ> [<ffffffff8112c127>] ? __alloc_pages_nodemask+0x757/0x8d0
[<ffffffff811669d2>] ? kmem_getpages+0x62/0x170
[<ffffffff811675ea>] ? fallback_alloc+0x1ba/0x270
[<ffffffff8116703f>] ? cache_grow+0x2cf/0x320
[<ffffffff81167369>] ? ____cache_alloc_node+0x99/0x160
[<ffffffff811682eb>] ? kmem_cache_alloc+0x11b/0x190
[<ffffffff81439b38>] ? sk_prot_alloc+0x48/0x1c0
[<ffffffff8143ac12>] ? sk_clone+0x22/0x2e0
[<ffffffff81489ae6>] ? inet_csk_clone+0x16/0xd0
[<ffffffff814a29f3>] ? tcp_create_openreq_child+0x23/0x450
[<ffffffff814a01ed>] ? tcp_v4_syn_recv_sock+0x4d/0x310
[<ffffffff814a2796>] ? tcp_check_req+0x226/0x460
[<ffffffff814981b6>] ? tcp_rcv_state_process+0x126/0xa10
[<ffffffff8149fc8b>] ? tcp_v4_do_rcv+0x35b/0x430
[<ffffffff814a5d1f>] ? __udp_queue_rcv_skb+0x4f/0x170
[<ffffffff814a149e>] ? tcp_v4_rcv+0x4fe/0x8d0
[<ffffffff8147f28d>] ? ip_local_deliver_finish+0xdd/0x2d0
[<ffffffff8147f518>] ? ip_local_deliver+0x98/0xa0
[<ffffffff8147e9dd>] ? ip_rcv_finish+0x12d/0x440
[<ffffffff8147ef65>] ? ip_rcv+0x275/0x350
[<ffffffff8144819b>] ? __netif_receive_skb+0x4ab/0x750
[<ffffffff8144a578>] ? netif_receive_skb+0x58/0x60
[<ffffffff8144a680>] ? napi_skb_finish+0x50/0x70
[<ffffffff8144cc29>] ? napi_gro_receive+0x39/0x50
[<ffffffffa01d07cb>] ? e1000_receive_skb+0x5b/0x90 [e1000e]
[<ffffffffa01d33a0>] ? e1000_clean_rx_irq+0x380/0x580 [e1000e]
[<ffffffffa01daffd>] ? e1000e_poll+0x8d/0x380 [e1000e]
[<ffffffff81052201>] ? resched_task+0x11/0x80
[<ffffffff8106317e>] ? try_to_wake_up+0x24e/0x3e0
[<ffffffff8144cd43>] ? net_rx_action+0x103/0x2f0
[<ffffffff81076fb1>] ? __do_softirq+0xc1/0x1e0
[<ffffffff810e1640>] ? handle_IRQ_event+0x60/0x170
[<ffffffff8100c1cc>] ? call_softirq+0x1c/0x30
[<ffffffff8100de05>] ? do_softirq+0x65/0xa0
[<ffffffff81076d95>] ? irq_exit+0x85/0x90
[<ffffffff81516c95>] ? do_IRQ+0x75/0xf0
[<ffffffff8100b9d3>] ? ret_from_intr+0x0/0x11
<EOI> [<ffffffff812d37ae>] ? intel_idle+0xde/0x170
[<ffffffff812d3791>] ? intel_idle+0xc1/0x170
[<ffffffff8109dd1d>] ? sched_clock_cpu+0xcd/0x110
[<ffffffff81414ef7>] ? cpuidle_idle_call+0xa7/0x140
[<ffffffff81009fc6>] ? cpu_idle+0xb6/0x110
[<ffffffff814f2f2a>] ? rest_init+0x7a/0x80
[<ffffffff81c27f7b>] ? start_kernel+0x424/0x430
[<ffffffff81c2733a>] ? x86_64_start_reservations+0x125/0x129
[<ffffffff81c27438>] ? x86_64_start_kernel+0xfa/0x109
swapper: page allocation failure. order:1, mode:0x20
Pid: 0, comm: swapper Not tainted 2.6.32-358.el6.x86_64 #1
Call Trace:
<IRQ> [<ffffffff8112c127>] ? __alloc_pages_nodemask+0x757/0x8d0
[<ffffffff810522fd>] ? check_preempt_curr+0x6d/0x90
[<ffffffff811669d2>] ? kmem_getpages+0x62/0x170
[<ffffffff811675ea>] ? fallback_alloc+0x1ba/0x270
[<ffffffff8116703f>] ? cache_grow+0x2cf/0x320
[<ffffffff81167369>] ? ____cache_alloc_node+0x99/0x160
[<ffffffff811682eb>] ? kmem_cache_alloc+0x11b/0x190
[<ffffffff81439b38>] ? sk_prot_alloc+0x48/0x1c0
[<ffffffff8143ac12>] ? sk_clone+0x22/0x2e0
[<ffffffff81489ae6>] ? inet_csk_clone+0x16/0xd0
[<ffffffff814a29f3>] ? tcp_create_openreq_child+0x23/0x450
[<ffffffff814a01ed>] ? tcp_v4_syn_recv_sock+0x4d/0x310
[<ffffffff814a2796>] ? tcp_check_req+0x226/0x460
[<ffffffff8149fc8b>] ? tcp_v4_do_rcv+0x35b/0x430
[<ffffffff814a5d1f>] ? __udp_queue_rcv_skb+0x4f/0x170
[<ffffffff814a149e>] ? tcp_v4_rcv+0x4fe/0x8d0
[<ffffffff8147f28d>] ? ip_local_deliver_finish+0xdd/0x2d0
[<ffffffff8147f518>] ? ip_local_deliver+0x98/0xa0
[<ffffffff8147e9dd>] ? ip_rcv_finish+0x12d/0x440
[<ffffffff8147ef65>] ? ip_rcv+0x275/0x350
[<ffffffff8144819b>] ? __netif_receive_skb+0x4ab/0x750
[<ffffffff8144a578>] ? netif_receive_skb+0x58/0x60
[<ffffffff8144a680>] ? napi_skb_finish+0x50/0x70
[<ffffffff8144cc29>] ? napi_gro_receive+0x39/0x50
[<ffffffffa01d07cb>] ? e1000_receive_skb+0x5b/0x90 [e1000e]
[<ffffffffa01d33a0>] ? e1000_clean_rx_irq+0x380/0x580 [e1000e]
[<ffffffffa01daffd>] ? e1000e_poll+0x8d/0x380 [e1000e]
[<ffffffff81052201>] ? resched_task+0x11/0x80
[<ffffffff8106317e>] ? try_to_wake_up+0x24e/0x3e0
[<ffffffff8144cd43>] ? net_rx_action+0x103/0x2f0
[<ffffffff81076fb1>] ? __do_softirq+0xc1/0x1e0
[<ffffffff810e1640>] ? handle_IRQ_event+0x60/0x170
[<ffffffff8100c1cc>] ? call_softirq+0x1c/0x30
[<ffffffff8100de05>] ? do_softirq+0x65/0xa0
[<ffffffff81076d95>] ? irq_exit+0x85/0x90
[<ffffffff81516c95>] ? do_IRQ+0x75/0xf0
[<ffffffff8100b9d3>] ? ret_from_intr+0x0/0x11
<EOI> [<ffffffff812d37ae>] ? intel_idle+0xde/0x170
[<ffffffff812d3791>] ? intel_idle+0xc1/0x170
[<ffffffff8109dd1d>] ? sched_clock_cpu+0xcd/0x110
[<ffffffff81414ef7>] ? cpuidle_idle_call+0xa7/0x140
[<ffffffff81009fc6>] ? cpu_idle+0xb6/0x110
[<ffffffff814f2f2a>] ? rest_init+0x7a/0x80
[<ffffffff81c27f7b>] ? start_kernel+0x424/0x430
[<ffffffff81c2733a>] ? x86_64_start_reservations+0x125/0x129
[<ffffffff81c27438>] ? x86_64_start_kernel+0xfa/0x109
swapper: page allocation failure. order:1, mode:0x20
Pid: 0, comm: swapper Not tainted 2.6.32-358.el6.x86_64 #1
Call Trace:
<IRQ> [<ffffffff8112c127>] ? __alloc_pages_nodemask+0x757/0x8d0
[<ffffffff811669d2>] ? kmem_getpages+0x62/0x170
[<ffffffff811675ea>] ? fallback_alloc+0x1ba/0x270
[<ffffffff8116703f>] ? cache_grow+0x2cf/0x320
[<ffffffff81167369>] ? ____cache_alloc_node+0x99/0x160
[<ffffffff811682eb>] ? kmem_cache_alloc+0x11b/0x190
[<ffffffff81439b38>] ? sk_prot_alloc+0x48/0x1c0
[<ffffffff8143ac12>] ? sk_clone+0x22/0x2e0
[<ffffffff81489ae6>] ? inet_csk_clone+0x16/0xd0
[<ffffffff814a29f3>] ? tcp_create_openreq_child+0x23/0x450
[<ffffffff814a01ed>] ? tcp_v4_syn_recv_sock+0x4d/0x310
[<ffffffff814a2796>] ? tcp_check_req+0x226/0x460
[<ffffffff814981b6>] ? tcp_rcv_state_process+0x126/0xa10
[<ffffffff8149fc8b>] ? tcp_v4_do_rcv+0x35b/0x430
[<ffffffff810522fd>] ? check_preempt_curr+0x6d/0x90
[<ffffffff814a149e>] ? tcp_v4_rcv+0x4fe/0x8d0
[<ffffffff81063322>] ? default_wake_function+0x12/0x20
[<ffffffff8147f28d>] ? ip_local_deliver_finish+0xdd/0x2d0
[<ffffffff8147f518>] ? ip_local_deliver+0x98/0xa0
[<ffffffff8147e9dd>] ? ip_rcv_finish+0x12d/0x440
[<ffffffff8147ef65>] ? ip_rcv+0x275/0x350
[<ffffffff8144819b>] ? __netif_receive_skb+0x4ab/0x750
[<ffffffff8144a578>] ? netif_receive_skb+0x58/0x60
[<ffffffff8144a680>] ? napi_skb_finish+0x50/0x70
[<ffffffff8144cc29>] ? napi_gro_receive+0x39/0x50
[<ffffffffa01d07cb>] ? e1000_receive_skb+0x5b/0x90 [e1000e]
[<ffffffffa01d33a0>] ? e1000_clean_rx_irq+0x380/0x580 [e1000e]
[<ffffffffa01daffd>] ? e1000e_poll+0x8d/0x380 [e1000e]
[<ffffffff8105ad3e>] ? scheduler_tick+0x11e/0x260
[<ffffffff8144cd43>] ? net_rx_action+0x103/0x2f0
[<ffffffff81076fb1>] ? __do_softirq+0xc1/0x1e0
[<ffffffff810e1640>] ? handle_IRQ_event+0x60/0x170
[<ffffffff8100c1cc>] ? call_softirq+0x1c/0x30
[<ffffffff8100de05>] ? do_softirq+0x65/0xa0
[<ffffffff81076d95>] ? irq_exit+0x85/0x90
[<ffffffff81516c95>] ? do_IRQ+0x75/0xf0
[<ffffffff8100b9d3>] ? ret_from_intr+0x0/0x11
<EOI> [<ffffffff812d37ae>] ? intel_idle+0xde/0x170
[<ffffffff812d3791>] ? intel_idle+0xc1/0x170
[<ffffffff8109dd1d>] ? sched_clock_cpu+0xcd/0x110
[<ffffffff81414ef7>] ? cpuidle_idle_call+0xa7/0x140
[<ffffffff81009fc6>] ? cpu_idle+0xb6/0x110
[<ffffffff814f2f2a>] ? rest_init+0x7a/0x80
[<ffffffff81c27f7b>] ? start_kernel+0x424/0x430
[<ffffffff81c2733a>] ? x86_64_start_reservations+0x125/0x129
[<ffffffff81c27438>] ? x86_64_start_kernel+0xfa/0x109
swapper: page allocation failure. order:1, mode:0x20
Pid: 0, comm: swapper Not tainted 2.6.32-358.el6.x86_64 #1
Call Trace:
<IRQ> [<ffffffff8112c127>] ? __alloc_pages_nodemask+0x757/0x8d0
[<ffffffff811669d2>] ? kmem_getpages+0x62/0x170
[<ffffffff811675ea>] ? fallback_alloc+0x1ba/0x270
[<ffffffff8116703f>] ? cache_grow+0x2cf/0x320
[<ffffffff81167369>] ? ____cache_alloc_node+0x99/0x160
[<ffffffff811682eb>] ? kmem_cache_alloc+0x11b/0x190
[<ffffffff81439b38>] ? sk_prot_alloc+0x48/0x1c0
[<ffffffff8143ac12>] ? sk_clone+0x22/0x2e0
[<ffffffff81489ae6>] ? inet_csk_clone+0x16/0xd0
[<ffffffff814a29f3>] ? tcp_create_openreq_child+0x23/0x450
[<ffffffff814a01ed>] ? tcp_v4_syn_recv_sock+0x4d/0x310
[<ffffffff814a2796>] ? tcp_check_req+0x226/0x460
[<ffffffff8149fc8b>] ? tcp_v4_do_rcv+0x35b/0x430
[<ffffffff810522fd>] ? check_preempt_curr+0x6d/0x90
[<ffffffff814a149e>] ? tcp_v4_rcv+0x4fe/0x8d0
[<ffffffff81063322>] ? default_wake_function+0x12/0x20
[<ffffffff8147f28d>] ? ip_local_deliver_finish+0xdd/0x2d0
[<ffffffff8147f518>] ? ip_local_deliver+0x98/0xa0
[<ffffffff8147e9dd>] ? ip_rcv_finish+0x12d/0x440
[<ffffffff8147ef65>] ? ip_rcv+0x275/0x350
[<ffffffff8144819b>] ? __netif_receive_skb+0x4ab/0x750
[<ffffffff8144a578>] ? netif_receive_skb+0x58/0x60
[<ffffffff8144a680>] ? napi_skb_finish+0x50/0x70
[<ffffffff8144cc29>] ? napi_gro_receive+0x39/0x50
[<ffffffffa01d07cb>] ? e1000_receive_skb+0x5b/0x90 [e1000e]
[<ffffffffa01d33a0>] ? e1000_clean_rx_irq+0x380/0x580 [e1000e]
[<ffffffffa01daffd>] ? e1000e_poll+0x8d/0x380 [e1000e]
[<ffffffff8105ad3e>] ? scheduler_tick+0x11e/0x260
[<ffffffff8144cd43>] ? net_rx_action+0x103/0x2f0
[<ffffffff81076fb1>] ? __do_softirq+0xc1/0x1e0
[<ffffffff810e1640>] ? handle_IRQ_event+0x60/0x170
[<ffffffff8100c1cc>] ? call_softirq+0x1c/0x30
[<ffffffff8100de05>] ? do_softirq+0x65/0xa0
[<ffffffff81076d95>] ? irq_exit+0x85/0x90
[<ffffffff81516c95>] ? do_IRQ+0x75/0xf0
[<ffffffff8100b9d3>] ? ret_from_intr+0x0/0x11
<EOI> [<ffffffff812d37ae>] ? intel_idle+0xde/0x170
[<ffffffff812d3791>] ? intel_idle+0xc1/0x170
[<ffffffff8109dd1d>] ? sched_clock_cpu+0xcd/0x110
[<ffffffff81414ef7>] ? cpuidle_idle_call+0xa7/0x140
[<ffffffff81009fc6>] ? cpu_idle+0xb6/0x110
[<ffffffff814f2f2a>] ? rest_init+0x7a/0x80
[<ffffffff81c27f7b>] ? start_kernel+0x424/0x430
[<ffffffff81c2733a>] ? x86_64_start_reservations+0x125/0x129
[<ffffffff81c27438>] ? x86_64_start_kernel+0xfa/0x109
md: data-check of RAID array md1
md: minimum _guaranteed_ speed: 200000 KB/sec/disk.
md: using maximum available idle IO bandwidth (but not more than 75000000 KB/sec) for data-check.
md: using 128k window, over a total of 976759936k.
md: data-check of RAID array md0
md: minimum _guaranteed_ speed: 200000 KB/sec/disk.
md: using maximum available idle IO bandwidth (but not more than 75000000 KB/sec) for data-check.
md: using 128k window, over a total of 5860270080k.
md: md0: data-check done.
md: md1: data-check done.
ata5: exception Emask 0x10 SAct 0x0 SErr 0x4050002 action 0xe frozen
ata5: irq_stat 0x00400040, connection status changed
ata5: SError: { RecovComm PHYRdyChg CommWake DevExch }
ata5: hard resetting link
ata5: link is slow to respond, please be patient (ready=0)
ata5: COMRESET failed (errno=-16)
ata5: hard resetting link
ata5: link is slow to respond, please be patient (ready=0)
ata5: SATA link up 1.5 Gbps (SStatus 113 SControl 300)
ata5.00: ATA-7: Hitachi HDS721010KLA330, GKAOAB0A, max UDMA/133
ata5.00: 1953525168 sectors, multi 0: LBA48 NCQ (depth 31/32), AA
ata5.00: configured for UDMA/133
ata5: EH complete
scsi 9:0:0:0: Direct-Access ATA Hitachi HDS72101 GKAO PQ: 0 ANSI: 5
sd 9:0:0:0: Attached scsi generic sg14 type 0
sd 9:0:0:0: [sdo] 1953525168 512-byte logical blocks: (1.00 TB/931 GiB)
sd 9:0:0:0: [sdo] Write Protect is off
sd 9:0:0:0: [sdo] Mode Sense: 00 3a 00 00
sd 9:0:0:0: [sdo] Write cache: enabled, read cache: enabled, doesn't support DPO or FUA
sdo: sdo1
sd 9:0:0:0: [sdo] Attached SCSI disk
kjournald starting. Commit interval 5 seconds
EXT3-fs (sdo1): warning: checktime reached, running e2fsck is recommended
EXT3-fs (sdo1): using internal journal
EXT3-fs (sdo1): mounted filesystem with ordered data mode
rsync: page allocation failure. order:1, mode:0x20
Pid: 19498, comm: rsync Not tainted 2.6.32-358.el6.x86_64 #1
Call Trace:
<IRQ> [<ffffffff8112c127>] ? __alloc_pages_nodemask+0x757/0x8d0
[<ffffffff811669d2>] ? kmem_getpages+0x62/0x170
[<ffffffff811675ea>] ? fallback_alloc+0x1ba/0x270
[<ffffffff8116703f>] ? cache_grow+0x2cf/0x320
[<ffffffff81167369>] ? ____cache_alloc_node+0x99/0x160
[<ffffffff811682eb>] ? kmem_cache_alloc+0x11b/0x190
[<ffffffff81439b38>] ? sk_prot_alloc+0x48/0x1c0
[<ffffffff8143ac12>] ? sk_clone+0x22/0x2e0
[<ffffffff81489ae6>] ? inet_csk_clone+0x16/0xd0
[<ffffffff814a29f3>] ? tcp_create_openreq_child+0x23/0x450
[<ffffffff814a01ed>] ? tcp_v4_syn_recv_sock+0x4d/0x310
[<ffffffff814a2796>] ? tcp_check_req+0x226/0x460
[<ffffffff814981b6>] ? tcp_rcv_state_process+0x126/0xa10
[<ffffffff8149fc8b>] ? tcp_v4_do_rcv+0x35b/0x430
[<ffffffff8105230c>] ? check_preempt_curr+0x7c/0x90
[<ffffffff814a149e>] ? tcp_v4_rcv+0x4fe/0x8d0
[<ffffffff81063322>] ? default_wake_function+0x12/0x20
[<ffffffff8147f28d>] ? ip_local_deliver_finish+0xdd/0x2d0
[<ffffffff8147f518>] ? ip_local_deliver+0x98/0xa0
[<ffffffff8147e9dd>] ? ip_rcv_finish+0x12d/0x440
[<ffffffff8147ef65>] ? ip_rcv+0x275/0x350
[<ffffffff8144819b>] ? __netif_receive_skb+0x4ab/0x750
[<ffffffff8144a578>] ? netif_receive_skb+0x58/0x60
[<ffffffff8144a680>] ? napi_skb_finish+0x50/0x70
[<ffffffff8144cc29>] ? napi_gro_receive+0x39/0x50
[<ffffffffa01d07cb>] ? e1000_receive_skb+0x5b/0x90 [e1000e]
[<ffffffffa01d33a0>] ? e1000_clean_rx_irq+0x380/0x580 [e1000e]
[<ffffffffa01daffd>] ? e1000e_poll+0x8d/0x380 [e1000e]
[<ffffffff8144cd43>] ? net_rx_action+0x103/0x2f0
[<ffffffff81076fb1>] ? __do_softirq+0xc1/0x1e0
[<ffffffff810e1640>] ? handle_IRQ_event+0x60/0x170
[<ffffffff8100c1cc>] ? call_softirq+0x1c/0x30
[<ffffffff8100de05>] ? do_softirq+0x65/0xa0
[<ffffffff81076d95>] ? irq_exit+0x85/0x90
[<ffffffff81516c95>] ? do_IRQ+0x75/0xf0
[<ffffffff8100b9d3>] ? ret_from_intr+0x0/0x11
<EOI>
rsync: page allocation failure. order:1, mode:0x20
Pid: 19498, comm: rsync Not tainted 2.6.32-358.el6.x86_64 #1
Call Trace:
<IRQ> [<ffffffff8112c127>] ? __alloc_pages_nodemask+0x757/0x8d0
[<ffffffff811669d2>] ? kmem_getpages+0x62/0x170
[<ffffffff811675ea>] ? fallback_alloc+0x1ba/0x270
[<ffffffff8116703f>] ? cache_grow+0x2cf/0x320
[<ffffffff81167369>] ? ____cache_alloc_node+0x99/0x160
[<ffffffff811682eb>] ? kmem_cache_alloc+0x11b/0x190
[<ffffffff81439b38>] ? sk_prot_alloc+0x48/0x1c0
[<ffffffff8143ac12>] ? sk_clone+0x22/0x2e0
[<ffffffff81489ae6>] ? inet_csk_clone+0x16/0xd0
[<ffffffff814a29f3>] ? tcp_create_openreq_child+0x23/0x450
[<ffffffff814a01ed>] ? tcp_v4_syn_recv_sock+0x4d/0x310
[<ffffffff814a2796>] ? tcp_check_req+0x226/0x460
[<ffffffff8149fc8b>] ? tcp_v4_do_rcv+0x35b/0x430
[<ffffffff8105230c>] ? check_preempt_curr+0x7c/0x90
[<ffffffff814a149e>] ? tcp_v4_rcv+0x4fe/0x8d0
[<ffffffff81063322>] ? default_wake_function+0x12/0x20
[<ffffffff8147f28d>] ? ip_local_deliver_finish+0xdd/0x2d0
[<ffffffff8147f518>] ? ip_local_deliver+0x98/0xa0
[<ffffffff8147e9dd>] ? ip_rcv_finish+0x12d/0x440
[<ffffffff8147ef65>] ? ip_rcv+0x275/0x350
[<ffffffff8144819b>] ? __netif_receive_skb+0x4ab/0x750
[<ffffffff8144a578>] ? netif_receive_skb+0x58/0x60
[<ffffffff8144a680>] ? napi_skb_finish+0x50/0x70
[<ffffffff8144cc29>] ? napi_gro_receive+0x39/0x50
[<ffffffffa01d07cb>] ? e1000_receive_skb+0x5b/0x90 [e1000e]
[<ffffffffa01d33a0>] ? e1000_clean_rx_irq+0x380/0x580 [e1000e]
[<ffffffffa01daffd>] ? e1000e_poll+0x8d/0x380 [e1000e]
[<ffffffff8144cd43>] ? net_rx_action+0x103/0x2f0
[<ffffffff81076fb1>] ? __do_softirq+0xc1/0x1e0
[<ffffffff810e1640>] ? handle_IRQ_event+0x60/0x170
[<ffffffff8100c1cc>] ? call_softirq+0x1c/0x30
[<ffffffff8100de05>] ? do_softirq+0x65/0xa0
[<ffffffff81076d95>] ? irq_exit+0x85/0x90
[<ffffffff81516c95>] ? do_IRQ+0x75/0xf0
[<ffffffff8100b9d3>] ? ret_from_intr+0x0/0x11
<EOI>
ata5: exception Emask 0x50 SAct 0x0 SErr 0x4090800 action 0xe frozen
ata5: irq_stat 0x00400040, connection status changed
ata5: SError: { HostInt PHYRdyChg 10B8B DevExch }
ata5: hard resetting link
ata5: SATA link down (SStatus 0 SControl 300)
ata5: hard resetting link
ata5: SATA link down (SStatus 0 SControl 300)
ata5: limiting SATA link speed to 1.5 Gbps
ata5: hard resetting link
ata5: SATA link down (SStatus 0 SControl 310)
ata5.00: disabled
ata5: EH complete
ata5.00: detaching (SCSI 9:0:0:0)
sd 9:0:0:0: [sdo] Synchronizing SCSI cache
sd 9:0:0:0: [sdo] Result: hostbyte=DID_BAD_TARGET driverbyte=DRIVER_OK
sd 9:0:0:0: [sdo] Stopping disk
sd 9:0:0:0: [sdo] START_STOP FAILED
sd 9:0:0:0: [sdo] Result: hostbyte=DID_BAD_TARGET driverbyte=DRIVER_OK
ata5: exception Emask 0x10 SAct 0x0 SErr 0x4050002 action 0xe frozen
ata5: irq_stat 0x00400040, connection status changed
ata5: SError: { RecovComm PHYRdyChg CommWake DevExch }
ata5: hard resetting link
ata5: SATA link down (SStatus 0 SControl 300)
ata5.00: link offline, clearing class 1 to NONE
ata5: EH complete
ata5.00: ATA-7: Hitachi HDS721010KLA330, GKAOAB0A, max UDMA/133
rsync: page allocation failure. order:1, mode:0x20
Pid: 18527, comm: rsync Not tainted 2.6.32-358.el6.x86_64 #1
Call Trace:
<IRQ> [<ffffffff8112c127>] ? __alloc_pages_nodemask+0x757/0x8d0
[<ffffffffa01d4069>] ? e1000_xmit_frame+0x769/0xdc0 [e1000e]
[<ffffffff811669d2>] ? kmem_getpages+0x62/0x170
[<ffffffff811675ea>] ? fallback_alloc+0x1ba/0x270
[<ffffffff8116703f>] ? cache_grow+0x2cf/0x320
[<ffffffff81167369>] ? ____cache_alloc_node+0x99/0x160
[<ffffffff811682eb>] ? kmem_cache_alloc+0x11b/0x190
[<ffffffff81439b38>] ? sk_prot_alloc+0x48/0x1c0
[<ffffffff8143ac12>] ? sk_clone+0x22/0x2e0
[<ffffffff81489ae6>] ? inet_csk_clone+0x16/0xd0
[<ffffffff814a29f3>] ? tcp_create_openreq_child+0x23/0x450
[<ffffffff814a01ed>] ? tcp_v4_syn_recv_sock+0x4d/0x310
[<ffffffff814a2796>] ? tcp_check_req+0x226/0x460
[<ffffffff814981b6>] ? tcp_rcv_state_process+0x126/0xa10
[<ffffffff8149fc8b>] ? tcp_v4_do_rcv+0x35b/0x430
[<ffffffff8105230c>] ? check_preempt_curr+0x7c/0x90
[<ffffffff814a149e>] ? tcp_v4_rcv+0x4fe/0x8d0
[<ffffffff8147f28d>] ? ip_local_deliver_finish+0xdd/0x2d0
[<ffffffff8147f518>] ? ip_local_deliver+0x98/0xa0
[<ffffffff8147e9dd>] ? ip_rcv_finish+0x12d/0x440
[<ffffffff8147ef65>] ? ip_rcv+0x275/0x350
[<ffffffff8144819b>] ? __netif_receive_skb+0x4ab/0x750
[<ffffffff8144a578>] ? netif_receive_skb+0x58/0x60
[<ffffffff8144a680>] ? napi_skb_finish+0x50/0x70
[<ffffffff8144cc29>] ? napi_gro_receive+0x39/0x50
[<ffffffffa01d07cb>] ? e1000_receive_skb+0x5b/0x90 [e1000e]
[<ffffffffa01d33a0>] ? e1000_clean_rx_irq+0x380/0x580 [e1000e]
[<ffffffffa01daffd>] ? e1000e_poll+0x8d/0x380 [e1000e]
[<ffffffff8144cd43>] ? net_rx_action+0x103/0x2f0
[<ffffffff81076fb1>] ? __do_softirq+0xc1/0x1e0
[<ffffffff810e1640>] ? handle_IRQ_event+0x60/0x170
[<ffffffff8107700f>] ? __do_softirq+0x11f/0x1e0
[<ffffffff8100c1cc>] ? call_softirq+0x1c/0x30
[<ffffffff8100de05>] ? do_softirq+0x65/0xa0
[<ffffffff81076d95>] ? irq_exit+0x85/0x90
[<ffffffff81516c95>] ? do_IRQ+0x75/0xf0
[<ffffffff8100b9d3>] ? ret_from_intr+0x0/0x11
<EOI>
CentOS 5: kernel-2.6.18-406.el5Whatever is the latest in CentOS
cat /etc/{centos,redhat}-release
uptime
uname -rv
rpm -q kernel
rpm -q rsync
df -h /boot
[root@isengard ~]# cat /etc/{centos,redhat}-release
CentOS release 6.7 (Final)
CentOS release 6.7 (Final)
[root@isengard ~]#
[root@isengard ~]#
[root@isengard ~]# uptime
18:47:25 up 449 days, 18:42, 1 user, load average: 1.82, 2.68, 2.45
[root@isengard ~]#
[root@isengard ~]#
[root@isengard ~]# uname -rv
2.6.32-358.el6.x86_64 #1 SMP Fri Feb 22 00:31:26 UTC 2013
[root@isengard ~]#
[root@isengard ~]#
[root@isengard ~]# rpm -q kernel
kernel-2.6.32-358.el6.x86_64
kernel-2.6.32-504.3.3.el6.x86_64
kernel-2.6.32-504.23.4.el6.x86_64
kernel-2.6.32-504.30.3.el6.x86_64
kernel-2.6.32-573.3.1.el6.x86_64
[root@isengard ~]#
[root@isengard ~]#
[root@isengard ~]#
[root@isengard ~]# rpm -q rsync
rsync-3.0.6-12.el6.x86_64
[root@isengard ~]#
[root@isengard ~]#
[root@isengard ~]#
[root@isengard ~]# df -h /boot
Filesystem Size Used Avail Use% Mounted on
/dev/sdn1 485M 157M 303M 35% /boot
[root@isengard ~]#
yum remove kernel-2.6.32-573.3.1.el6
yum update
I can't reboot that machine, it's the file server which also acts as storage for VMs, is there another way to load the kernel so it's seamless? How would an enterprise situation do it? I guess they'd have multiple servers that act in a shared storage/HA config? Something I do want to look into at some point but no money now.
You really can't reboot this machine? What is your plan if you have a power outage that outlasts your UPS?![]()