2.6.6-rc1-mm1 (and earlier): pdflush taking 100% cpu time (profile, .config etc. provided)
Subsystems (5)
at times, pdflush is taking over my system:
kernel 2.6.6-rc1-mm1
PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
31276 jurriaan 25 0 76772 59m 8764 R 98.7 2.0 776:39.05 mencoder
3479 root 25 0 0 0 0 R 96.7 0.0 134:29.90 pdflush
1 root 16 0 1520 516 1368 S 0.0 0.0 0:10.50 initprocs -----------memory---------- ---swap-- -----io---- --system-- ----cpu----
r b swpd free buff cache si so bi bo in cs us sy id wa 2 0 3064 7428 71536 2696036 0 0 94 154 201 196 53 5 41 1 2 0 3064 7108 71556 2696424 0 0 128 303 1146 182 50 50 0 0 2 0 3064 6540 71556 2697036 0 0 192 238 1129 155 50 50 0 0 2 0 3064 6284 71556 2697512 0 0 128 228 1127 278 50 50 0 0 2 0 3064 7692 71564 2696076 0 0 192 266 1134 161 50 50 0 0 2 0 3064 7108 71564 2696688 0 0 192 302 1140 203 50 50 0 0 2 0 3064 6660 71568 2697092 0 0 128 232 1128 145 50 50 0 0 2 0 3064 6276 71588 2697616 0 0 128 309 1141 178 50 50 0 0 2 0 3064 7108 71588 2696732 0 0 192 290 1139 169 50 50 0 0 2 0 3064 6596 71600 2697332 0 0 192 260 1133 165 50 50 0 0
output from
#!/bin/sh
readprofile -r
readprofile -M10
sleep 30
readprofile -n -v -m /proc/kallsyms | sort +0.50 -n | tail -20
c0170978 __iget 172 2.1235 c0178af1 writeback_acquire 220 8.8000 c0178b18 writeback_release 223 4.7447 c0142757 __pagevec_release 237 4.3889 c0172151 wake_up_inode 270 4.2857 c02909b0 atomic_dec_and_lock 271 2.3362 c0339333 ide_intr 308 0.6498 c013ec60 mapping_tagged 376 3.5140 c0178224 sync_sb_inodes 393 0.5406 c0177f49 __sync_single_inode 469 0.9214 c0103029 default_idle 525 10.7143 c0142aff pagevec_lookup_tag 587 9.7833 c0142432 lru_add_drain 689 6.2072 c0138943 wait_on_page_bit 1112 5.2701 c0138910 page_waitqueue 9518 186.6275 c0142579 release_pages 23597 49.3661 c0138a16 unlock_page 23666 281.7381 c013e945 clear_page_dirty_for_io 24791 255.5773 c01797c5 mpage_writepages 30701 40.0274
c0138ec9 find_get_pages_tag 181934 1166.2436
.config:CONFIG_X86=yCONFIG_MMU=yCONFIG_UID16=yCONFIG_GENERIC_ISA_DMA=yCONFIG_EXPERIMENTAL=yCONFIG_BROKEN=yCONFIG_BROKEN_ON_SMP=yCONFIG_SWAP=yCONFIG_SYSVIPC=yCONFIG_SYSCTL=yCONFIG_LOG_BUF_SHIFT=16CONFIG_IKCONFIG=yCONFIG_IKCONFIG_PROC=yCONFIG_KALLSYMS=yCONFIG_FUTEX=yCONFIG_EPOLL=yCONFIG_IOSCHED_NOOP=yCONFIG_IOSCHED_AS=yCONFIG_IOSCHED_DEADLINE=yCONFIG_IOSCHED_CFQ=yCONFIG_MODULES=yCONFIG_MODULE_UNLOAD=yCONFIG_OBSOLETE_MODPARM=yCONFIG_KMOD=yCONFIG_STOP_MACHINE=yCONFIG_X86_PC=yCONFIG_MPENTIUM4=yCONFIG_X86_CMPXCHG=yCONFIG_X86_XADD=yCONFIG_X86_L1_CACHE_SHIFT=7CONFIG_RWSEM_XCHGADD_ALGORITHM=yCONFIG_X86_WP_WORKS_OK=yCONFIG_X86_INVLPG=yCONFIG_X86_BSWAP=yCONFIG_X86_POPAD_OK=yCONFIG_X86_GOOD_APIC=yCONFIG_X86_INTEL_USERCOPY=yCONFIG_X86_USE_PPRO_CHECKSUM=yCONFIG_HPET_TIMER=yCONFIG_HPET_EMULATE_RTC=yCONFIG_SMP=yCONFIG_NR_CPUS=2CONFIG_SCHED_SMT=yCONFIG_PREEMPT=yCONFIG_X86_LOCAL_APIC=yCONFIG_X86_IO_APIC=yCONFIG_X86_TSC=yCONFIG_X86_MCE=yCONFIG_X86_MCE_NONFATAL=yCONFIG_X86_MCE_P4THERMAL=yCONFIG_MICROCODE=yCONFIG_X86_MSR=yCONFIG_X86_CPUID=yCONFIG_HIGHMEM4G=yCONFIG_HIGHMEM=yCONFIG_HIGHPTE=yCONFIG_MTRR=yCONFIG_HAVE_DEC_LOCK=yCONFIG_PM=yCONFIG_ACPI=yCONFIG_ACPI_BOOT=yCONFIG_ACPI_INTERPRETER=yCONFIG_ACPI_SLEEP=yCONFIG_ACPI_SLEEP_PROC_FS=yCONFIG_ACPI_AC=yCONFIG_ACPI_BUTTON=yCONFIG_ACPI_FAN=yCONFIG_ACPI_PROCESSOR=yCONFIG_ACPI_THERMAL=yCONFIG_ACPI_DEBUG=yCONFIG_ACPI_BUS=yCONFIG_ACPI_EC=yCONFIG_ACPI_POWER=yCONFIG_ACPI_PCI=yCONFIG_ACPI_SYSTEM=yCONFIG_PCI=yCONFIG_PCI_GOANY=yCONFIG_PCI_BIOS=yCONFIG_PCI_DIRECT=yCONFIG_PCI_MMCONFIG=yCONFIG_PCI_NAMES=yCONFIG_ISA=yCONFIG_BINFMT_ELF=yCONFIG_BINFMT_AOUT=yCONFIG_BINFMT_MISC=yCONFIG_PARPORT=yCONFIG_PARPORT_PC=yCONFIG_PARPORT_PC_CML1=yCONFIG_PARPORT_1284=yCONFIG_BLK_DEV_FD=yCONFIG_BLK_DEV_LOOP=yCONFIG_IDE=yCONFIG_BLK_DEV_IDE=yCONFIG_BLK_DEV_IDEDISK=yCONFIG_BLK_DEV_IDECD=yCONFIG_BLK_DEV_IDESCSI=yCONFIG_IDE_GENERIC=yCONFIG_BLK_DEV_IDEPCI=yCONFIG_IDEPCI_SHARE_IRQ=yCONFIG_BLK_DEV_IDEDMA_PCI=yCONFIG_IDEDMA_PCI_AUTO=yCONFIG_BLK_DEV_ADMA=yCONFIG_BLK_DEV_AMD74XX=yCONFIG_BLK_DEV_HPT366=yCONFIG_BLK_DEV_PIIX=yCONFIG_BLK_DEV_PDC202XX_OLD=yCONFIG_BLK_DEV_VIA82CXXX=yCONFIG_BLK_DEV_IDEDMA=yCONFIG_IDEDMA_AUTO=yCONFIG_SCSI=yCONFIG_SCSI_PROC_FS=yCONFIG_BLK_DEV_SD=yCONFIG_CHR_DEV_ST=yCONFIG_BLK_DEV_SR=yCONFIG_CHR_DEV_SG=yCONFIG_SCSI_MULTI_LUN=yCONFIG_SCSI_REPORT_LUNS=yCONFIG_SCSI_CONSTANTS=yCONFIG_SCSI_SPI_ATTRS=yCONFIG_SCSI_SYM53C8XX_2=yCONFIG_SCSI_SYM53C8XX_DMA_ADDRESSING_MODE=1CONFIG_SCSI_SYM53C8XX_DEFAULT_TAGS=16CONFIG_SCSI_SYM53C8XX_MAX_TAGS=64CONFIG_SCSI_QLA2XXX=yCONFIG_MD=yCONFIG_BLK_DEV_MD=yCONFIG_MD_LINEAR=yCONFIG_MD_RAID0=yCONFIG_MD_RAID1=yCONFIG_MD_RAID5=yCONFIG_NET=yCONFIG_PACKET=yCONFIG_UNIX=yCONFIG_INET=yCONFIG_IP_MULTICAST=yCONFIG_IP_ADVANCED_ROUTER=yCONFIG_IP_ROUTE_VERBOSE=yCONFIG_SYN_COOKIES=yCONFIG_NETDEVICES=yCONFIG_DUMMY=yCONFIG_NET_ETHERNET=yCONFIG_MII=yCONFIG_NET_TULIP=yCONFIG_TULIP=yCONFIG_TULIP_MWI=yCONFIG_TULIP_MMIO=yCONFIG_NET_PCI=yCONFIG_8139TOO=yCONFIG_8139_RXBUF_IDX=2CONFIG_TIGON3=mCONFIG_INPUT=yCONFIG_INPUT_MOUSEDEV=yCONFIG_INPUT_MOUSEDEV_PSAUX=yCONFIG_INPUT_MOUSEDEV_SCREEN_X=1024CONFIG_INPUT_MOUSEDEV_SCREEN_Y=768CONFIG_SOUND_GAMEPORT=yCONFIG_SERIO=yCONFIG_SERIO_I8042=yCONFIG_SERIO_SERPORT=yCONFIG_INPUT_KEYBOARD=yCONFIG_KEYBOARD_ATKBD=yCONFIG_INPUT_MOUSE=yCONFIG_MOUSE_PS2=yCONFIG_INPUT_MISC=yCONFIG_INPUT_PCSPKR=yCONFIG_VT=yCONFIG_VT_CONSOLE=yCONFIG_HW_CONSOLE=yCONFIG_SERIAL_8250=yCONFIG_SERIAL_8250_NR_UARTS=4CONFIG_SERIAL_CORE=yCONFIG_UNIX98_PTYS=yCONFIG_LEGACY_PTYS=yCONFIG_LEGACY_PTY_COUNT=256CONFIG_PRINTER=yCONFIG_WATCHDOG=yCONFIG_SOFT_WATCHDOG=yCONFIG_HW_RANDOM=yCONFIG_RTC=yCONFIG_AGP=yCONFIG_AGP_INTEL=yCONFIG_DRM=yCONFIG_DRM_RADEON=yCONFIG_DRM_MGA=yCONFIG_HANGCHECK_TIMER=yCONFIG_I2C=yCONFIG_I2C_CHARDEV=mCONFIG_I2C_ALGOBIT=yCONFIG_I2C_ISA=mCONFIG_I2C_VIAPRO=mCONFIG_I2C_SENSOR=mCONFIG_SENSORS_VIA686A=mCONFIG_SENSORS_W83781D=mCONFIG_SENSORS_EEPROM=mCONFIG_FB=yCONFIG_VIDEO_SELECT=yCONFIG_FB_MATROX=yCONFIG_FB_MATROX_G450=yCONFIG_FB_MATROX_G100=yCONFIG_FB_RADEON=yCONFIG_FB_RADEON_I2C=yCONFIG_FB_RADEON_DEBUG=yCONFIG_FB_3DFX=yCONFIG_VGA_CONSOLE=yCONFIG_DUMMY_CONSOLE=yCONFIG_FRAMEBUFFER_CONSOLE=yCONFIG_PCI_CONSOLE=yCONFIG_FONTS=y
CONFIG_FONT_SUN12x22=yCONFIG_LOGO=yCONFIG_LOGO_LINUX_MONO=yCONFIG_LOGO_LINUX_VGA16=yCONFIG_LOGO_LINUX_CLUT224=yCONFIG_SOUND=yCONFIG_SND=yCONFIG_SND_TIMER=yCONFIG_SND_PCM=yCONFIG_SND_HWDEP=yCONFIG_SND_RAWMIDI=yCONFIG_SND_SEQUENCER=yCONFIG_SND_OSSEMUL=yCONFIG_SND_MIXER_OSS=yCONFIG_SND_PCM_OSS=yCONFIG_SND_SEQUENCER_OSS=yCONFIG_SND_RTCTIMER=yCONFIG_SND_MPU401_UART=yCONFIG_SND_OPL3_LIB=yCONFIG_SND_AC97_CODEC=yCONFIG_SND_EMU10K1=yCONFIG_SND_CMIPCI=yCONFIG_SND_INTEL8X0=yCONFIG_SND_VIA82XX=yCONFIG_USB=yCONFIG_USB_DEVICEFS=yCONFIG_USB_EHCI_HCD=yCONFIG_USB_UHCI_HCD=yCONFIG_USB_PRINTER=yCONFIG_USB_HID=yCONFIG_USB_HIDINPUT=yCONFIG_EXT2_FS=yCONFIG_EXT3_FS=yCONFIG_EXT3_FS_XATTR=yCONFIG_JBD=yCONFIG_FS_MBCACHE=yCONFIG_REISERFS_FS=yCONFIG_XFS_FS=yCONFIG_ISO9660_FS=yCONFIG_JOLIET=yCONFIG_UDF_FS=yCONFIG_FAT_FS=yCONFIG_VFAT_FS=yCONFIG_NTFS_FS=yCONFIG_PROC_FS=yCONFIG_PROC_KCORE=yCONFIG_SYSFS=yCONFIG_TMPFS=yCONFIG_RAMFS=yCONFIG_NFS_FS=yCONFIG_NFS_V3=yCONFIG_LOCKD=yCONFIG_LOCKD_V4=yCONFIG_SUNRPC=yCONFIG_MSDOS_PARTITION=yCONFIG_NLS=yCONFIG_NLS_DEFAULT="iso8859-1"CONFIG_NLS_CODEPAGE_437=yCONFIG_NLS_ISO8859_1=yCONFIG_DEBUG_KERNEL=yCONFIG_EARLY_PRINTK=yCONFIG_MAGIC_SYSRQ=yCONFIG_FRAME_POINTER=yCONFIG_4KSTACKS=yCONFIG_SCHEDSTATS=yCONFIG_X86_FIND_SMP_CONFIG=yCONFIG_X86_MPPARSE=yCONFIG_CRC32=yCONFIG_X86_SMP=yCONFIG_X86_HT=yCONFIG_X86_BIOS_REBOOT=yCONFIG_X86_TRAMPOLINE=yCONFIG_X86_STD_RESOURCES=yCONFIG_PC=y
/proc/sys/vm/block_dump:0/proc/sys/vm/dirty_background_ratio:10/proc/sys/vm/dirty_expire_centisecs:3000/proc/sys/vm/dirty_ratio:40/proc/sys/vm/dirty_writeback_centisecs:500/proc/sys/vm/laptop_mode:0/proc/sys/vm/lower_zone_protection:0/proc/sys/vm/max_map_count:65536/proc/sys/vm/min_free_kbytes:957/proc/sys/vm/nr_pdflush_threads:2/proc/sys/vm/overcommit_memory:0/proc/sys/vm/overcommit_ratio:50/proc/sys/vm/page-cluster:3/proc/sys/vm/swappiness:60
MemTotal: 3107928 kB MemFree: 6540 kB Buffers: 72764 kB Cached: 2696996 kB SwapCached: 1240 kB Active: 342292 kB Inactive: 2625092 kB HighTotal: 2228160 kB HighFree: 1088 kB LowTotal: 879768 kB LowFree: 5452 kB SwapTotal: 4016232 kB SwapFree: 4013180 kB Dirty: 11868 kB Writeback: 0 kB Mapped: 212944 kB Slab: 107076 kB Committed_AS: 236832 kB PageTables: 1252 kB VmallocTotal: 114680 kB VmallocUsed: 27256 kB VmallocChunk: 87420 kB
Any ideas? I've seen this in earlier kernels, but this is the first time I've been able to profile it. Usually, when I leave the system like this for the night, pdlush has quieted down by morning. That does hamper the encoding speed, however.
Thanks,
Jurriaan
Signature · 3 lines
He was back, and they could like it or lump it. Simon R Green - Blue Moon Rising Debian (Unstable) GNU/Linux 2.6.6-rc1-mm1 2x6062 bogomips 2.53 2.32
Quoted 2 lines · Jurriaan
at times, pdflush is taking over my system:
yup, there's some logic error in there. If I could reproduce it, it would be fixed in a jiffy :(
Which filesystems are in active use at the time? reiserfs?
From: Andrew Morton <[email protected]>
Date: Tue, Apr 20, 2004 at 07:15:33PM -0700
Quoted 9 lines
Quoted 2 lines · Jurriaan
at times, pdflush is taking over my system:
yup, there's some logic error in there. If I could reproduce it, it would be fixed in a jiffy :(
Which filesystems are in active use at the time? reiserfs?
mostly, yes, also ext3, and raid-1 and linear raid. I can't always reproduce it either, unfortunately :-(
/dev/md3 on / type ext3 (rw,errors=remount-ro) proc on /proc type proc (rw)
sysfs on /sys type sysfs (rw)
devpts on /dev/pts type devpts (rw,gid=5,mode=620) tmpfs on /dev/shm type tmpfs (rw)
usbfs on /proc/bus/usb type usbfs (rw)/dev/md4 on /usr type ext3 (rw,errors=remount-ro) /dev/md2 on /var type ext3 (rw,errors=remount-ro) sysfs on /devices type sysfs (rw)/dev/md5 on /vmware type reiserfs (rw)/dev/md0 on /home type reiserfs (rw)/dev/md1 on /var/spool/news_binary type reiserfs (rw) /dev/hde1 on /space1 type reiserfs (rw,noexec,nosuid,nodev) /dev/hdk1 on /space2 type reiserfs (rw,noexec,nosuid,nodev) /dev/hdg1 on /space3 type reiserfs (rw,noexec,nosuid,nodev)
Personalities : [linear] [raid0] [raid1] [raid5]
md3 : active raid1 hdc1[0] hda1[1]
497856 blocks [2/2] [UU]
md4 : active raid1 hdc3[0] hda3[1]
8008320 blocks [2/2] [UU]
md2 : active raid1 hdc5[0] hda5[1]
16008640 blocks [2/2] [UU]
md5 : active raid1 hdc6[0] hda6[1]
8008256 blocks [2/2] [UU]
md1 : active linear hdi1[2] hdc7[1] hda7[0]
76909568 blocks 64k rounding
md0 : active raid1 hdc8[0] hda8[1]
62645312 blocks [2/2] [UU]
unused devices: <none>
Kind regards,
Jurriaan
Signature · 3 lines
But what can you do with it? ubiquitous cry from Linux-user partner Debian (Unstable) GNU/Linux 2.6.6-rc1-mm1 2x6062 bogomips load av: 2.07 2.06 2.17
Quoted 7 lines · Jurriaan
From: Andrew Morton <[email protected]>
Date: Tue, Apr 20, 2004 at 07:15:33PM -0700Quoted 4 lines
Quoted 2 lines · Jurriaan
at times, pdflush is taking over my system:
Quoted 8 lines
Quoted 5 lines
yup, there's some logic error in there. If I could reproduce it, it would be fixed in a jiffy :(
Which filesystems are in active use at the time? reiserfs?
mostly, yes, also ext3, and raid-1 and linear raid. I can't always reproduce it either, unfortunately :-(
I'm suspecting that reiserfs_writepage() is redirty pages rather than writing them and writeback is livelocking.
Could you try this?
---
25-akpm/fs/buffer.c | 2 +- 25-akpm/fs/ext3/inode.c | 6 +++--- 25-akpm/fs/ntfs/aops.c | 15 +++------------ 25-akpm/fs/reiserfs/inode.c | 2 +- 25-akpm/include/linux/mm.h | 2 ++ 25-akpm/mm/page-writeback.c | 12 ++++++++++++ 6 files changed, 22 insertions(+), 17 deletions(-)
fs/buffer.c renamed from fs/buffer.c~writeback-livelock-fix +1 -1
@@ -1826,7 +1826,7 @@ static int __block_write_full_page(strucif (wbc->sync_mode !=WB_SYNC_NONE || !wbc->nonblocking) {lock_buffer (bh); }else if (test_set_buffer_locked (bh)) { -__set_page_dirty_nobuffers (page); +redirty_page_for_writepage (wbc, page);continue ; }if (test_clear_buffer_dirty (bh)) {
fs/ext3/inode.c renamed from fs/ext3/inode.c~writeback-livelock-fix +3 -3
@@ -1311,7 +1311,7 @@ static int ext3_ordered_writepage(structreturn ret; out_fail: -__set_page_dirty_nobuffers (page); +redirty_page_for_writepage (wbc, page);unlock_page (page);return ret; }
@@ -1340,7 +1340,7 @@ static int ext3_writeback_writepage(strureturn ret; out_fail: -__set_page_dirty_nobuffers (page); +redirty_page_for_writepage (wbc, page);unlock_page (page);return ret; }
@@ -1396,7 +1396,7 @@ out:return ret; no_write: -__set_page_dirty_nobuffers (page); +redirty_page_for_writepage (wbc, page); out_unlock:unlock_page (page); goto out;
fs/ntfs/aops.c renamed from fs/ntfs/aops.c~writeback-livelock-fix +3 -12
@@ -499,10 +499,7 @@ static int ntfs_write_block(struct page* Put the page back on mapping->dirty_pages, but leave its * buffer's dirty state as-is. */ -// FIXME: Once Andrew's -EAGAIN patch goes in, remove the -// __set_page_dirty_nobuffers(page) and return -EAGAIN instead -// of zero. -__set_page_dirty_nobuffers (page); +redirty_page_for_writepage (wbc, page);unlock_page (page);return 0 ; }
@@ -733,10 +730,7 @@ lock_retry_remap:* Put the page back on mapping->dirty_pages, but * leave its buffer's dirty state as-is. */ -// FIXME: Once Andrew's -EAGAIN patch goes in, remove -// the __set_page_dirty_nobuffers(page) and set err to -// -EAGAIN instead of zero. -__set_page_dirty_nobuffers (page); +redirty_page_for_writepage (wbc, page); err =0 ; }else SetPageError (page);
@@ -986,10 +980,7 @@ err_out:* Put the page back on mapping->dirty_pages, but leave its * buffer's dirty state as-is. */ -// FIXME: Once Andrew's -EAGAIN patch goes in, remove the -// __set_page_dirty_nobuffers(page) and set err to -EAGAIN -// instead of zero. -__set_page_dirty_nobuffers (page); +redirty_page_for_writepage (wbc, page); err =0 ; }else { ntfs_error(vi->i_sb, "Resident attribute write failed with "
fs/reiserfs/inode.c renamed from fs/reiserfs/inode.c~writeback-livelock-fix +1 -1
@@ -2112,7 +2112,7 @@ static int reiserfs_write_full_page(strulock_buffer (bh); }else {if (test_set_buffer_locked (bh)) { -__set_page_dirty_nobuffers (page); +redirty_page_for_writepage (wbc, page);continue ; } }
include/linux/mm.h renamed from include/linux/mm.h~writeback-livelock-fix +2 -0
@@ -499,6 +499,8 @@ int get_user_pages(struct task_struct *tint __set_page_dirty_buffers (struct page *page);int __set_page_dirty_nobuffers (struct page *page); +int redirty_page_for_writepage (struct writeback_control *wbc, +struct page *page);int FASTCALL (set_page_dirty (struct page *page));int set_page_dirty_lock (struct page *page);int clear_page_dirty_for_io (struct page *page);
mm/page-writeback.c renamed from mm/page-writeback.c~writeback-livelock-fix +12 -0
@@ -581,6 +581,18 @@ int __set_page_dirty_nobuffers(struct paEXPORT_SYMBOL (__set_page_dirty_nobuffers);/* +* When a writepage implementation decides that it doesn't want to write this +* page for some reason, it should redirty the locked page via +* redirty_page_for_writepage() and it should then unlock the page and return 0 +*/ +int redirty_page_for_writepage (struct writeback_control *wbc,struct page *page) +{ + wbc->pages_skipped++; +return __set_page_dirty_nobuffers (page); +} +EXPORT_SYMBOL (redirty_page_for_writepage); + +/* * If the mapping doesn't provide a set_page_dirty a_op, then * just fall through and assume that it wants buffer_heads. */
_