Skip to contentSkip to conversation

Slow long-term increase in dirty pages

DISCUSSION Paul Evans 2 participants 3 messages Mar 18, 2009 – Mar 19, 2009

Subsystems (10)
lore
Share & tools
lore

Paul Evans

author
#0 lore raw

We have a server whose dirty page count keeps increasing all the time, to the point where 'sync' takes ages to flush the pages:

  root@freehand:~# time sync
  real    1m15.570s
  user    0m0.000s
  sys     0m0.052s

We have some graphs of the dirty page count, as captured from /proc/vmstat's "nr_dirty" entry:

http://opensource.mxtelecom.com/tmp/freehand-dirty-day.png
http://opensource.mxtelecom.com/tmp/freehand-dirty-week.png

I have tuned the dirty page flushing sysctls to the following:

  root@freehand:~# for F in /proc/sys/vm/dirty_*; do echo -n "$F: "; cat $F; done
  /proc/sys/vm/dirty_background_ratio: 1
  /proc/sys/vm/dirty_expire_centisecs: 3000
  /proc/sys/vm/dirty_ratio: 3
  /proc/sys/vm/dirty_writeback_centisecs: 500

The role of the machine itself is that it performings large amount of kernel iptables routing/firewalling traffic, and runs a set of apache servers as HTTP<->Tomcat gateways.

  root@freehand:~# uname -r
  2.6.27-fes

(this is a build of stock 2.6.27 source, with some extra iptables patches. There shouldn't be anything mm-related here)

By my understanding of the dirty page flush algorithm, we shouldn't be accumulating these pages all the time; any page older than 30 seconds ought to be written out, yes?

If we manually 'sync', as above, then the count drops to zero, but then slowly starts ramping up again as observed.

As a temporary workaround I've put 'sync' in cron every 10 minutes, but is there some more tuning I can do; or at least probing to see where these pages are being accumulated from?

Signature · 4 lines
Paul Evans <[email protected]>
Tel: +44 (0) 845 666 7778
Fax: +44 (0) 870 163 4694
http://www.mxtelecom.com
#1 lore raw
Quoted 37 lines · Paul Evans

We have a server whose dirty page count keeps increasing all the time, to the point where 'sync' takes ages to flush the pages:

  root@freehand:~# time sync
  real    1m15.570s
  user    0m0.000s
  sys     0m0.052s

We have some graphs of the dirty page count, as captured from /proc/vmstat's "nr_dirty" entry:

http://opensource.mxtelecom.com/tmp/freehand-dirty-day.png
http://opensource.mxtelecom.com/tmp/freehand-dirty-week.png

I have tuned the dirty page flushing sysctls to the following:

  root@freehand:~# for F in /proc/sys/vm/dirty_*; do echo -n "$F: "; cat $F; done
  /proc/sys/vm/dirty_background_ratio: 1
  /proc/sys/vm/dirty_expire_centisecs: 3000
  /proc/sys/vm/dirty_ratio: 3
  /proc/sys/vm/dirty_writeback_centisecs: 500

The role of the machine itself is that it performings large amount of kernel iptables routing/firewalling traffic, and runs a set of apache servers as HTTP<->Tomcat gateways.

  root@freehand:~# uname -r
  2.6.27-fes

(this is a build of stock 2.6.27 source, with some extra iptables patches. There shouldn't be anything mm-related here)

By my understanding of the dirty page flush algorithm, we shouldn't be accumulating these pages all the time; any page older than 30 seconds ought to be written out, yes?

Quoted 6 lines

If we manually 'sync', as above, then the count drops to zero, but then slowly starts ramping up again as observed.

As a temporary workaround I've put 'sync' in cron every 10 minutes, but is there some more tuning I can do; or at least probing to see where these pages are being accumulated from?

Hi Paul,

The attached filecache patch shall help identify the dirty files/pages.

Usage:

        # run patched kernel
        modprobe filecache
        cat /proc/filecache

The dirty files will have the 'D' flag in its "state" field.

Thanks,
Fengguang

---
 Documentation/filesystems/proc.txt |   83 ++
 fs/Kconfig                         |   30 
 fs/dcache.c                        |    5 
 fs/inode.c                         |   25 
 fs/open.c                          |    1 
 fs/proc/Makefile                   |    3 
 fs/proc/filecache.c                | 1045 +++++++++++++++++++++++++++
 fs/super.c                         |    3 
 include/linux/fs.h                 |   14 
 include/linux/mm.h                 |    1 
 lib/radix-tree.c                   |    2 
 mm/page_alloc.c                    |    2 
 mm/swap_state.c                    |    1 
 mm/vmscan.c                        |    1 
 14 files changed, 1211 insertions(+), 5 deletions(-)
include/linux/mm.h +1 -0
@@ -26,6 +26,7 @@ extern unsigned long max_mapnr;
 extern unsigned long num_physpages;
 extern void * high_memory;
 extern int page_cluster;
+extern char * const zone_names[];
 
 #ifdef CONFIG_SYSCTL
 extern int sysctl_legacy_va_layout;
mm/page_alloc.c +1 -1
@@ -104,7 +104,7 @@ int sysctl_lowmem_reserve_ratio[MAX_NR_Z
 
 EXPORT_SYMBOL(totalram_pages);
 
-static char * const zone_names[MAX_NR_ZONES] = {
+char * const zone_names[MAX_NR_ZONES] = {
 #ifdef CONFIG_ZONE_DMA
 	 "DMA",
 #endif
fs/dcache.c +4 -1
@@ -1933,7 +1933,10 @@ char *__d_path(const struct path *path, 
 
 		if (dentry == root->dentry && vfsmnt == root->mnt)
 			break;
-		if (dentry == vfsmnt->mnt_root || IS_ROOT(dentry)) {
+		if (unlikely(!vfsmnt)) {
+			if (IS_ROOT(dentry))
+				break;
+		} else if (dentry == vfsmnt->mnt_root || IS_ROOT(dentry)) {
 			/* Global root? */
 			if (vfsmnt->mnt_parent == vfsmnt) {
 				goto global_root;
lib/radix-tree.c +0 -2
@@ -564,7 +564,6 @@ out:
 }
 EXPORT_SYMBOL(radix_tree_tag_clear);
 
-#ifndef __KERNEL__	/* Only the test harness uses this at present */
 /**
  * radix_tree_tag_get - get a tag on a radix tree node
  * @root:		radix tree root
@@ -627,7 +626,6 @@ int radix_tree_tag_get(struct radix_tree
 	}
 }
 EXPORT_SYMBOL(radix_tree_tag_get);
-#endif
 
 /**
  *	radix_tree_next_hole    -    find the next hole (not-present entry)
fs/inode.c +25 -0
@@ -82,6 +82,10 @@ static struct hlist_head *inode_hashtabl
  */
 DEFINE_SPINLOCK(inode_lock);
 
+EXPORT_SYMBOL(inode_in_use);
+EXPORT_SYMBOL(inode_unused);
+EXPORT_SYMBOL(inode_lock);
+
 /*
  * iprune_mutex provides exclusion between the kswapd or try_to_free_pages
  * icache shrinking path, and the umount path.  Without this exclusion,
@@ -108,6 +112,14 @@ static void wake_up_inode(struct inode *
 	wake_up_bit(&inode->i_state, __I_LOCK);
 }
 
+static inline void inode_created_by(struct inode *inode, struct task_struct *task)
+{
+#ifdef CONFIG_PROC_FILECACHE_EXTRAS
+	inode->i_cuid = task->uid;
+	memcpy(inode->i_comm, task->comm, sizeof(task->comm));
+#endif
+}
+
 static struct inode *alloc_inode(struct super_block *sb)
 {
 	static const struct address_space_operations empty_aops;
@@ -183,6 +195,7 @@ static struct inode *alloc_inode(struct 
 		}
 		inode->i_private = NULL;
 		inode->i_mapping = mapping;
+		inode_created_by(inode, current);
 	}
 	return inode;
 }
@@ -247,6 +260,8 @@ void __iget(struct inode * inode)
 	inodes_stat.nr_unused--;
 }
 
+EXPORT_SYMBOL(__iget);
+
 /**
  * clear_inode - clear an inode
  * @inode: inode to clear
@@ -1353,6 +1368,16 @@ void inode_double_unlock(struct inode *i
 }
 EXPORT_SYMBOL(inode_double_unlock);
 
+
+struct hlist_head * get_inode_hash_budget(unsigned long index)
+{
+       if (index >= (1 << i_hash_shift))
+               return NULL;
+
+       return inode_hashtable + index;
+}
+EXPORT_SYMBOL_GPL(get_inode_hash_budget);
+
 static __initdata unsigned long ihash_entries;
 static int __init set_ihash_entries(char *str)
 {
fs/super.c +3 -0
@@ -45,6 +45,9 @@
 LIST_HEAD(super_blocks);
 DEFINE_SPINLOCK(sb_lock);
 
+EXPORT_SYMBOL(super_blocks);
+EXPORT_SYMBOL(sb_lock);
+
 /**
  *	alloc_super	-	create new superblock
  *	@type:	filesystem type superblock should belong to
mm/vmscan.c +1 -0
@@ -229,6 +229,7 @@ unsigned long shrink_slab(unsigned long 
 	up_read(&shrinker_rwsem);
 	return ret;
 }
+EXPORT_SYMBOL(shrink_slab);
 
 /* Called without lock on whether page is mapped, so answer is unstable */
 static inline int page_mapping_inuse(struct page *page)
mm/swap_state.c +1 -0
@@ -44,6 +44,7 @@ struct address_space swapper_space = {
 	.i_mmap_nonlinear = LIST_HEAD_INIT(swapper_space.i_mmap_nonlinear),
 	.backing_dev_info = &swap_backing_dev_info,
 };
+EXPORT_SYMBOL_GPL(swapper_space);
 
 #define INC_CACHE_INFO(x)	do { swap_cache_info.x++; } while (0)
 
Documentation/filesystems/proc.txt +83 -0

95 lines not on this page · raw message

fs/proc/filecache.c new file +1045 -0

1,045 lines not on this page · raw message

include/linux/fs.h +14 -0
@@ -681,6 +681,12 @@ struct inode {
 	void			*i_security;
 #endif
 	void			*i_private; /* fs or device private pointer */
+
+#ifdef CONFIG_PROC_FILECACHE_EXTRAS
+	unsigned int		i_access_count;	/* opened how many times? */
+	uid_t			i_cuid;		/* opened first by which user? */
+	char			i_comm[16];	/* opened first by which app? */
+#endif
 };
 
 /*
@@ -769,6 +775,13 @@ static inline unsigned imajor(const stru
 	return MAJOR(inode->i_rdev);
 }
 
+static inline void inode_accessed(struct inode *inode)
+{
+#ifdef CONFIG_PROC_FILECACHE_EXTRAS
+	inode->i_access_count++;
+#endif
+}
+
 extern struct block_device *I_BDEV(struct inode *inode);
 
 struct fown_struct {
@@ -1843,6 +1856,7 @@ extern void remove_inode_hash(struct ino
 static inline void insert_inode_hash(struct inode *inode) {
 	__insert_inode_hash(inode, inode->i_ino);
 }
+struct hlist_head * get_inode_hash_budget(unsigned long index);
 
 extern struct file * get_empty_filp(void);
 extern void file_move(struct file *f, struct list_head *list);
fs/open.c +1 -0
@@ -828,6 +828,7 @@ static struct file *__dentry_open(struct
 			goto cleanup_all;
 	}
 
+	inode_accessed(inode);
 	f->f_flags &= ~(O_CREAT | O_EXCL | O_NOCTTY | O_TRUNC);
 
 	file_ra_state_init(&f->f_ra, f->f_mapping->host->i_mapping);
fs/Kconfig +30 -0
@@ -979,6 +979,36 @@ config CONFIGFS_FS
 	  Both sysfs and configfs can and should exist together on the
 	  same system. One is not a replacement for the other.
 
+config PROC_FILECACHE
+	tristate "/proc/filecache support"
+	default m
+	depends on PROC_FS
+	help
+	  This option creates a file /proc/filecache which enables one to
+	  query/drop the cached files in memory.
+
+	  A quick start guide:
+
+	  # echo 'ls' > /proc/filecache
+	  # head /proc/filecache
+
+	  # echo 'cat /bin/bash' > /proc/filecache
+	  # head /proc/filecache
+
+	  # echo 'drop pagecache' > /proc/filecache
+	  # echo 'drop slabcache' > /proc/filecache
+
+	  For more details, please check Documentation/filesystems/proc.txt .
+
+	  It can be a handy tool for sysadms and desktop users.
+
+config PROC_FILECACHE_EXTRAS
+	bool "track extra states"
+	default y
+	depends on PROC_FILECACHE
+	help
+	  Track extra states that costs a little more time/space.
+
 endmenu
 
 menu "Miscellaneous filesystems"
fs/proc/Makefile +2 -1
@@ -2,7 +2,8 @@
 # Makefile for the Linux proc filesystem routines.
 #
 
-obj-$(CONFIG_PROC_FS) += proc.o
+obj-$(CONFIG_PROC_FS)		+= proc.o
+obj-$(CONFIG_PROC_FILECACHE)	+= filecache.o
 
 proc-y			:= nommu.o task_nommu.o
 proc-$(CONFIG_MMU)	:= mmu.o task_mmu.o
author
#2 lore raw
Quoted 9 lines · On Thu, 19 Mar 2009 12:47:51 +0800 Wu Fengguang <[email protected]> wrote:

The attached filecache patch shall help identify the dirty files/pages.

Usage:

        # run patched kernel
        modprobe filecache
        cat /proc/filecache

The dirty files will have the 'D' flag in its "state" field.

Wow, that looks very useful. I'll have a play with it and see if it yields anything of interest.

Thanks,

Signature · 4 lines
Paul Evans <[email protected]>
Tel: +44 (0) 845 666 7778
Fax: +44 (0) 870 163 4694
http://www.mxtelecom.com