Skip to contentSkip to conversation

Adaptive read-ahead v4

PATCH In review WU Fengguang 4 participants 7 messages Oct 15, 2005 – Oct 21, 2005

Subsystems (10)
lore
Share & tools
lore
#0 lore raw

PATCH 1/1 Adaptive read-ahead v4

This is the 4th version of the adaptive read-ahead patch.

New features:
- detect and prevent most read-ahead thrashing
- support database workload
- support reading backward
- radix tree lookup look-aside cache

It has evolved into a near complete solution over time. I'll split it up and describe each component at the next time.

Quoted 2 lines · Andrew Morton

Try one process serving 1000 files slowly on a 64MB machine. Something like that.

Dreams come true :)
The patch can now support 1000 slow reads on 64MB hands down. The newly added thrashing protection method allows one to force a large read-ahead request size without causing serious thrashing.

The test data below were collected with readahead_ratio=200.

Regards,
Wu Fengguang

/proc/meminfo
=============

MemTotal:        60664 kB
MemFree:          1556 kB
Buffers:         22020 kB
Cached:          17760 kB
SwapCached:          0 kB
Active:          14720 kB
Inactive:        27792 kB

iostat -x 100
=============

rrqm/s    r/s   rsec/s     rkB/s    avgrq-sz avgqu-sz   await  svctm  %util
  0.17  65.64  2107.94   1053.97       29.67     1.13    9.18   2.26  27.95
  0.14  56.52  1911.22    955.61       30.11     1.23   10.46   2.15  25.31
  0.23  61.12  2104.96   1052.48       30.62     1.40   11.74   2.05  24.48
  0.21  55.34  1884.40    942.20       30.09     1.39   11.93   2.01  23.50
  0.09  55.66  1928.88    964.44       30.58     1.12    9.61   1.74  20.20
  0.18  60.02  2044.43   1022.21       30.16     1.42   11.92   1.95  23.20
  0.25  55.22  1913.39    956.70       30.28     1.18   10.05   2.14  25.06
  0.10  59.66  2019.96   1009.98       30.25     1.34   11.33   2.35  27.77
  0.22  58.43  1964.48    982.24       30.21     1.10    9.22   2.07  24.76

/proc/vmstat
============
cache_miss 116127
readrandom 3
pgreadrandom 3
readahead_rescue 76974
pgreadahead_rescue 155047
readahead_end 2
readahead 118988
readahead_return 1716
readahead_eof 1120
pgreadahead 517119
pgreadahead_hit 512447
pgreadahead_eof 4421
ra_newfile 1151
ra_newfile_return 1018
ra_newfile_eof 91
pgra_newfile 4428
pgra_newfile_hit 4353
pgra_newfile_eof 188
ra_state 115483
ra_state_return 697
ra_state_eof 1012
pgra_state 505918
pgra_state_hit 501349
pgra_state_eof 4180
ra_context 12
ra_context_return 0
ra_context_eof 0
pgra_context 41
pgra_context_hit 41
pgra_context_eof 0
ra_contexta 0
ra_contexta_return 0
ra_contexta_eof 0
pgra_contexta 0
pgra_contexta_hit 0
pgra_contexta_eof 0
ra_backward 0
ra_backward_return 0
ra_backward_eof 0
pgra_backward 0
pgra_backward_hit 0
pgra_backward_eof 0
ra_random 2342
ra_random_return 1
ra_random_eof 17
pgra_random 6732
pgra_random_hit 6704
pgra_random_eof 53

oprofile
========
The two thrashing detection/prevention functions are marked with leading '+++++++'. The overheads are high. But there's good room for speed up.

diffprofile oprofile.50.4k oprofile.100.4k
------------------------------------------

       109    19.5% dnotify_parent
       103     6.3% ll_rw_block
        89    13.7% add_to_page_cache
++++++++83     0.0% save_chunk
        80    15.7% unlock_buffer
        64     8.5% shrink_list
        54     1.5% __copy_to_user_ll
        54    10.5% __brelse
        50    50.5% radix_tree_lookup_node
        41  1366.7% kunmap_atomic
        35   102.9% do_wp_page
        33    29.2% vfs_write
        25    25.0% megaraid_mbox_build_cmd
++++++++24     0.0% rescue_ra_pages
        24    53.3% __do_softirq
        21     4.0% inotify_dentry_parent_queue_event

       -21   -15.0% load_balance
       -22   -47.8% lock_timer_base
       -23    -7.1% find_busiest_group
       -23    -7.5% unlock_page
       -24    -5.4% __mod_page_state
       -24   -15.7% current_fs_time
       -25   -16.1% __pagevec_release_nonlru
       -25    -5.6% radix_tree_delete
       -26    -8.0% __do_page_cache_readahead
       -26   -20.5% kmem_cache_alloc
       -27    -3.7% default_idle
       -31   -13.1% release_pages
       -32    -7.5% mpage_end_io_read
       -36   -12.1% mark_offset_pmtmr
       -45    -5.0% delay_pmtmr
       -54    -8.9% find_get_page
 drivers/block/loop.c       |    5 
 include/linux/fs.h         |   21 
 include/linux/mm.h         |   12 
 include/linux/mm_inline.h  |    1 
 include/linux/mmzone.h     |   51 +
 include/linux/page-flags.h |   71 ++
 include/linux/radix-tree.h |   19 
 include/linux/sysctl.h     |    1 
 kernel/sysctl.c            |   11 
 lib/radix-tree.c           |   71 ++
 mm/filemap.c               |   70 +-
 mm/page_alloc.c            |  154 ++++-
 mm/readahead.c             | 1384 ++++++++++++++++++++++++++++++++++++++++++++-
 mm/swap.c                  |   12 
 mm/vmscan.c                |   45 +
 15 files changed, 1876 insertions(+), 52 deletions(-)
drivers/block/loop.c +5 -0
diff -rup linux-2.6.14-rc4-git4-orig/drivers/block/loop.c linux-2.6.14-rc4-git4/drivers/block/loop.c
@@ -768,6 +768,11 @@ static int loop_set_fd(struct loop_devic
 
 	mapping = file->f_mapping;
 	inode = mapping->host;
+	/*
+	 * The upper layer should already do proper read-ahead,
+	 * unlimited read-ahead here only ruins the cache hit rate.
+	 */
+	file->f_ra.ra_pages = 32 >> (PAGE_CACHE_SHIFT - 10);
 
 	if (!(file->f_mode & FMODE_WRITE))
 		lo_flags |= LO_FLAGS_READ_ONLY;
include/linux/fs.h +20 -1
diff -rup linux-2.6.14-rc4-git4-orig/include/linux/fs.h linux-2.6.14-rc4-git4/include/linux/fs.h
@@ -562,17 +562,36 @@ struct file_ra_state {
 	unsigned long start;		/* Current window */
 	unsigned long size;
 	unsigned long flags;		/* ra flags RA_FLAG_xxx*/
-	unsigned long cache_hit;	/* cache hit count*/
+	uint64_t      cache_hit;	/* cache hit count*/
 	unsigned long prev_page;	/* Cache last read() position */
 	unsigned long ahead_start;	/* Ahead window */
 	unsigned long ahead_size;
 	unsigned long ra_pages;		/* Maximum readahead window */
 	unsigned long mmap_hit;		/* Cache hit stat for mmap accesses */
 	unsigned long mmap_miss;	/* Cache miss stat for mmap accesses */
+
+	unsigned long la_index;
+	unsigned long ra_index;
+	unsigned long lookahead_index;
+	unsigned long readahead_index;
+	unsigned long nr_page_aging;
 };
 #define RA_FLAG_MISS 0x01	/* a cache miss occured against this file */
 #define RA_FLAG_INCACHE 0x02	/* file is already in cache */
 
+#define RA_CLASS_SHIFT 3
+#define RA_CLASS_MASK  ((1 << RA_CLASS_SHIFT) - 1)
+enum file_ra_class { /* the same order must be kept in page_state */
+	RA_CLASS_NEWFILE = 1,
+	RA_CLASS_STATE,
+	RA_CLASS_CONTEXT,
+	RA_CLASS_CONTEXT_ACCELERATED,
+	RA_CLASS_BACKWARD,
+	/* RA_CLASS_AROUND, */
+	RA_CLASS_RANDOM,
+	RA_CLASS_END,
+};
+
 struct file {
 	struct list_head	f_list;
 	struct dentry		*f_dentry;
include/linux/mm.h +11 -1
diff -rup linux-2.6.14-rc4-git4-orig/include/linux/mm.h linux-2.6.14-rc4-git4/include/linux/mm.h
@@ -875,7 +875,7 @@ extern int filemap_populate(struct vm_ar
 int write_one_page(struct page *page, int wait);
 
 /* readahead.c */
-#define VM_MAX_READAHEAD	128	/* kbytes */
+#define VM_MAX_READAHEAD	1024	/* kbytes */
 #define VM_MIN_READAHEAD	16	/* kbytes (includes current page) */
 #define VM_MAX_CACHE_HIT    	256	/* max pages in a row in cache before
 					 * turning readahead off */
@@ -892,6 +892,16 @@ unsigned long  page_cache_readahead(stru
 void handle_ra_miss(struct address_space *mapping, 
 		    struct file_ra_state *ra, pgoff_t offset);
 unsigned long max_sane_readahead(unsigned long nr);
+unsigned long
+page_cache_readahead_adaptive(struct address_space *mapping,
+			struct file_ra_state *ra, struct file *filp,
+			struct page *prev_page, struct page *page,
+			unsigned long first_index,
+			unsigned long index, unsigned long last_index);
+void fastcall ra_access(struct file_ra_state *ra, struct page *page);
+int rescue_ra_pages(struct list_head *page_list, struct list_head *save_list);
+int save_chunk(struct page *head, struct page *tail,
+		struct list_head *save_list);
 
 /* Do stack extension */
 extern int expand_stack(struct vm_area_struct * vma, unsigned long address);
include/linux/mm_inline.h +1 -0
diff -rup linux-2.6.14-rc4-git4-orig/include/linux/mm_inline.h linux-2.6.14-rc4-git4/include/linux/mm_inline.h
@@ -11,6 +11,7 @@ add_page_to_inactive_list(struct zone *z
 {
 	list_add(&page->lru, &zone->inactive_list);
 	zone->nr_inactive++;
+	zone->nr_page_aging++;
 }
 
 static inline void
include/linux/mmzone.h +51 -0

63 lines not on this page · raw message

include/linux/page-flags.h +71 -0

89 lines not on this page · raw message

include/linux/radix-tree.h +18 -1
diff -rup linux-2.6.14-rc4-git4-orig/include/linux/radix-tree.h linux-2.6.14-rc4-git4/include/linux/radix-tree.h
@@ -45,8 +45,10 @@ do {									\
 } while (0)
 
 int radix_tree_insert(struct radix_tree_root *, unsigned long, void *);
-void *radix_tree_lookup(struct radix_tree_root *, unsigned long);
 void *radix_tree_delete(struct radix_tree_root *, unsigned long);
+void *radix_tree_lookup(struct radix_tree_root *, unsigned long);
+void *radix_tree_lookup_node(struct radix_tree_root *, unsigned long,
+							unsigned int);
 unsigned int
 radix_tree_gang_lookup(struct radix_tree_root *root, void **results,
 			unsigned long first_index, unsigned int max_items);
@@ -68,4 +70,19 @@ static inline void radix_tree_preload_en
 	preempt_enable();
 }
 
+/*
+ * Support access patterns with locality.
+ */
+struct radix_tree_cache {
+	unsigned long first_index;
+	struct radix_tree_node *tree_node;
+};
+
+void radix_tree_cache_init(struct radix_tree_cache *cache);
+void *radix_tree_cache_lookup(struct radix_tree_root *,
+				struct radix_tree_cache *, unsigned long);
+int radix_tree_cache_size(struct radix_tree_cache *cache);
+int radix_tree_cache_count(struct radix_tree_cache *cache);
+int radix_tree_cache_first_index(struct radix_tree_cache *cache);
+
 #endif /* _LINUX_RADIX_TREE_H */
include/linux/sysctl.h +1 -0
diff -rup linux-2.6.14-rc4-git4-orig/include/linux/sysctl.h linux-2.6.14-rc4-git4/include/linux/sysctl.h
@@ -180,6 +180,7 @@ enum
 	VM_VFS_CACHE_PRESSURE=26, /* dcache/icache reclaim pressure */
 	VM_LEGACY_VA_LAYOUT=27, /* legacy/compatibility virtual address space layout */
 	VM_SWAP_TOKEN_TIMEOUT=28, /* default time for token time out */
+	VM_READAHEAD_RATIO=29, /* percent of read-ahead size to thrashing-threshold */
 };
 
 
kernel/sysctl.c +11 -0
diff -rup linux-2.6.14-rc4-git4-orig/kernel/sysctl.c linux-2.6.14-rc4-git4/kernel/sysctl.c
@@ -67,6 +67,7 @@ extern int min_free_kbytes;
 extern int printk_ratelimit_jiffies;
 extern int printk_ratelimit_burst;
 extern int pid_max_min, pid_max_max;
+extern int readahead_ratio;
 
 #if defined(CONFIG_X86_LOCAL_APIC) && defined(CONFIG_X86)
 int unknown_nmi_panic;
@@ -849,6 +850,16 @@ static ctl_table vm_table[] = {
 		.strategy	= &sysctl_jiffies,
 	},
 #endif
+	{
+		.ctl_name	= VM_READAHEAD_RATIO,
+		.procname	= "readahead_ratio",
+		.data		= &readahead_ratio,
+		.maxlen		= sizeof(readahead_ratio),
+		.mode		= 0644,
+		.proc_handler	= &proc_dointvec,
+		.strategy	= &sysctl_intvec,
+		.extra1		= &zero,
+	},
 	{ .ctl_name = 0 }
 };
 
lib/radix-tree.c +62 -9

91 lines not on this page · raw message

mm/filemap.c +59 -11

187 lines not on this page · raw message

mm/page_alloc.c +139 -15

238 lines not on this page · raw message

mm/readahead.c +1378 -6

1,431 lines not on this page · raw message

mm/swap.c +10 -2
diff -rup linux-2.6.14-rc4-git4-orig/mm/swap.c linux-2.6.14-rc4-git4/mm/swap.c
@@ -96,6 +96,8 @@ int rotate_reclaimable_page(struct page 
 	return 0;
 }
 
+extern int readahead_ratio;
+
 /*
  * FIXME: speed this up?
  */
@@ -122,8 +124,13 @@ void fastcall activate_page(struct page 
  */
 void fastcall mark_page_accessed(struct page *page)
 {
-	if (!PageActive(page) && PageReferenced(page) && PageLRU(page)) {
-		activate_page(page);
+	if (!PageActive(page) && !PageActivate(page) &&
+			PageReferenced(page) && PageLRU(page)) {
+		if (readahead_ratio > 9 || (readahead_ratio & 1)) {
+			page_zone(page)->nr_page_aging++;
+			SetPageActivate(page);
+		} else
+			activate_page(page);
 		ClearPageReferenced(page);
 	} else if (!PageReferenced(page)) {
 		SetPageReferenced(page);
@@ -299,6 +306,7 @@ void __pagevec_lru_add(struct pagevec *p
 			if (zone)
 				spin_unlock_irq(&zone->lru_lock);
 			zone = pagezone;
+			update_page_age(zone);
 			spin_lock_irq(&zone->lru_lock);
 		}
 		if (TestSetPageLRU(page))
mm/vmscan.c +39 -6

110 lines not on this page · raw message

#1 lore raw
Quoted 53 lines · WU Fengguang
include/linux/page-flags.h +48 -0
@@ -131,6 +133,63 @@ struct page_state {
 
 	unsigned long pgrotated;	/* pages rotated to tail of the LRU */
 	unsigned long nr_bounce;	/* pages for bounce buffers */
+
+	unsigned long cache_miss;	/* read cache misses */
+	unsigned long readrandom;	/* random reads */
+	unsigned long pgreadrandom;	/* random read pages */
+	unsigned long readahead_rescue; /* read-aheads rescued*/
+	unsigned long pgreadahead_rescue;
+	unsigned long readahead_end;	/* read-aheads passed EOF */
+
+	unsigned long readahead;	/* read-aheads issued */
+	unsigned long readahead_return;	/* look-ahead marks returned */
+	unsigned long readahead_eof;	/* read-aheads stop at EOF */
+	unsigned long pgreadahead;	/* read-ahead pages issued */
+	unsigned long pgreadahead_hit;	/* read-ahead pages accessed */
+	unsigned long pgreadahead_eof;
+
+	unsigned long ra_newfile;	/* read-ahead on start of file */
+	unsigned long ra_newfile_return;
+	unsigned long ra_newfile_eof;
+	unsigned long pgra_newfile;
+	unsigned long pgra_newfile_hit;
+	unsigned long pgra_newfile_eof;
+
+	unsigned long ra_state;		/* state based read-ahead */
+	unsigned long ra_state_return;
+	unsigned long ra_state_eof;
+	unsigned long pgra_state;
+	unsigned long pgra_state_hit;
+	unsigned long pgra_state_eof;
+
+	unsigned long ra_context;	/* context based read-ahead */
+	unsigned long ra_context_return;
+	unsigned long ra_context_eof;
+	unsigned long pgra_context;
+	unsigned long pgra_context_hit;
+	unsigned long pgra_context_eof;
+
+	unsigned long ra_contexta;	/* accelerated context based read-ahead */
+	unsigned long ra_contexta_return;
+	unsigned long ra_contexta_eof;
+	unsigned long pgra_contexta;
+	unsigned long pgra_contexta_hit;
+	unsigned long pgra_contexta_eof;
+
+	unsigned long ra_backward;	/* prefetch pages for backward reading */
+	unsigned long ra_backward_return;
+	unsigned long ra_backward_eof;
+	unsigned long pgra_backward;
+	unsigned long pgra_backward_hit;

include/linux/page-flags.h struct page_state { · line 193

Quoted 10 lines
include/linux/page-flags.h +9 -0
+	unsigned long pgra_backward_eof;
+
+	unsigned long ra_random;	/* read-ahead on seek-and-read-pages */
+	unsigned long ra_random_return;
+	unsigned long ra_random_eof;
+	unsigned long pgra_random;
+	unsigned long pgra_random_hit;
+	unsigned long pgra_random_eof;
+
 };

Without actually understanding what you're doing, wouldn't a struct for all those groups make sense? I bet it can simplify the actual code as well.

Jörn

Signature · 3 lines
When people work hard for you for a pat on the back, you've got
to give them that pat.
-- Robert Heinlein
#2 lore raw
Quoted 60 lines · J?rn Engel
Quoted 59 lines · WU Fengguang
include/linux/page-flags.h +54 -0
@@ -131,6 +133,63 @@ struct page_state {
 
 	unsigned long pgrotated;	/* pages rotated to tail of the LRU */
 	unsigned long nr_bounce;	/* pages for bounce buffers */
+
+	unsigned long cache_miss;	/* read cache misses */
+	unsigned long readrandom;	/* random reads */
+	unsigned long pgreadrandom;	/* random read pages */
+	unsigned long readahead_rescue; /* read-aheads rescued*/
+	unsigned long pgreadahead_rescue;
+	unsigned long readahead_end;	/* read-aheads passed EOF */
+
+	unsigned long readahead;	/* read-aheads issued */
+	unsigned long readahead_return;	/* look-ahead marks returned */
+	unsigned long readahead_eof;	/* read-aheads stop at EOF */
+	unsigned long pgreadahead;	/* read-ahead pages issued */
+	unsigned long pgreadahead_hit;	/* read-ahead pages accessed */
+	unsigned long pgreadahead_eof;
+
+	unsigned long ra_newfile;	/* read-ahead on start of file */
+	unsigned long ra_newfile_return;
+	unsigned long ra_newfile_eof;
+	unsigned long pgra_newfile;
+	unsigned long pgra_newfile_hit;
+	unsigned long pgra_newfile_eof;
+
+	unsigned long ra_state;		/* state based read-ahead */
+	unsigned long ra_state_return;
+	unsigned long ra_state_eof;
+	unsigned long pgra_state;
+	unsigned long pgra_state_hit;
+	unsigned long pgra_state_eof;
+
+	unsigned long ra_context;	/* context based read-ahead */
+	unsigned long ra_context_return;
+	unsigned long ra_context_eof;
+	unsigned long pgra_context;
+	unsigned long pgra_context_hit;
+	unsigned long pgra_context_eof;
+
+	unsigned long ra_contexta;	/* accelerated context based read-ahead */
+	unsigned long ra_contexta_return;
+	unsigned long ra_contexta_eof;
+	unsigned long pgra_contexta;
+	unsigned long pgra_contexta_hit;
+	unsigned long pgra_contexta_eof;
+
+	unsigned long ra_backward;	/* prefetch pages for backward reading */
+	unsigned long ra_backward_return;
+	unsigned long ra_backward_eof;
+	unsigned long pgra_backward;
+	unsigned long pgra_backward_hit;
+	unsigned long pgra_backward_eof;
+
+	unsigned long ra_random;	/* read-ahead on seek-and-read-pages */
+	unsigned long ra_random_return;
+	unsigned long ra_random_eof;
+	unsigned long pgra_random;
Quoted 8 lines

include/linux/page-flags.h struct page_state { · line 193

Quoted 4 lines
include/linux/page-flags.h +3 -0
+	unsigned long pgra_random_hit;
+	unsigned long pgra_random_eof;
+
 };

Without actually understanding what you're doing, wouldn't a struct for all those groups make sense? I bet it can simplify the actual code as well.

Yeah, I did not expect the structure to grow so large.

It should be better to create a stand alone struct and create a dedicated entry in /proc/. The data would be a nice hint for administrators who care the performance.

Thanks,
Wu Fengguang

#3 lore raw
Quoted 18 lines · WU Fengguang

...

Quoted 8 lines
Quoted 8 lines
include/linux/page-flags.h +8 -0
+	unsigned long pgra_backward;
+	unsigned long pgra_backward_hit;
+	unsigned long pgra_backward_eof;
+
+	unsigned long ra_random;	/* read-ahead on seek-and-read-pages */
+	unsigned long ra_random_return;
+	unsigned long ra_random_eof;
+	unsigned long pgra_random;
Quoted 8 lines

include/linux/page-flags.h struct page_state { · line 193

Quoted 4 lines
include/linux/page-flags.h +3 -0
+	unsigned long pgra_random_hit;
+	unsigned long pgra_random_eof;
+
 };

Without actually understanding what you're doing, wouldn't a struct for all those groups make sense? I bet it can simplify the actual code as well.

Quoted 1 line

Yeah, I did not expect the structure to grow so large.

All that stuff shouldn't remain there long-term.

Quoted 3 lines

It should be better to create a stand alone struct and create a dedicated entry in /proc/. The data would be a nice hint for administrators who care the performance.

You should treat such informaton as a development aid, really. People are very unlikely to look at it in real life.

#4 lore raw
Quoted 9 lines · Andrew Morton

All that stuff shouldn't remain there long-term.

Quoted 3 lines

It should be better to create a stand alone struct and create a dedicated entry in /proc/. The data would be a nice hint for administrators who care the performance.

You should treat such informaton as a development aid, really. People are very unlikely to look at it in real life.

ok.

btw, I have a slightly new version against the latest -mm tree. It fixed a silly bug of save_chunk() and corrected a misbehavior of try_read_backward(). More cleanups are under the way.

Regards,
Wu Fengguang

drivers/block/loop.c +5 -0
diff -rup linux-2.6.14-rc4-mm1-orig/drivers/block/loop.c linux-2.6.14-rc4-mm1/drivers/block/loop.c
@@ -768,6 +768,11 @@ static int loop_set_fd(struct loop_devic
 
 	mapping = file->f_mapping;
 	inode = mapping->host;
+	/*
+	 * The upper layer should already do proper read-ahead,
+	 * unlimited read-ahead here only ruins the cache hit rate.
+	 */
+	file->f_ra.ra_pages = 32 >> (PAGE_CACHE_SHIFT - 10);
 
 	if (!(file->f_mode & FMODE_WRITE))
 		lo_flags |= LO_FLAGS_READ_ONLY;
include/linux/fs.h +20 -1
diff -rup linux-2.6.14-rc4-mm1-orig/include/linux/fs.h linux-2.6.14-rc4-mm1/include/linux/fs.h
@@ -562,17 +562,36 @@ struct file_ra_state {
 	unsigned long start;		/* Current window */
 	unsigned long size;
 	unsigned long flags;		/* ra flags RA_FLAG_xxx*/
-	unsigned long cache_hit;	/* cache hit count*/
+	uint64_t      cache_hit;	/* cache hit count*/
 	unsigned long prev_page;	/* Cache last read() position */
 	unsigned long ahead_start;	/* Ahead window */
 	unsigned long ahead_size;
 	unsigned long ra_pages;		/* Maximum readahead window */
 	unsigned long mmap_hit;		/* Cache hit stat for mmap accesses */
 	unsigned long mmap_miss;	/* Cache miss stat for mmap accesses */
+
+	unsigned long la_index;
+	unsigned long ra_index;
+	unsigned long lookahead_index;
+	unsigned long readahead_index;
+	unsigned long nr_page_aging;
 };
 #define RA_FLAG_MISS 0x01	/* a cache miss occured against this file */
 #define RA_FLAG_INCACHE 0x02	/* file is already in cache */
 
+#define RA_CLASS_SHIFT 3
+#define RA_CLASS_MASK  ((1 << RA_CLASS_SHIFT) - 1)
+enum file_ra_class { /* the same order must be kept in page_state */
+	RA_CLASS_NEWFILE = 1,
+	RA_CLASS_STATE,
+	RA_CLASS_CONTEXT,
+	RA_CLASS_CONTEXT_ACCELERATED,
+	RA_CLASS_BACKWARD,
+	/* RA_CLASS_AROUND, */
+	RA_CLASS_RANDOM,
+	RA_CLASS_END,
+};
+
 struct file {
 	/*
 	 * fu_list becomes invalid after file_free is called and queued via
include/linux/mm.h +9 -1
diff -rup linux-2.6.14-rc4-mm1-orig/include/linux/mm.h linux-2.6.14-rc4-mm1/include/linux/mm.h
@@ -897,7 +897,7 @@ extern int filemap_populate(struct vm_ar
 int write_one_page(struct page *page, int wait);
 
 /* readahead.c */
-#define VM_MAX_READAHEAD	128	/* kbytes */
+#define VM_MAX_READAHEAD	1024	/* kbytes */
 #define VM_MIN_READAHEAD	16	/* kbytes (includes current page) */
 #define VM_MAX_CACHE_HIT    	256	/* max pages in a row in cache before
 					 * turning readahead off */
@@ -914,6 +914,14 @@ unsigned long  page_cache_readahead(stru
 void handle_ra_miss(struct address_space *mapping, 
 		    struct file_ra_state *ra, pgoff_t offset);
 unsigned long max_sane_readahead(unsigned long nr);
+unsigned long
+page_cache_readahead_adaptive(struct address_space *mapping,
+			struct file_ra_state *ra, struct file *filp,
+			struct page *prev_page, struct page *page,
+			unsigned long first_index,
+			unsigned long index, unsigned long last_index);
+void fastcall ra_access(struct file_ra_state *ra, struct page *page);
+int rescue_ra_pages(struct list_head *page_list, struct list_head *save_list);
 
 /* Do stack extension */
 extern int expand_stack(struct vm_area_struct *vma, unsigned long address);
include/linux/mm_inline.h +1 -0
diff -rup linux-2.6.14-rc4-mm1-orig/include/linux/mm_inline.h linux-2.6.14-rc4-mm1/include/linux/mm_inline.h
@@ -11,6 +11,7 @@ add_page_to_inactive_list(struct zone *z
 {
 	list_add(&page->lru, &zone->inactive_list);
 	zone->nr_inactive++;
+	zone->nr_page_aging++;
 }
 
 static inline void
include/linux/mmzone.h +51 -0

63 lines not on this page · raw message

include/linux/page-flags.h +71 -0

89 lines not on this page · raw message

include/linux/radix-tree.h +111 -2

135 lines not on this page · raw message

include/linux/sysctl.h +1 -0
diff -rup linux-2.6.14-rc4-mm1-orig/include/linux/sysctl.h linux-2.6.14-rc4-mm1/include/linux/sysctl.h
@@ -181,6 +181,7 @@ enum
 	VM_LEGACY_VA_LAYOUT=27, /* legacy/compatibility virtual address space layout */
 	VM_SWAP_TOKEN_TIMEOUT=28, /* default time for token time out */
 	VM_SWAP_PREFETCH=29,	/* int: amount to swap prefetch */
+	VM_READAHEAD_RATIO=30, /* percent of read-ahead size to thrashing-threshold */
 };
 
 
kernel/sysctl.c +11 -0
diff -rup linux-2.6.14-rc4-mm1-orig/kernel/sysctl.c linux-2.6.14-rc4-mm1/kernel/sysctl.c
@@ -67,6 +67,7 @@ extern int min_free_kbytes;
 extern int printk_ratelimit_jiffies;
 extern int printk_ratelimit_burst;
 extern int pid_max_min, pid_max_max;
+extern int readahead_ratio;
 
 #if defined(CONFIG_X86_LOCAL_APIC) && defined(CONFIG_X86)
 int unknown_nmi_panic;
@@ -861,6 +862,16 @@ static ctl_table vm_table[] = {
 	},
 #endif
 #endif
+	{
+		.ctl_name	= VM_READAHEAD_RATIO,
+		.procname	= "readahead_ratio",
+		.data		= &readahead_ratio,
+		.maxlen		= sizeof(readahead_ratio),
+		.mode		= 0644,
+		.proc_handler	= &proc_dointvec,
+		.strategy	= &sysctl_intvec,
+		.extra1		= &zero,
+	},
 	{ .ctl_name = 0 }
 };
 
lib/radix-tree.c +18 -53

96 lines not on this page · raw message

mm/filemap.c +61 -11

192 lines not on this page · raw message

mm/page_alloc.c +139 -16

241 lines not on this page · raw message

mm/readahead.c +1398 -8

1,465 lines not on this page · raw message

mm/swap.c +10 -2
diff -rup linux-2.6.14-rc4-mm1-orig/mm/swap.c linux-2.6.14-rc4-mm1/mm/swap.c
@@ -96,6 +96,8 @@ int rotate_reclaimable_page(struct page 
 	return 0;
 }
 
+extern int readahead_ratio;
+
 /*
  * FIXME: speed this up?
  */
@@ -122,8 +124,13 @@ void fastcall activate_page(struct page 
  */
 void fastcall mark_page_accessed(struct page *page)
 {
-	if (!PageActive(page) && PageReferenced(page) && PageLRU(page)) {
-		activate_page(page);
+	if (!PageActive(page) && !PageActivate(page) &&
+			PageReferenced(page) && PageLRU(page)) {
+		if (readahead_ratio > 9 || (readahead_ratio & 1)) {
+			page_zone(page)->nr_page_aging++;
+			SetPageActivate(page);
+		} else
+			activate_page(page);
 		ClearPageReferenced(page);
 	} else if (!PageReferenced(page)) {
 		SetPageReferenced(page);
@@ -299,6 +306,7 @@ void __pagevec_lru_add(struct pagevec *p
 			if (zone)
 				spin_unlock_irq(&zone->lru_lock);
 			zone = pagezone;
+			update_page_age(zone);
 			spin_lock_irq(&zone->lru_lock);
 		}
 		if (TestSetPageLRU(page))
mm/vmscan.c +39 -6

110 lines not on this page · raw message

#5 lore raw
In reply to Andrew Morton

On Tuesday 18 October 2005 05:31, Andrew Morton wrote:
[lot of additional page stats]

Quoted 2 lines

You should treat such informaton as a development aid, really. People are very unlikely to look at it in real life.

And put the whole stuff under debugfs, which was made for this AFAIR.

Regards

Ingo Oeser

#6 lore raw
Quoted 6 lines · Ingo Oeser

On Tuesday 18 October 2005 05:31, Andrew Morton wrote:
[lot of additional page stats]

Quoted 2 lines

You should treat such informaton as a development aid, really. People are very unlikely to look at it in real life.

And put the whole stuff under debugfs, which was made for this AFAIR.

Ok, I'll do it right away.

Thanks,
Wu Fengguang