Adaptive read-ahead V6
Subsystems (11)
PATCH 1/1 Adaptive read-ahead V6
Hi, all!
This is the adaptive read-ahead patch for the vanilla 2.6.14 kernel. It is ready for wild tests.
There are some minor fixes since V5, and added support for laptop mode. The read-ahead hard limit was lifted to 255MB, which might help laptop users to play movies from memory ;)
You can help improve it by sending me the detailed statistics:
- compile kernel with 'Kernel hacking ---> Debug Filesystem' enabled
- mkdir /debugfs; mount -t debugfs none /debugfs
- cat /debugfs/readahead
Best Regards,
Wu Fengguang
---
Documentation/sysctl/vm.txt +14 -0
@@ -26,6 +26,7 @@ Currently, these files are in /proc/sys/ - min_free_kbytes - laptop_mode - block_dump +- readahead_ratio ==============================================================
@@ -102,3 +103,16 @@ This is used to force the Linux VM to ke of kilobytes free. The VM uses this number to compute a pages_min value for each lowmem zone in the system. Each lowmem zone gets a number of reserved free pages based proportionally on its size. + +============================================================== + +readahead_ratio + +This limits read-ahead size to percent of the thrashing-threshold. +The thrashing-threshold is dynamicly estimated according to the +_history_ read speed and system load, and used to limit the +_future_ read-ahead request size. So you should set it to a low +value if you have not enough memory to counteract the I/O load +fluctuation. + +The default value is 50.
drivers/block/loop.c +5 -0
@@ -768,6 +768,11 @@ static int loop_set_fd(struct loop_devic mapping = file->f_mapping; inode = mapping->host; +/* +* The upper layer should already do proper read-ahead, +* unlimited read-ahead here only ruins the cache hit rate. +*/ + file->f_ra.ra_pages =32 >> (PAGE_CACHE_SHIFT -10 );if (!(file->f_mode &FMODE_WRITE )) lo_flags |=LO_FLAGS_READ_ONLY ;
include/linux/fs.h +7 -1
@@ -562,13 +562,19 @@ struct file_ra_state {unsigned long start;/* Current window */ unsigned long size;unsigned long flags;/* ra flags RA_FLAG_xxx*/ -unsigned long cache_hit;/* cache hit count*/ +uint64_t cache_hit;/* cache hit count*/ unsigned long prev_page;/* Cache last read() position */ unsigned long ahead_start;/* Ahead window */ unsigned long ahead_size;unsigned long ra_pages;/* Maximum readahead window */ unsigned long mmap_hit;/* Cache hit stat for mmap accesses */ unsigned long mmap_miss;/* Cache miss stat for mmap accesses */ + +unsigned long age; +unsigned long la_index; +unsigned long ra_index; +unsigned long lookahead_index; +unsigned long readahead_index; };#define RA_FLAG_MISS 0x01/* a cache miss occured against this file */ #define RA_FLAG_INCACHE 0x02/* file is already in cache */
include/linux/mm.h +13 -1
@@ -875,11 +875,14 @@ extern int filemap_populate(struct vm_arint write_one_page (struct page *page,int wait);/* readahead.c */ -#define VM_MAX_READAHEAD 128/* kbytes */ +#define VM_MAX_READAHEAD 1024/* kbytes */ #define VM_MIN_READAHEAD 16/* kbytes (includes current page) */ #define VM_MAX_CACHE_HIT 256/* max pages in a row in cache before * turning readahead off */ +/* turn on read-ahead thrashing protection if (readahead_ratio >= ##) */ +#define VM_READAHEAD_PROTECT_RATIO 80 +int do_page_cache_readahead (struct address_space *mapping,struct file *filp,unsigned long offset,unsigned long nr_to_read); int force_page_cache_readahead(struct address_space *mapping, struct file *filp,
@@ -892,6 +895,15 @@ unsigned long page_cache_readahead(struvoid handle_ra_miss (struct address_space *mapping,struct file_ra_state *ra,pgoff_t offset);unsigned long max_sane_readahead (unsigned long nr); +unsigned long +page_cache_readahead_adaptive (struct address_space *mapping, +struct file_ra_state *ra,struct file *filp, +struct page *prev_page,struct page *page, +unsigned long first_index, +unsigned long index,unsigned long last_index); +void fastcall ra_access(struct file_ra_state *ra, struct page *page); +int rescue_ra_pages (struct list_head *page_list,struct list_head *save_list); +/* Do stack extension */ extern int expand_stack (struct vm_area_struct * vma,unsigned long address);
include/linux/mmzone.h +53 -0
65 lines not on this page · raw message
include/linux/page-flags.h +38 -0
@@ -75,6 +75,8 @@#define PG_reclaim 17/* To be reclaimed asap */ #define PG_nosave_free 18/* Free, should not be written */ #define PG_uncached 19/* Page has been mapped as uncached */ +#define PG_activate 20/* delayed activate */ +#define PG_readahead 21/* check readahead when reading this page */ /* * Global page accounting. One instance per CPU. Only unsigned longs are
@@ -104,6 +106,8 @@ struct page_state {unsigned long pgfree;/* page freeings */ unsigned long pgactivate;/* pages moved inactive->active */ unsigned long pgdeactivate;/* pages moved active->inactive */ +unsigned long pgkeephot;/* pages sent back to active */ +unsigned long pgkeepcold;/* pages sent back to inactive */ unsigned long pgfault;/* faults (major+minor) */ unsigned long pgmajfault;/* faults (major only) */
@@ -307,6 +311,16 @@ extern void __mod_page_state(unsigned lo#define SetPageUncached (page) set_bit(PG_uncached, &(page)->flags)#define ClearPageUncached (page) clear_bit(PG_uncached, &(page)->flags) +#define PageActivate (page) test_bit(PG_activate, &(page)->flags) +#define SetPageActivate (page) set_bit(PG_activate, &(page)->flags) +#define ClearPageActivate (page) clear_bit(PG_activate, &(page)->flags) +#define TestClearPageActivate (page) test_and_clear_bit(PG_activate, &(page)->flags) +#define TestSetPageActivate (page) test_and_set_bit(PG_activate, &(page)->flags) + +#define PageReadahead (page) test_bit(PG_readahead, &(page)->flags) +#define SetPageReadahead (page) set_bit(PG_readahead, &(page)->flags) +#define TestClearPageReadahead (page) test_and_clear_bit(PG_readahead, &(page)->flags) +struct page ;/* forward declaration */ int test_clear_page_dirty (struct page *page);
@@ -323,4 +337,28 @@ static inline void set_page_writeback(st test_set_page_writeback(page); } +#if PG_activate < PG_referenced +#error unexpected page flags order +#endif + +#define PAGE_REFCNT_0 0 +#define PAGE_REFCNT_1 (1 << PG_referenced) +#define PAGE_REFCNT_2 (1 << PG_activate) +#define PAGE_REFCNT_3 ((1 << PG_activate) | (1 << PG_referenced)) +#define PAGE_REFCNT_MASK PAGE_REFCNT_3 + +/* +* STATUS REFERENCE COUNT +* __ 0 +* _R PAGE_REFCNT_1 +* A_ PAGE_REFCNT_2 +* AR PAGE_REFCNT_3 +* +* A/R: Active / Referenced +*/ +static inline unsigned long page_refcnt (struct page *page) +{ +return page->flags &PAGE_REFCNT_MASK ; +} +#endif /* PAGE_FLAGS_H */
include/linux/radix-tree.h +147 -1
171 lines not on this page · raw message
include/linux/sysctl.h +1 -0
@@ -180,6 +180,7 @@ enumVM_VFS_CACHE_PRESSURE =26 ,/* dcache/icache reclaim pressure */ VM_LEGACY_VA_LAYOUT =27 ,/* legacy/compatibility virtual address space layout */ VM_SWAP_TOKEN_TIMEOUT =28 ,/* default time for token time out */ + VM_READAHEAD_RATIO=29, /* percent of read-ahead size to thrashing-threshold */ };
include/linux/writeback.h +6 -0
@@ -90,6 +90,12 @@ void laptop_io_completion(void);void laptop_sync_completion (void );void throttle_vm_writeout (void ); +extern struct timer_list laptop_mode_wb_timer; +static inline int laptop_spinned_down (void ) +{ +return !timer_pending (&laptop_mode_wb_timer); +} +/* These are exported to sysctl. */ extern int dirty_background_ratio;extern int vm_dirty_ratio;
kernel/sysctl.c +11 -0
@@ -67,6 +67,7 @@ extern int min_free_kbytes;extern int printk_ratelimit_jiffies;extern int printk_ratelimit_burst;extern int pid_max_min, pid_max_max; +extern int readahead_ratio;#if defined(CONFIG_X86_LOCAL_APIC ) && defined(CONFIG_X86 )int unknown_nmi_panic;
@@ -849,6 +850,16 @@ static ctl_table vm_table[] = { .strategy = &sysctl_jiffies, },#endif + { + .ctl_name = VM_READAHEAD_RATIO, + .procname = "readahead_ratio", + .data = &readahead_ratio, + .maxlen = sizeof(readahead_ratio), + .mode = 0644, + .proc_handler = &proc_dointvec, + .strategy = &sysctl_intvec, + .extra1 = &zero, + }, { .ctl_name = 0 } };
lib/radix-tree.c +119 -23
177 lines not on this page · raw message
mm/filemap.c +62 -11
196 lines not on this page · raw message
mm/page_alloc.c +83 -15
178 lines not on this page · raw message
mm/page-writeback.c +1 -1
@@ -369,7 +369,7 @@ static void wb_timer_fn(unsigned long unstatic void laptop_timer_fn (unsigned long unused); static DEFINE_TIMER(wb_timer, wb_timer_fn, 0, 0); -static DEFINE_TIMER(laptop_mode_wb_timer, laptop_timer_fn, 0, 0); +DEFINE_TIMER (laptop_mode_wb_timer, laptop_timer_fn,0 ,0 );/* * Periodic writeback of "old" data.
mm/readahead.c +1670 -9
1,744 lines not on this page · raw message
mm/swap.c +7 -5
@@ -29,7 +29,6 @@#include <linux/percpu.h> #include <linux/cpu.h> #include <linux/notifier.h> -#include <linux/init.h> /* How many pages do we try to swap or page in/out together? */ int page_cluster;
@@ -113,20 +112,23 @@ void fastcall activate_page(struct pagespin_unlock_irq (&zone->lru_lock); } +DECLARE_PER_CPU(unsigned long, smooth_aging); +/* * Mark a page as having seen activity. * * inactive,unreferenced -> inactive,referenced -* inactive,referenced -> active,unreferenced -* active,unreferenced -> active,referenced +* inactive,referenced -> activate,unreferenced +* activate,unreferenced -> activate,referenced */ void fastcall mark_page_accessed(struct page *page) { -if (!PageActive (page) &&PageReferenced (page) &&PageLRU (page)) { -activate_page (page); +if (!PageActivate (page) &&PageReferenced (page) &&PageLRU (page)) { +SetPageActivate (page);ClearPageReferenced (page); }else if (!PageReferenced (page)) {SetPageReferenced (page); +__get_cpu_var (smooth_aging)++; } }
mm/vmscan.c +44 -6
128 lines not on this page · raw message
A message that is not in the archive (sent off-list or not yet received): Pine.LNX.4.61.0511012131290.20381@lnxos
Hi, Alexandre,
Quoted 14 lines · Alexandre Cassen
I just replayed a benchmark with your new patch. You will find attached new bench output.
Experiment Protocol: . 10 mpeg2 video bit/rate=3.5Mb/s ATM
. around 4Mb/s ip+udp per stream . Linux Kernel 2.6.14 + your v6 patch . stored on a raid0/dev/md0- XFS . blockdev --setra 2048/dev/md0. 6GB RAM + 2*GiGE NICs Trunk +
2*36GB HDD on a U320 adaptec controler
. R-A LOOKAHEAD_RATIO=4
. ADP R-A /proc/sys/vm/readahead_ratio=200 . Lauch video at true random positions
Quoted 4 lines
I launched 20 clients per video, say 200 streams. I collected infos during around 1hours running.
Let me know if you want some more inputs.
Thanks.
Quoted 6 lines
[table requests] total newfile state context contexta backward onthrash onraseek none
cache_miss 2847 6 1525 694 0 0 0 4 618 read_random 295 2 3 2 0 0 0 0 288 io_congestion 0 0 0 0 0 0 0 0 0 io_cache_hit 7420 3 5941 1472 0 0 0 4 0 io_block 4159 807 36 2648 0 0 0 4 664
This reflects times of IO delays.
The main delays(2648) occur in context method, which are triggered by cache misses(2847). The cache misses might be caused by canceled look-ahead marks due to cache hits. I'll try fixing it.
Quoted 9 lines
readahead 116568 801 113031 2731 1 0 0 4 0 lookahead 115904 223 112954 2726 1 0 0 0 0 lookahead_hit 113376 31 111256 2072 1 0 0 0 16 readahead_eof 660 578 77 5 0 0 0 0 0 readahead_shrink 0 0 0 0 0 0 0 0 0 readahead_thrash 0 0 0 0 0 0 0 0 0 readahead_rescue 18517 0 0 0 0 0 0 0 18517[table pages] total newfile state context contexta backward onthrash onraseek none
Quoted 13 lines
cache_miss 5452 6 3037 1377 0 0 0 7 1025 read_random 391 2 5 2 0 0 0 0 382 io_congestion 0 0 0 0 0 0 0 0 0 io_cache_hit 622587 4 386637 235281 0 0 0 665 0 io_block 547948 1954 1443 542863 0 0 0 1024 664 readahead 28748734 1959 28414048 332302 66 0 0 359 0 readahead_hit 28830698 1764 28414032 412507 3 0 0 856 1536 lookahead 29202363 448 28780361 421521 33 0 0 0 0 lookahead_hit 28671617 64 28358286 313232 35 0 0 0 0 readahead_eof 15225 1065 13458 702 0 0 0 0 0 readahead_shrink 0 0 0 0 0 0 0 0 0 readahead_thrash 0 0 0 0 0 0 0 0 0 readahead_rescue 265680 0 0 0 0 0 0 0 265680
Your system is memory bounty and thrashing free.
The ~1G rescued pages are expected to be saved for a following reader.
Quoted 7 lines
[table summary] total newfile state context contexta backward onthrash onraseek none
random_rate 0% 0% 0% 0% 0% 0% 0% 0% 99% ra_hit_rate 100% 90% 99% 124% 4% 0% 0% 237% 153600% la_hit_rate 97% 13% 98% 75% 50% 0% 0% 0% 1600% avg_ra_size 247 2 251 122 33 0 0 72 0 avg_la_size 252 2 255 155 16 0 0 0 0
Regards,
Wu Fengguang