VM pageout scalability - one big patch
Subsystems (20)
cover VM pageout scalability - one big patch
spot the typo :)
Date: Wed, 2 Jan 2008 18:17:26 -0500
From: Rik van Riel <[email protected]>
To: [email protected]
Cc: [email protected], [email protected]
Subject: [patch 00/19] VM pageout scalability - one big patch
By popular request, here are all the VM pageout scalability patches rolled up into one big patch.
drivers/base/node.c | 65 +- drivers/block/brd.c | 13 fs/cifs/file.c | 4 fs/nfs/dir.c | 2 fs/ntfs/file.c | 4 fs/proc/proc_misc.c | 78 +- fs/ramfs/file-nommu.c | 4 fs/ramfs/inode.c | 1 include/linux/memcontrol.h | 19 include/linux/migrate.h | 3 include/linux/mm_inline.h | 136 ++++- include/linux/mmzone.h | 66 ++ include/linux/page-flags.h | 43 + include/linux/pagemap.h | 28 + include/linux/pagevec.h | 22 include/linux/rmap.h | 12 include/linux/swap.h | 44 + include/linux/vmstat.h | 10 kernel/sysctl.c | 10 mm/Kconfig | 24 mm/filemap.c | 19 mm/internal.h | 57 ++ mm/memcontrol.c | 170 ++---- mm/memory.c | 13 mm/mempolicy.c | 12 mm/migrate.c | 87 +-- mm/mlock.c | 290 ++++++++++ mm/mmap.c | 42 + mm/mremap.c | 8 mm/page-writeback.c | 8 mm/page_alloc.c | 110 +++- mm/readahead.c | 2 mm/rmap.c | 160 +++++ mm/shmem.c | 8 mm/swap.c | 249 ++++++++- mm/swap_state.c | 6 mm/truncate.c | 4 mm/vmscan.c | 939 ++++++++++++++++++++++++----------- mm/vmstat.c | 25 39 files changed, 2171 insertions(+), 626 deletions(-)
include/linux/migrate.h +0 -3
@@ -25,7 +25,6 @@ static inline int vma_migratable(structreturn 1 ; } -extern int isolate_lru_page (struct page *p,struct list_head *pagelist);extern int putback_lru_pages (struct list_head *l);extern int migrate_page (struct address_space *,struct page *,struct page *);
@@ -42,8 +41,6 @@ extern int migrate_vmas(struct mm_structstatic inline int vma_migratable (struct vm_area_struct *vma) {return 0 ; } -static inline int isolate_lru_page (struct page *p,struct list_head *list) - {return -ENOSYS ; }static inline int putback_lru_pages (struct list_head *l) {return 0 ; }static inline int migrate_pages (struct list_head *l,new_page_t x,unsigned long private) {return -ENOSYS ; }
mm/internal.h +57 -0
66 lines not on this page · raw message
mm/migrate.c +45 -42
136 lines not on this page · raw message
mm/vmscan.c +665 -274
1,282 lines not on this page · raw message
mm/mempolicy.c +9 -3
@@ -93,6 +93,8 @@#include <asm/tlbflush.h> #include <asm/uaccess.h> +#include "internal.h" +/* Internal flags */ #define MPOL_MF_DISCONTIG_OK (MPOL_MF_INTERNAL << 0)/* Skip checks for continuous vmas */ #define MPOL_MF_INVERT (MPOL_MF_INTERNAL << 1)/* Invert check for nodemask */
@@ -603,8 +605,12 @@ static void migrate_page_add(struct page/* * Avoid migrating a page that is shared with others. */ -if ((flags &MPOL_MF_MOVE_ALL ) ||page_mapcount (page) ==1 ) -isolate_lru_page (page, pagelist); +if ((flags &MPOL_MF_MOVE_ALL ) ||page_mapcount (page) ==1 ) { +if (!isolate_lru_page (page)) { +get_page (page); +list_add_tail (&page->lru, pagelist); + } + } }static struct page *new_node_page (struct page *page,unsigned long node,int **x)
@@ -1906,7 +1912,7 @@ static void gather_stats(struct page *paif (PageSwapCache (page)) md->swapcache++; -if (PageActive (page)) +if (PageActive (page) ||PageNoreclaim (page)) md->active++;if (PageWriteback (page))
mm/swap.c +220 -29
368 lines not on this page · raw message
include/linux/pagevec.h +18 -4
@@ -23,9 +23,17 @@ struct pagevec {void __pagevec_release (struct pagevec *pvec);void __pagevec_release_nonlru (struct pagevec *pvec);void __pagevec_free (struct pagevec *pvec); -void __pagevec_lru_add (struct pagevec *pvec); -void __pagevec_lru_add_active (struct pagevec *pvec); +void __pagevec_lru_add_file (struct pagevec *pvec); +void __pagevec_lru_add_active_file (struct pagevec *pvec); +void __pagevec_lru_add_anon (struct pagevec *pvec); +void __pagevec_lru_add_active_anon (struct pagevec *pvec); +#ifdef CONFIG_NORECLAIM +void __pagevec_lru_add_noreclaim (struct pagevec *pvec); +#else +static inline void __pagevec_lru_add_noreclaim (struct pagevec *pvec) { } +#endif void pagevec_strip (struct pagevec *pvec); +void pagevec_swap_free (struct pagevec *pvec);unsigned pagevec_lookup (struct pagevec *pvec,struct address_space *mapping,pgoff_t start,unsigned nr_pages); unsigned pagevec_lookup_tag(struct pagevec *pvec,
@@ -81,10 +89,16 @@ static inline void pagevec_free(struct p __pagevec_free(pvec); } -static inline void pagevec_lru_add (struct pagevec *pvec) +static inline void pagevec_lru_add_file (struct pagevec *pvec) {if (pagevec_count (pvec)) -__pagevec_lru_add (pvec); +__pagevec_lru_add_file (pvec); +} + +static inline void pagevec_lru_add_anon (struct pagevec *pvec) +{ +if (pagevec_count (pvec)) +__pagevec_lru_add_anon (pvec); }#endif /* _LINUX_PAGEVEC_H */
include/linux/mm_inline.h +120 -16
160 lines not on this page · raw message
mm/shmem.c +6 -2
@@ -179,7 +179,7 @@ static inline void shmem_unacct_blocks(u }static const struct super_operations shmem_ops; -static const struct address_space_operations shmem_aops; +const struct address_space_operations shmem_aops;static const struct file_operations shmem_file_operations;static const struct inode_operations shmem_inode_operations;static const struct inode_operations shmem_dir_inode_operations;
@@ -1377,6 +1377,7 @@ repeat: goto failed; } +SetPageSwapBacked (filepage);spin_lock (&info->lock); entry =shmem_swp_alloc (info, idx, sgp);if (IS_ERR (entry))
@@ -1467,10 +1468,13 @@ int shmem_lock(struct file *file, int loif (!user_shm_lock (inode->i_size, user)) goto out_nomem; info->flags |=VM_LOCKED ; +mapping_set_noreclaim (file->f_mapping); }if (!lock && (info->flags &VM_LOCKED ) && user) {user_shm_unlock (inode->i_size, user); info->flags &= ~VM_LOCKED ; +mapping_clear_noreclaim (file->f_mapping); +scan_mapping_noreclaim_pages (file->f_mapping); } retval =0 ; out_nomem:
@@ -2343,7 +2347,7 @@ static void destroy_inodecache(void)kmem_cache_destroy (shmem_inode_cachep); } -static const struct address_space_operations shmem_aops = { +const struct address_space_operations shmem_aops = { .writepage = shmem_writepage, .set_page_dirty = __set_page_dirty_no_writeback, #ifdef CONFIG_TMPFS
include/linux/page-flags.h +43 -0
77 lines not on this page · raw message
mm/memory.c +8 -5
@@ -1664,7 +1664,8 @@ gotten:ptep_clear_flush (vma, address, page_table);set_pte_at (mm, address, page_table, entry);update_mmu_cache (vma, address, entry); -lru_cache_add_active (new_page); +SetPageSwapBacked (new_page); +lru_cache_add_active_or_noreclaim (new_page, vma);page_add_new_anon_rmap (new_page, vma, address);/* Free the old page.. */
@@ -2131,7 +2132,8 @@ static int do_anonymous_page(struct mm_sif (!pte_none (*page_table)) goto release;inc_mm_counter (mm, anon_rss); -lru_cache_add_active (page); +SetPageSwapBacked (page); +lru_cache_add_active_or_noreclaim (page, vma);page_add_new_anon_rmap (page, vma, address);set_pte_at (mm, address, page_table, entry);
@@ -2283,9 +2285,10 @@ static int __do_fault(struct mm_struct * entry =maybe_mkwrite (pte_mkdirty (entry), vma);set_pte_at (mm, address, page_table, entry);if (anon) { -inc_mm_counter (mm, anon_rss); -lru_cache_add_active (page); -page_add_new_anon_rmap (page, vma, address); +inc_mm_counter (mm, anon_rss); +SetPageSwapBacked (page); +lru_cache_add_active_or_noreclaim (page, vma); +page_add_new_anon_rmap (page, vma, address); }else {inc_mm_counter (mm, file_rss);page_add_file_rmap (page);
mm/swap_state.c +5 -1
@@ -82,6 +82,7 @@ int add_to_swap_cache(struct page *page,if (!error) {page_cache_get (page);SetPageSwapCache (page); +SetPageSwapBacked (page);set_page_private (page, entry.val); total_swapcache_pages++;__inc_zone_page_state (page,NR_FILE_PAGES );
@@ -299,7 +300,10 @@ struct page *read_swap_cache_async(swp_e/* * Initiate read into locked page and return. */ -lru_cache_add_active (new_page); +if (!page_reclaimable (new_page, vma)) +lru_cache_add_noreclaim (new_page); +else +lru_cache_add_anon (new_page);swap_readpage (NULL , new_page);return new_page; }
mm/page_alloc.c +98 -12
191 lines not on this page · raw message
include/linux/mmzone.h +59 -7
102 lines not on this page · raw message
mm/vmstat.c +19 -6
@@ -686,8 +686,16 @@ const struct seq_operations pagetypeinfostatic const char *const vmstat_text[] = {/* Zoned VM counters */ "nr_free_pages" , -"nr_inactive" , -"nr_active" , +"nr_inactive_anon" , +"nr_active_anon" , +"nr_inactive_file" , +"nr_active_file" , +#ifdef CONFIG_NORECLAIM +"nr_noreclaim" , +#endif +#ifdef CONFIG_NORECLAIM_MLOCK +"nr_mlock" , +#endif"nr_anon_pages" ,"nr_mapped" ,"nr_file_pages" ,
@@ -750,7 +758,7 @@ static void zoneinfo_show_print(struct s"\n min %lu" "\n low %lu" "\n high %lu" -"\n scanned %lu (a: %lu i: %lu)" +"\n scanned %lu (aa: %lu ia: %lu af: %lu if: %lu)" "\n spanned %lu" "\n present %lu" , zone_page_state(zone, NR_FREE_PAGES),
@@ -758,7 +766,10 @@ static void zoneinfo_show_print(struct s zone->pages_low, zone->pages_high, zone->pages_scanned, - zone->nr_scan_active, zone->nr_scan_inactive, + zone->nr_scan[LRU_ACTIVE_ANON ], + zone->nr_scan[LRU_INACTIVE_ANON ], + zone->nr_scan[LRU_ACTIVE_FILE ], + zone->nr_scan[LRU_INACTIVE_FILE ], zone->spanned_pages, zone->present_pages);
@@ -795,10 +806,12 @@ static void zoneinfo_show_print(struct sseq_printf (m,"\n all_unreclaimable: %u" "\n prev_priority: %i" -"\n start_pfn: %lu" , +"\n start_pfn: %lu" +"\n inactive_ratio: %u" ,zone_is_all_unreclaimable (zone), zone->prev_priority, - zone->zone_start_pfn); + zone->zone_start_pfn, + zone->inactive_ratio);seq_putc (m,'\n' ); }
fs/proc/proc_misc.c +47 -31
90 lines not on this page · raw message
fs/cifs/file.c +2 -2
@@ -1783,7 +1783,7 @@ static void cifs_copy_cache_pages(structSetPageUptodate (page);unlock_page (page);if (!pagevec_add (plru_pvec, page)) -__pagevec_lru_add (plru_pvec); +__pagevec_lru_add_file (plru_pvec); data +=PAGE_CACHE_SIZE ; }return ;
@@ -1921,7 +1921,7 @@ static int cifs_readpages(struct file *f bytes_read =0 ; } -pagevec_lru_add (&lru_pvec); +pagevec_lru_add_file (&lru_pvec);/* need to free smb_read_data buf before exit */ if (smb_read_data) {
fs/ntfs/file.c +2 -2
@@ -439,7 +439,7 @@ static inline int __ntfs_grab_cache_page pages[nr] = *cached_page;page_cache_get (*cached_page);if (unlikely (!pagevec_add (lru_pvec, *cached_page))) -__pagevec_lru_add (lru_pvec); +__pagevec_lru_add_file (lru_pvec); *cached_page =NULL ; } index++;
@@ -2084,7 +2084,7 @@ err_out: OSYNC_METADATA|OSYNC_DATA); } } -pagevec_lru_add (&lru_pvec); +pagevec_lru_add_file (&lru_pvec);ntfs_debug ("Done. Returning %s (written 0x%lx, status %li)." , written ?"written" :"status" , (unsigned long )written, (long )status);
fs/nfs/dir.c +1 -1
@@ -1497,7 +1497,7 @@ static int nfs_symlink(struct inode *dirif (!add_to_page_cache (page, dentry->d_inode->i_mapping,0 ,GFP_KERNEL )) {pagevec_add (&lru_pvec, page); -pagevec_lru_add (&lru_pvec); +pagevec_lru_add_file (&lru_pvec);SetPageUptodate (page);unlock_page (page); }else
fs/ramfs/file-nommu.c +2 -2
@@ -111,12 +111,12 @@ static int ramfs_nommu_expand_for_mappin goto add_error;if (!pagevec_add (&lru_pvec, page)) -__pagevec_lru_add (&lru_pvec); +__pagevec_lru_add_file (&lru_pvec);unlock_page (page); } -pagevec_lru_add (&lru_pvec); +pagevec_lru_add_file (&lru_pvec);return 0 ; fsize_exceeded:
drivers/base/node.c +43 -22
94 lines not on this page · raw message
mm/readahead.c +1 -1
@@ -229,7 +229,7 @@ int do_page_cache_readahead(struct addre*/ unsigned long max_sane_readahead (unsigned long nr) { -return min (nr, (node_page_state (numa_node_id (),NR_INACTIVE ) +return min (nr, (node_page_state (numa_node_id (),NR_INACTIVE_FILE ) +node_page_state (numa_node_id (),NR_FREE_PAGES )) /2 ); }
mm/filemap.c +16 -3
@@ -34,6 +34,7 @@#include <linux/cpuset.h> #include <linux/hardirq.h> /* for BUG_ON(!in_atomic()) only */ #include <linux/memcontrol.h> +#include <linux/mm_inline.h> /* for page_file_cache() */ #include "internal.h" /*
@@ -493,8 +494,12 @@ int add_to_page_cache_lru(struct page *p pgoff_t offset, gfp_t gfp_mask) { int ret = add_to_page_cache(page, mapping, offset, gfp_mask); -if (ret ==0 ) -lru_cache_add (page); +if (ret ==0 ) { +if (page_file_cache (page)) +lru_cache_add_file (page); +else +lru_cache_add_active_anon (page); + }return ret; }
@@ -2520,8 +2525,16 @@ generic_file_direct_IO(int rw, struct kiif (rw ==WRITE ) { write_len =iov_length (iov, nr_segs); end = (offset + write_len -1 ) >>PAGE_CACHE_SHIFT ; -if (mapping_mapped (mapping)) +if (mapping_mapped (mapping)) { +/* +* Calling unmap_mapping_range like this is wrong, +* because it can lead to mlocked pages being +* discarded (this is true even before the Noreclaim +* mlock work). direct-IO vs pagecache is a load of +* junk anyway, so who cares. +*/ unmap_mapping_range (mapping, offset, write_len,0 ); + } } retval =filemap_write_and_wait (mapping);
include/linux/vmstat.h +10 -0
@@ -149,6 +149,16 @@ static inline unsigned long zone_page_streturn x; } +extern unsigned long global_lru_pages (void ); + +static inline unsigned long zone_lru_pages (struct zone *zone) +{ +return (zone_page_state (zone,NR_ACTIVE_ANON ) + +zone_page_state (zone,NR_ACTIVE_FILE ) + +zone_page_state (zone,NR_INACTIVE_ANON ) + +zone_page_state (zone,NR_INACTIVE_FILE )); +} +#ifdef CONFIG_NUMA /* * Determine the per node value of a stat item. This function
mm/page-writeback.c +2 -6
@@ -270,9 +270,7 @@ static unsigned long highmem_dirtyable_mstruct zone *z = &NODE_DATA (node)->node_zones[ZONE_HIGHMEM ]; - x +=zone_page_state (z,NR_FREE_PAGES ) - +zone_page_state (z,NR_INACTIVE ) - +zone_page_state (z,NR_ACTIVE ); + x +=zone_page_state (z,NR_FREE_PAGES ) +zone_lru_pages (z); }/* * Make sure that the number of highmem pages is never larger
@@ -290,9 +288,7 @@ static unsigned long determine_dirtyable {unsigned long x; - x =global_page_state (NR_FREE_PAGES ) - +global_page_state (NR_INACTIVE ) - +global_page_state (NR_ACTIVE ); + x =global_page_state (NR_FREE_PAGES ) +global_lru_pages ();if (!vm_highmem_is_dirtyable) x -=highmem_dirtyable_memory (x);
include/linux/swap.h +41 -3
68 lines not on this page · raw message
include/linux/memcontrol.h +6 -13
@@ -42,7 +42,7 @@ extern unsigned long mem_cgroup_isolate_ unsigned long *scanned, int order, int mode, struct zone *z, struct mem_cgroup *mem_cont, - int active); + int active, int file);extern void mem_cgroup_out_of_memory (struct mem_cgroup *mem,gfp_t gfp_mask);extern int mem_cgroup_cache_charge (struct page *page,struct mm_struct *mm,gfp_t gfp_mask);
@@ -69,10 +69,8 @@ extern void mem_cgroup_note_reclaim_prioextern void mem_cgroup_record_reclaim_priority (struct mem_cgroup *mem,int priority); -extern long mem_cgroup_calc_reclaim_active(struct mem_cgroup *mem,
Lines of this file are not shown. View the raw message.
mm/memcontrol.c +78 -92
170 lines not on this page · raw message
mm/mlock.c +279 -11
290 lines not on this page · raw message
mm/rmap.c +135 -25
160 lines not on this page · raw message
mm/mmap.c +37 -5
42 lines not on this page · raw message
The rest of this diff is not shown. View the raw message.
Signature · 1 line
All rights reversed.
Part of the diff is not shown here. View the raw message.