mm: replicated pagecache
Subsystems (6)
RFC PATCH 1/1 mm: replicated pagecache
Hi,
This patch is getting a bit more stable now, if anyone wants to play with it.
Nick
--
Page-based NUMA pagecache replication.
This is a scheme for page replication replicates read-only pagecache pages opportunistically, at pagecache lookup time (at points where we know the page is being looked up for read only).
The page will be replicated if it resides on a different node to what the requesting CPU is on. Also, the original page must meet some conditions:
it must be clean, uptodate, not under writeback, and not have an elevated refcount or filesystem private data. However it is allowed to be mapped into pagetables.
Replication is done at the pagecache level, where a replicated pagecache (inode,offset) key will have a special bit set in its radix-tree entry, which tells us the entry points to a descriptor rather than a page.
This descriptor (struct pcache_desc) has another radix-tree which is keyed by node. The pagecache gains an (optional) 3rd dimension!
Pagecache lookups which are not explicitly denoted as being read-only are treaded as writes, and they collapse the replication before proceeding. Writes into pagetables are caught by using the same mechanism as dirty page throttling uses, and also collapse the replication.
After collapsing a replication, all process page tables are unmapped, so that any processes mapping discarded pages will refault in the correct one.
/proc/vmstat has nr_repl_pages, which is the number of _additional_ pages replicated, beyond the first.
Status:
- Use PG_replicated page flag rather than tagging the radix tree. Radix-tree code is now completely unchanged from upstream. We're very close to optimum performance in all fastpaths (eg. no extra radix-tree walks needed).
- Can now replicate S_ISREG and S_ISBLK pages with PagePrivate set. These guys should never get dirtied via the fs.
- Will promote a slave to be the new master when reclaiming the master. This Just transfer relevant flags, and only promote if !PagePrivate (reclaim has already taken care of this for us).
- Many more debug checks to catch violated invariants.
- Some more bugs fixed.
- Round-robin the replication source, rather than always copying from master. This should even out load on the interconnect.
- Lee showed that ~10s (1%) user time was cut off a kernel compile benchmark on his 4 node 16-way box.
Todo:
- find_get_page locking semantics are slightly changed. This doesn't appear to be a problem but I need to have a more thorough look.
- Would like to be able to control replication via userspace, and maybe even internally to the kernel.
- Ideally, reclaim might reclaim replicated pages preferentially, however I aim to be _minimally_ intrusive, and this conflicts with that.
- More correctness testing.
- Eventually, have to look at playing nicely with migration.
- radix-tree nodes start using up a large amount of memory. Try to improve. (eg. smaller tree, or don't load master immediately).
include/linux/mm.h | 6 include/linux/mm_types.h | 8 include/linux/mmzone.h | 3 include/linux/page-flags.h | 17 + init/main.c | 1 mm/Kconfig | 11 mm/Makefile | 1 mm/filemap.c | 105 ++------ mm/internal.h | 52 +++ mm/memory.c | 5 mm/page-writeback.c | 5 mm/page_alloc.c | 9 mm/replication.c | 588 +++++++++++++++++++++++++++++++++++++++++++++ mm/vmscan.c | 7 mm/vmstat.c | 3 15 files changed, 742 insertions(+), 79 deletions(-)
include/linux/mm_types.h +8 -0
@@ -5,6 +5,8 @@#include <linux/threads.h> #include <linux/list.h> #include <linux/spinlock.h> +#include <linux/radix-tree.h> +#include <linux/nodemask.h> struct address_space ;
@@ -64,4 +66,10 @@ struct page { #endif /* WANT_PAGE_VIRTUAL */ }; +struct pcache_desc { +struct page *master; +nodemask_t nodes_present; +struct radix_tree_root page_tree; +}; +#endif /* _LINUX_MM_TYPES_H */
mm/filemap.c +32 -73
216 lines not on this page · raw message
mm/internal.h +52 -0
62 lines not on this page · raw message
mm/vmscan.c +6 -1
@@ -390,6 +390,7 @@ int remove_mapping(struct address_spaceBUG_ON (!PageLocked (page));BUG_ON (mapping !=page_mapping (page)); +again:write_lock_irq (&mapping->tree_lock);/* * The non racy check for a busy page.
@@ -431,7 +432,11 @@ int remove_mapping(struct address_spacereturn 1 ; } -__remove_from_page_cache (page); +if (PageReplicated (page)) { +if (reclaim_replicated_page (mapping, page)) + goto again; + }else +__remove_from_page_cache (page);write_unlock_irq (&mapping->tree_lock);__put_page (page);return 1 ;
include/linux/mm.h +6 -0
@@ -1006,6 +1006,12 @@ extern void show_mem(void);extern void si_meminfo (struct sysinfo * val);extern void si_meminfo_node (struct sysinfo *val,int nid); +#ifdef CONFIG_REPLICATION +extern void replication_init (void ); +#else +static inline void replication_init (void ) {} +#endif +#ifdef CONFIG_NUMA extern void setup_per_cpu_pageset (void );#else
init/main.c +1 -0
@@ -599,6 +599,7 @@ asmlinkage void __init start_kernel(voidkmem_cache_init ();setup_per_cpu_pageset ();numa_policy_init (); +replication_init ();if (late_time_init)late_time_init ();calibrate_delay ();
include/linux/mmzone.h +3 -0
@@ -55,6 +55,9 @@ enum zone_stat_item {NR_FILE_MAPPED ,/* pagecache pages mapped into pagetables. only modified from process context */ NR_FILE_PAGES , +#ifdef CONFIG_REPLICATION +NR_REPL_PAGES , +#endifNR_FILE_DIRTY , NR_WRITEBACK,/* Second 128 byte cacheline */
mm/vmstat.c +3 -0
@@ -437,6 +437,9 @@ static const char * const vmstat_text[]"nr_anon_pages" ,"nr_mapped" ,"nr_file_pages" , +#ifdef CONFIG_REPLICATION +"nr_repl_pages" , +#endif"nr_dirty" ,"nr_writeback" , "nr_slab_reclaimable",
mm/Kconfig +11 -0
@@ -152,6 +152,17 @@ config MIGRATION example on NUMA systems to put pages nearer to the processors accessing the page. +# +# support for NUMA pagecache replication +# +config REPLICATION + bool "Pagecache replication" + def_bool n + depends on NUMA + help + Enables NUMA pagecache page replication + + config RESOURCES_64BIT bool "64 bit Memory and IO resources (EXPERIMENTAL)" if (!64BIT && EXPERIMENTAL) default 64BIT
mm/Makefile +1 -0
@@ -29,3 +29,4 @@ obj-$(CONFIG_MEMORY_HOTPLUG) += memory_h obj-$(CONFIG_FS_XIP) += filemap_xip.o obj-$(CONFIG_MIGRATION) += migrate.o obj-$(CONFIG_SMP) += allocpercpu.o +obj-$(CONFIG_REPLICATION) += replication.o
mm/replication.c new file +588 -0
588 lines not on this page · raw message
mm/memory.c +5 -0
@@ -60,6 +60,8 @@#include <linux/swapops.h> #include <linux/elf.h> +#include "internal.h" +#ifndef CONFIG_NEED_MULTIPLE_NODES /* use the per-pgdat data instead for discontigmem - mbligh */ unsigned long max_mapnr;
@@ -1573,6 +1575,7 @@ static int do_wp_page(struct mm_struct *page_cache_get (old_page);pte_unmap_unlock (page_table, ptl); + old_page =get_unreplicated_page_fault (old_page);if (vma->vm_ops->page_mkwrite (vma, old_page) <0 ) goto unwritable_page;
@@ -2248,6 +2251,8 @@ retry: anon =1 ; }else { + new_page =get_unreplicated_page_fault (new_page); +/* if the page will be shareable, see if the backing * address space wants to know that the page is about * to become writable */
mm/page-writeback.c +5 -0
@@ -864,7 +864,12 @@ int clear_page_dirty_for_io(struct page {struct address_space *mapping =page_mapping (page); +BUG_ON (!PageLocked (page)); +#ifndef CONFIG_REPLICATION if (mapping &&mapping_cap_account_dirty (mapping)) { +#else +if (mapping) { +#endif /* * Yes, Virginia, this is indeed insane. *
include/linux/page-flags.h +15 -2
@@ -91,6 +91,8 @@#define PG_nosave_free 18/* Used for system suspend/resume */ #define PG_buddy 19/* Page is free, on buddy lists */ +#define PG_replicated 20/* Page is replicated pagecache */ +#if (BITS_PER_LONG >32 )/*
@@ -139,8 +141,8 @@ static inline void SetPageUptodate(struc#define ClearPageUptodate (page) clear_bit(PG_uptodate, &(page)->flags)#define PageDirty (page) test_bit(PG_dirty, &(page)->flags) -#define SetPageDirty (page) set_bit(PG_dirty, &(page)->flags) -#define TestSetPageDirty (page) test_and_set_bit(PG_dirty, &(page)->flags) +#define SetPageDirty (page) do { BUG_ON(PageReplicated(page)); set_bit(PG_dirty, &(page)->flags); } while (0) +#define TestSetPageDirty (page) ({ BUG_ON(PageReplicated(page)); test_and_set_bit(PG_dirty, &(page)->flags); })#define ClearPageDirty (page) clear_bit(PG_dirty, &(page)->flags)#define __ClearPageDirty (page) __clear_bit(PG_dirty, &(page)->flags)#define TestClearPageDirty (page) test_and_clear_bit(PG_dirty, &(page)->flags)
@@ -183,6 +185,7 @@ static inline void SetPageUptodate(struc#define PageWriteback (page) test_bit(PG_writeback, &(page)->flags)#define SetPageWriteback (page) \ do { \ + BUG_ON(PageReplicated(page)); \ if (!test_and_set_bit(PG_writeback, \ &(page)->flags)) \ inc_zone_page_state(page, NR_WRITEBACK); \
@@ -190,6 +193,7 @@ static inline void SetPageUptodate(struc#define TestSetPageWriteback (page) \ ({ \ int ret; \ + BUG_ON(PageReplicated(page)); \ ret = test_and_set_bit(PG_writeback, \ &(page)->flags); \ if (!ret) \
@@ -226,6 +230,15 @@ static inline void SetPageUptodate(struc#define __SetPageBuddy (page) __set_bit(PG_buddy, &(page)->flags)#define __ClearPageBuddy (page) __clear_bit(PG_buddy, &(page)->flags) +#ifdef CONFIG_REPLICATION +#define PageReplicated (page) test_bit(PG_replicated, &(page)->flags) +#define __SetPageReplicated (page) do { BUG_ON(PageDirty(page) || PageWriteback(page)); __set_bit(PG_replicated, &(page)->flags); } while (0) +#define SetPageReplicated (page) do { BUG_ON(PageDirty(page) || PageWriteback(page)); set_bit(PG_replicated, &(page)->flags); } while (0) +#define ClearPageReplicated (page) clear_bit(PG_replicated, &(page)->flags) +#else +#define PageReplicated (page) 0 +#endif +#define PageMappedToDisk (page) test_bit(PG_mappedtodisk, &(page)->flags)#define SetPageMappedToDisk (page) set_bit(PG_mappedtodisk, &(page)->flags)#define ClearPageMappedToDisk (page) clear_bit(PG_mappedtodisk, &(page)->flags)
mm/page_alloc.c +6 -3
@@ -203,7 +203,8 @@ static void bad_page(struct page *page)1 << PG_slab |1 << PG_swapcache |1 << PG_writeback | - 1 << PG_buddy ); +1 << PG_buddy | + 1 << PG_replicated );set_page_count (page,0 );reset_page_mapcount (page); page->mapping =NULL ;
@@ -438,7 +439,8 @@ static inline int free_pages_check(struc 1 << PG_swapcache | 1 << PG_writeback | 1 << PG_reserved | - 1 << PG_buddy )))) + 1 << PG_buddy | + 1 << PG_replicated))))bad_page (page);if (PageDirty (page))__ClearPageDirty (page);
@@ -588,7 +590,8 @@ static int prep_new_page(struct page *pa 1 << PG_swapcache | 1 << PG_writeback | 1 << PG_reserved | - 1 << PG_buddy )))) + 1 << PG_buddy | + 1 << PG_replicated ))))bad_page (page);/*
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"[email protected]"> [email protected] </a>