Skip to contentSkip to conversation

NUMA policy support for file mappings

PATCH New Steve Longerbeam 1 participant 1 message Oct 13, 2004

Subsystems (7)
Memory management VFS MM core exec and binfmt +3 more subsystems
lore
Share & tools
lore

Steve Longerbeam

author
#0 lore raw

PATCH 1/1 NUMA policy support for file mappings

Hi,

Attached is a patch to 2.6.8-rc3 that adds NUMA mem policy
support for file mappings. With the patch, mbind() will "do the right thing" for file mapped regions, ie. page cache pages for the region will be allocated using the given policy, and all other vma's mapping the same region of the file will use the same policy. It does this by adding a shared_policy Red-Black tree to address_space. In more detail, the patch is doing the following:

1. add a shared_policy tree to the address_space object in fs.h.

2. modify page_cache_alloc() in pagemap.h to take a page index in addition to a mapping object, and use those to allocate a page for the page cache using the appopriate policy found in the mapping->policy tree.

3. modify filemap.c to pass the additional page offset to page_cache_alloc().

4. Also in filemap.c, implement generic file {set|get}_policy() methods and add those to generic_file_vm_ops.

5. In filemap_nopage(), verify that any existing page located in the cache is located in a node that satisfies the file's policy. If it's not
in a node that
satisfies the policy, it must be because the page was allocated
before the
file had any policies. If it's unused, free it and goto retry_find
(replace with
a new page using the file's policy). In addition to the page being
unused, the
replace is qualified with a new page flag, signifying that this page was allocated without the guidance of a shared policy. A similar
replacement is
done in exec.c:setup_arg_pages() for arg pages that were allocated
before any
policy was known for the stack region.

6. Init the file's shared policy in alloc_inode(), and free the shared policy in destroy_inode().

This patch also paves the way for executable images to contain policy info, so that their text and initialized data regions can be bound to specific nodes. I'm working on a utility ("elfmembind") that tags ELF executables and shared libs with policy info for text and data. A kernel patch to fs/binfmt_elf.c parses a new ELF section added by the tool, and generates a policy for the file's text and initialized data regions. The policies are then inserted into the file mapping's shared_policy tree via sys_mbind().

But I'm not sure a post-link time tool is the right approach for applying policies to executable loadable segments. ELF sections are really not supposed to be used at load time, only the PT_LOAD segments are applicable for program loading. Maybe a good long-term solution is to add policy support to the linker, ld. That way the policy info can be folded into the loadable segments (a NOTE segment might be the right approach). It's very difficult to modify loadable segments post-link time, unless there are some free bits or fields that can be hijacked. Then again, a post-link utility is more convenient than having to rebuild an app. Comments? Are there already plans afoot to add NUMA mem policies to ELF executables, and if so how?

Steve

fs/exec.c +6 -0
diff -Nuar -X /home/stevel/dontdiff 2.6.8-rc3.orig/fs/exec.c 2.6.8-rc3/fs/exec.c
@@ -439,6 +439,12 @@
 	for (i = 0 ; i < MAX_ARG_PAGES ; i++) {
 		struct page *page = bprm->page[i];
 		if (page) {
+			page = mpol_replace_invalid_anon_page(page, mpnt, 0);
+			if (!page) {
+				up_write(&mm->mmap_sem);
+				kmem_cache_free(vm_area_cachep, mpnt);
+				return -ENOMEM;
+			}
 			bprm->page[i] = NULL;
 			install_arg_page(mpnt, page, stack_base);
 		}
fs/inode.c +4 -1
diff -Nuar -X /home/stevel/dontdiff 2.6.8-rc3.orig/fs/inode.c 2.6.8-rc3/fs/inode.c
@@ -150,6 +150,7 @@
 		mapping_set_gfp_mask(mapping, GFP_HIGHUSER);
 		mapping->assoc_mapping = NULL;
 		mapping->backing_dev_info = &default_backing_dev_info;
+ 		mpol_shared_policy_init(&mapping->policy);
 
 		/*
 		 * If the block_device provides a backing_dev_info for client

@@ -177,8 +178,10 @@
 	security_inode_free(inode);
 	if (inode->i_sb->s_op->destroy_inode)
 		inode->i_sb->s_op->destroy_inode(inode);
-	else
+	else {
+		mpol_free_shared_policy(&inode->i_mapping->policy);
 		kmem_cache_free(inode_cachep, (inode));
+	}
 }
include/linux/fs.h +2 -0
diff -Nuar -X /home/stevel/dontdiff 2.6.8-rc3.orig/include/linux/fs.h 2.6.8-rc3/include/linux/fs.h
@@ -18,6 +18,7 @@
 #include <linux/cache.h>
 #include <linux/prio_tree.h>
 #include <linux/kobject.h>
+#include <linux/mempolicy.h>
 #include <asm/atomic.h>
 
 struct iovec;

@@ -339,6 +340,7 @@
 	atomic_t		truncate_count;	/* Cover race condition with truncate */
 	unsigned long		flags;		/* error bits/gfp mask */
 	struct backing_dev_info *backing_dev_info; /* device readahead, etc */
+	struct shared_policy    policy;         /* page alloc policy */
 	spinlock_t		private_lock;	/* for use by the address_space */
 	struct list_head	private_list;	/* ditto */
 	struct address_space	*assoc_mapping;	/* ditto */
include/linux/mempolicy.h +31 -1
diff -Nuar -X /home/stevel/dontdiff 2.6.8-rc3.orig/include/linux/mempolicy.h 2.6.8-rc3/include/linux/mempolicy.h
@@ -120,6 +120,12 @@
 extern int mpol_first_node(struct vm_area_struct *vma, unsigned long addr);
 extern int mpol_node_valid(int nid, struct vm_area_struct *vma,
 			unsigned long addr);
+extern struct page * mpol_replace_invalid_anon_page(struct page * page,
+						    struct vm_area_struct *vma,
+						    unsigned long addr);
+extern int mpol_remove_invalid_filemap_page(struct page * page,
+					    struct vm_area_struct *vma,
+					    unsigned long addr);
 
 /*
  * Tree of shared policies for a shared memory region.

@@ -152,7 +158,8 @@
 void mpol_free_shared_policy(struct shared_policy *p);
 struct mempolicy *mpol_shared_policy_lookup(struct shared_policy *sp,
 					    unsigned long idx);
-
+struct page *alloc_page_shared_policy(unsigned gfp, struct shared_policy *sp,
+				      unsigned long idx);
 extern void numa_default_policy(void);
 extern void numa_policy_init(void);
 

@@ -192,6 +199,22 @@
 	return 1;
 }
 
+static inline struct page *
+mpol_replace_invalid_anon_page(struct page * page,
+			       struct vm_area_struct *vma,
+			       unsigned long addr)
+{
+	return page;
+}
+
+static inline int
+mpol_remove_invalid_filemap_page(struct page * page,
+				 struct vm_area_struct *vma,
+				 unsigned long addr)
+{
+	return 0;
+}
+
 struct shared_policy {};
 
 static inline int mpol_set_shared_policy(struct shared_policy *info,

@@ -218,6 +241,13 @@
 #define vma_policy(vma) NULL
 #define vma_set_policy(vma, pol) do {} while(0)
 
+static inline struct page *
+alloc_page_shared_policy(unsigned gfp, struct shared_policy *sp,
+			 unsigned long idx)
+{
+	return alloc_pages(gfp, 0);
+}
+
 static inline void numa_policy_init(void)
 {
 }
include/linux/page-flags.h +6 -0
diff -Nuar -X /home/stevel/dontdiff 2.6.8-rc3.orig/include/linux/page-flags.h 2.6.8-rc3/include/linux/page-flags.h
@@ -77,6 +77,8 @@
 #define PG_compound		19	/* Part of a compound page */
 
 #define PG_anon			20	/* Anonymous: anon_vma in mapping */
+#define PG_sharedpolicy		21	/* Page was allocated for a file
+					   mapping using a shared_policy */
 
 
 /*

@@ -296,6 +298,10 @@
 #define SetPageAnon(page)	set_bit(PG_anon, &(page)->flags)
 #define ClearPageAnon(page)	clear_bit(PG_anon, &(page)->flags)
 
+#define PageSharedPolicy(page)      test_bit(PG_sharedpolicy, &(page)->flags)
+#define SetPageSharedPolicy(page)   set_bit(PG_sharedpolicy, &(page)->flags)
+#define ClearPageSharedPolicy(page) clear_bit(PG_sharedpolicy, &(page)->flags)
+
 #ifdef CONFIG_SWAP
 #define PageSwapCache(page)	test_bit(PG_swapcache, &(page)->flags)
 #define SetPageSwapCache(page)	set_bit(PG_swapcache, &(page)->flags)
include/linux/pagemap.h +14 -4
diff -Nuar -X /home/stevel/dontdiff 2.6.8-rc3.orig/include/linux/pagemap.h 2.6.8-rc3/include/linux/pagemap.h
@@ -50,14 +50,24 @@
 #define page_cache_release(page)	put_page(page)
 void release_pages(struct page **pages, int nr, int cold);
 
-static inline struct page *page_cache_alloc(struct address_space *x)
+
+static inline struct page *__page_cache_alloc(struct address_space *x,
+					      unsigned long idx,
+					      unsigned int gfp_mask)
+{
+	return alloc_page_shared_policy(gfp_mask, &x->policy, idx);
+}
+
+static inline struct page *page_cache_alloc(struct address_space *x,
+					    unsigned long idx)
 {
-	return alloc_pages(mapping_gfp_mask(x), 0);
+	return __page_cache_alloc(x, idx, mapping_gfp_mask(x));
 }
 
-static inline struct page *page_cache_alloc_cold(struct address_space *x)
+static inline struct page *page_cache_alloc_cold(struct address_space *x,
+						 unsigned long idx)
 {
-	return alloc_pages(mapping_gfp_mask(x)|__GFP_COLD, 0);
+	return __page_cache_alloc(x, idx, mapping_gfp_mask(x)|__GFP_COLD);
 }
 
 typedef int filler_t(void *, struct page *);
mm/filemap.c +36 -6

93 lines not on this page · raw message

mm/mempolicy.c +127 -3

161 lines not on this page · raw message

mm/readahead.c +1 -1
diff -Nuar -X /home/stevel/dontdiff 2.6.8-rc3.orig/mm/readahead.c 2.6.8-rc3/mm/readahead.c
@@ -246,7 +246,7 @@
 			continue;
 
 		spin_unlock_irq(&mapping->tree_lock);
-		page = page_cache_alloc_cold(mapping);
+		page = page_cache_alloc_cold(mapping, page_offset);
 		spin_lock_irq(&mapping->tree_lock);
 		if (!page)
 			break;
mm/shmem.c +1 -10
diff -Nuar -X /home/stevel/dontdiff 2.6.8-rc3.orig/mm/shmem.c 2.6.8-rc3/mm/shmem.c
@@ -824,16 +824,7 @@
 shmem_alloc_page(unsigned long gfp, struct shmem_inode_info *info,
 		 unsigned long idx)
 {
-	struct vm_area_struct pvma;
-	struct page *page;
-
-	memset(&pvma, 0, sizeof(struct vm_area_struct));
-	pvma.vm_policy = mpol_shared_policy_lookup(&info->policy, idx);
-	pvma.vm_pgoff = idx;
-	pvma.vm_end = PAGE_SIZE;
-	page = alloc_page_vma(gfp, &pvma, 0);
-	mpol_free(pvma.vm_policy);
-	return page;
+	return alloc_page_shared_policy(gfp, &info->policy, idx);
 }
 #else
 static inline struct page *