mempolicies: private pointer in check_range and MPOL_MF_INVERT
Subsystems (4)
PATCH 1/4 mempolicies: private pointer in check_range and MPOL_MF_INVERT
[PATCH] memory policies: Genericize check_range()
This was was first posted at
http://marc.theaimsgroup.com/?l=linux-mm&m=113149240227584&w=2
(Part of this functionality is also contained in the direct migration pathset. The functionality here is more generic and independent of that patchset.)
- Add internal flags MPOL_MF_INVERT to control check_range() behavior.
- Replace the pagelist passed through by check_range by a general private pointer that may be used for other purposes. (The following patches will use that to merge numa_maps into mempolicy.c and to better group the page migration code in the policy layer)
- Improve some comments.
mm/mempolicy.c +27 -23
@@ -88,8 +88,9 @@#include <asm/tlbflush.h> #include <asm/uaccess.h> -/* Internal MPOL_MF_xxx flags */ +/* Internal flags */ #define MPOL_MF_DISCONTIG_OK (MPOL_MF_INTERNAL << 0)/* Skip checks for continuous vmas */ +#define MPOL_MF_INVERT (MPOL_MF_INTERNAL << 1)/* Invert check for nodemask */ static kmem_cache_t *policy_cache;static kmem_cache_t *sn_cache;
@@ -232,11 +233,11 @@ static void migrate_page_add(struct vm_a } } -/* Ensure all existing pages follow the policy. */ +/* Scan through pages checking if pages follow certain conditions. */ static int check_pte_range (struct vm_area_struct *vma,pmd_t *pmd,unsigned long addr,unsigned long end,const nodemask_t *nodes,unsigned long flags, -struct list_head *pagelist) +void *private) {pte_t *orig_pte;pte_t *pte;
@@ -246,6 +247,7 @@ static int check_pte_range(struct vm_are do { unsigned long pfn; unsigned int nid; + struct page *page; if (!pte_present(*pte)) continue;
@@ -254,15 +256,16 @@ static int check_pte_range(struct vm_areprint_bad_pte (vma, *pte, addr);continue ; } - nid =pfn_to_nid (pfn); -if (!node_isset (nid, *nodes)) { -if (pagelist) { -struct page *page =pfn_to_page (pfn); + page =pfn_to_page (pfn); + nid =page_to_nid (page); +if (node_isset (nid, *nodes) == !!(flags &MPOL_MF_INVERT )) +continue ; + +if (flags & (MPOL_MF_MOVE |MPOL_MF_MOVE_ALL )) +migrate_page_add (vma, page, private, flags); +else +break ; -migrate_page_add (vma, page, pagelist, flags); - }else -break ; - } }while (pte++, addr +=PAGE_SIZE , addr != end);pte_unmap_unlock (orig_pte, ptl);return addr != end;
@@ -271,7 +274,7 @@ static int check_pte_range(struct vm_arestatic inline int check_pmd_range (struct vm_area_struct *vma,pud_t *pud,unsigned long addr,unsigned long end,const nodemask_t *nodes,unsigned long flags, -struct list_head *pagelist) +void *private) {pmd_t *pmd;unsigned long next;
@@ -282,7 +285,7 @@ static inline int check_pmd_range(structif (pmd_none_or_clear_bad (pmd))continue ;if (check_pte_range (vma, pmd, addr, next, nodes, - flags, pagelist)) + flags, private))return -EIO ; }while (pmd++, addr = next, addr != end);return 0 ;
@@ -291,7 +294,7 @@ static inline int check_pmd_range(structstatic inline int check_pud_range (struct vm_area_struct *vma,pgd_t *pgd,unsigned long addr,unsigned long end,const nodemask_t *nodes,unsigned long flags, -struct list_head *pagelist) +void *private) {pud_t *pud;unsigned long next;
@@ -302,7 +305,7 @@ static inline int check_pud_range(structif (pud_none_or_clear_bad (pud))continue ;if (check_pmd_range (vma, pud, addr, next, nodes, - flags, pagelist)) + flags, private))return -EIO ; }while (pud++, addr = next, addr != end);return 0 ;
@@ -311,7 +314,7 @@ static inline int check_pud_range(structstatic inline int check_pgd_range (struct vm_area_struct *vma,unsigned long addr,unsigned long end,const nodemask_t *nodes,unsigned long flags, -struct list_head *pagelist) +void *private) {pgd_t *pgd;unsigned long next;
@@ -322,7 +325,7 @@ static inline int check_pgd_range(structif (pgd_none_or_clear_bad (pgd))continue ;if (check_pud_range (vma, pgd, addr, next, nodes, - flags, pagelist)) + flags, private))return -EIO ; }while (pgd++, addr = next, addr != end);return 0 ;
@@ -344,7 +347,7 @@ static inline int vma_migratable(struct*/ static struct vm_area_struct *check_range (struct mm_struct *mm,unsigned long start,unsigned long end, -const nodemask_t *nodes,unsigned long flags,struct list_head *pagelist) +const nodemask_t *nodes,unsigned long flags,void *private) {int err;struct vm_area_struct *first, *vma, *prev;
@@ -373,7 +376,7 @@ check_range(struct mm_struct *mm, unsignif (vma->vm_start > start) start = vma->vm_start; err =check_pgd_range (vma, start, endvma, nodes, - flags, pagelist); + flags, private);if (err) { first =ERR_PTR (err);break ;
@@ -461,7 +464,8 @@ long do_mbind(unsigned long start, unsigint err;LIST_HEAD (pagelist); -if ((flags & ~(unsigned long )(MPOL_MF_STRICT |MPOL_MF_MOVE |MPOL_MF_MOVE_ALL )) +if ((flags & ~(unsigned long )(MPOL_MF_STRICT | +MPOL_MF_MOVE |MPOL_MF_MOVE_ALL )) || mode >MPOL_MAX )return -EINVAL ;if ((flags &MPOL_MF_MOVE_ALL ) && !capable (CAP_SYS_RESOURCE ))
@@ -499,8 +503,9 @@ long do_mbind(unsigned long start, unsig mode,nodes_addr(nodes)[0]);down_write (&mm->mmap_sem); - vma =check_range (mm, start, end, nmask, flags, - (flags & (MPOL_MF_MOVE |MPOL_MF_MOVE_ALL )) ? &pagelist :NULL ); + vma =check_range (mm, start, end, nmask, + flags |MPOL_MF_INVERT , &pagelist); + err =PTR_ERR (vma);if (!IS_ERR (vma)) {int nr_failed =0 ;
@@ -655,7 +660,6 @@ int do_migrate_pages(struct mm_struct *mnodemask_t nodes;nodes_andnot (nodes, *from_nodes, *to_nodes); -nodes_complement (nodes, nodes);down_read (&mm->mmap_sem); check_range(mm, mm->mmap->vm_start, TASK_SIZE, &nodes,
PATCH 2/4 Fold numa_maps into memopolicies.c
Fold numa_maps functionality into mempolicy.c
First discussed at http://marc.theaimsgroup.com/?t=113149255100001&r=1&w=2
- Use the check_range() in mempolicy.c to gather statistics.
- Improve the numa_maps code in general and fix some comments.
mm/mempolicy.c +137 -1
@@ -84,6 +84,8 @@#include <linux/compat.h> #include <linux/mempolicy.h> #include <linux/swap.h> +#include <linux/seq_file.h> +#include <linux/proc_fs.h> #include <asm/tlbflush.h> #include <asm/uaccess.h>
@@ -91,6 +93,7 @@/* Internal flags */ #define MPOL_MF_DISCONTIG_OK (MPOL_MF_INTERNAL << 0)/* Skip checks for continuous vmas */ #define MPOL_MF_INVERT (MPOL_MF_INTERNAL << 1)/* Invert check for nodemask */ +#define MPOL_MF_STATS (MPOL_MF_INTERNAL << 2)/* Gather statistics */ static kmem_cache_t *policy_cache;static kmem_cache_t *sn_cache;
@@ -233,6 +236,8 @@ static void migrate_page_add(struct vm_a } } +static void gather_stats (struct page *,void *); +/* Scan through pages checking if pages follow certain conditions. */ static int check_pte_range(struct vm_area_struct *vma, pmd_t *pmd, unsigned long addr, unsigned long end,
@@ -261,7 +266,9 @@ static int check_pte_range(struct vm_areif (node_isset (nid, *nodes) == !!(flags &MPOL_MF_INVERT ))continue ; -if (flags & (MPOL_MF_MOVE |MPOL_MF_MOVE_ALL )) +if (flags &MPOL_MF_STATS ) +gather_stats (page, private); +else if (flags & (MPOL_MF_MOVE |MPOL_MF_MOVE_ALL ))migrate_page_add (vma, page, private, flags);else break ;
@@ -1498,3 +1511,132 @@ void numa_policy_rebind(const nodemask_t {rebind_policy (current->mempolicy, old, new); } + +/* +* Display pages allocated per node and memory policy via /proc. +*/ + +static const char *policy_types[] = {"default" ,"prefer" ,"bind" , +"interleave" }; + +/* +* Convert a mempolicy into a string. +* Returns the number of characters in buffer (if positive) +* or an error (negative) +*/ +static inline int mpol_to_str (char *buffer,int maxlen,struct mempolicy *pol) +{ +char *p = buffer; +int l; +nodemask_t nodes; +int mode = pol ? pol->policy :MPOL_DEFAULT ; + +switch (mode) { +case MPOL_DEFAULT : +nodes_clear (nodes); +break ; + +case MPOL_PREFERRED : +nodes_clear (nodes); +node_set (pol->v.preferred_node, nodes); +break ; + +case MPOL_BIND : +get_zonemask (pol, &nodes); +break ; + +case MPOL_INTERLEAVE : + nodes = pol->v.nodes; +break ; + +default : +BUG (); +return -EFAULT ; + } + + l =strlen (policy_types[mode]); +if (buffer + maxlen < p + l +1 ) +return -ENOSPC ; + +strcpy (p, policy_types[mode]); + p += l; + +if (!nodes_empty (nodes)) { +if (buffer + maxlen < p +2 ) +return -ENOSPC ; + *p++ ='=' ; + p +=nodelist_scnprintf (p, buffer + maxlen - p, nodes); + } +return p - buffer; +} + +struct numa_maps { +unsigned long pages; +unsigned long anon; +unsigned long mapped; +unsigned long mapcount_max; +unsigned long node[MAX_NUMNODES ]; +}; + +static void gather_stats (struct page *page,void *private) +{ +struct numa_maps *md = private; +int count =page_mapcount (page); + +if (count) + md->mapped++; + +if (count > md->mapcount_max) + md->mapcount_max = count; + + md->pages++; + +if (PageAnon (page)) + md->anon++; + + md->node[page_to_nid (page)]++; +cond_resched (); +} + +int show_numa_map (struct seq_file *m,void *v) +{ +struct task_struct *task = m->private; +struct vm_area_struct *vma = v; +struct numa_maps *md; +int n; +char buffer[50 ]; + +if (!vma->vm_mm) +return 0 ; + + md =kzalloc (sizeof (struct numa_maps ),GFP_KERNEL ); +if (!md) +return 0 ; + +check_pgd_range (vma, vma->vm_start, vma->vm_end, + &node_online_map,MPOL_MF_STATS , md); + +if (md->pages) { +mpol_to_str (buffer,sizeof (buffer), +get_vma_policy (task, vma, vma->vm_start)); + +seq_printf (m,"%08lx %s pages=%lu mapped=%lu maxref=%lu" , + vma->vm_start, buffer, md->pages, + md->mapped, md->mapcount_max); + +if (md->anon) +seq_printf (m," anon=%lu" ,md->anon); + +for_each_online_node (n) +if (md->node[n]) +seq_printf (m," N%d=%lu" , n, md->node[n]); + +seq_putc (m,'\n' ); + } +kfree (md); + +if (m->count < m->size) + m->version = (vma !=get_gate_vma (task)) ? vma->vm_start :0 ; +return 0 ; +} +
fs/proc/task_mmu.c +5 -123
135 lines not on this page · raw message
PATCH 3/4 mempolicies: unexport get_vma_policy()
In reply to Christoph LameterSince the numa_maps functionality is now in mempolicy.c we no longer need to export get_vma_policy().
mm/mempolicy.c +2 -2
@@ -943,8 +943,8 @@ asmlinkage long compat_sys_mbind(compat_#endif /* Return effective policy for a VMA */ -struct mempolicy * -get_vma_policy (struct task_struct *task,struct vm_area_struct *vma,unsigned long addr) +static struct mempolicy *get_vma_policy (struct task_struct *task, +struct vm_area_struct *vma,unsigned long addr) {struct mempolicy *pol = task->mempolicy;
include/linux/mempolicy.h +0 -3
@@ -144,9 +144,6 @@ void mpol_free_shared_policy(struct sharstruct mempolicy *mpol_shared_policy_lookup (struct shared_policy *sp,unsigned long idx); -struct mempolicy *get_vma_policy (struct task_struct *task, -struct vm_area_struct *vma,unsigned long addr); -extern void numa_default_policy (void );extern void numa_policy_init (void );extern void numa_policy_rebind (const nodemask_t *old,const nodemask_t *new);
PATCH 4/4 Move page migration related functions near do_migrate_pages()
In reply to Christoph LameterGroup page migration functions in mempolicy.c
Add a forward declaration for migrate_page_add (like gather_stats()) and use our new found mobility to group all page migration related function around do_migrate_pages().
mm/mempolicy.c +138 -132
@@ -188,55 +188,9 @@ static struct mempolicy *mpol_new(int moreturn policy; } -/* Check if we are the only process mapping the page in question */ -static inline int single_mm_mapping (struct mm_struct *mm, -struct address_space *mapping) -{ -struct vm_area_struct *vma; -struct prio_tree_iter iter; -int rc =1 ; - -spin_lock (&mapping->i_mmap_lock); - vma_prio_tree_foreach(vma, &iter, &mapping->i_mmap, 0, ULONG_MAX) -if (mm != vma->vm_mm) { - rc =0 ; - goto out; - } -list_for_each_entry (vma, &mapping->i_mmap_nonlinear, shared.vm_set.list) -if (mm != vma->vm_mm) { - rc =0 ; - goto out; - } -out: -spin_unlock (&mapping->i_mmap_lock); -return rc; -} - -/* -* Add a page to be migrated to the pagelist -*/ -static void migrate_page_add (struct vm_area_struct *vma, -struct page *page,struct list_head *pagelist,unsigned long flags) -{ -/* -* Avoid migrating a page that is shared by others and not writable. -*/ -if ((flags &MPOL_MF_MOVE_ALL ) || !page->mapping ||PageAnon (page) || -mapping_writably_mapped (page->mapping) || -single_mm_mapping (vma->vm_mm, page->mapping)) { -int rc =isolate_lru_page (page); - -if (rc ==1 ) -list_add (&page->lru, pagelist); -/* -* If the isolate attempt was not successful then we just -* encountered an unswappable page. Something must be wrong. -*/ -WARN_ON (rc ==0 ); - } -} -static void gather_stats (struct page *,void *); +static void migrate_page_add (struct vm_area_struct *vma, +struct page *page,struct list_head *pagelist,unsigned long flags);/* Scan through pages checking if pages follow certain conditions. */ static int check_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
@@ -448,90 +402,6 @@ static int contextualize_policy(int modereturn mpol_check_policy (mode, nodes); } -static int swap_pages (struct list_head *pagelist) -{ -LIST_HEAD (moved); -LIST_HEAD (failed); -int n; - - n =migrate_pages (pagelist,NULL , &moved, &failed); -putback_lru_pages (&failed); -putback_lru_pages (&moved); - -return n; -} - -long do_mbind (unsigned long start,unsigned long len, -unsigned long mode,nodemask_t *nmask,unsigned long flags) -{ -struct vm_area_struct *vma; -struct mm_struct *mm = current->mm; -struct mempolicy *new; -unsigned long end; -int err; -LIST_HEAD (pagelist); - -if ((flags & ~(unsigned long )(MPOL_MF_STRICT | -MPOL_MF_MOVE |MPOL_MF_MOVE_ALL )) - || mode >MPOL_MAX ) -return -EINVAL ; -if ((flags &MPOL_MF_MOVE_ALL ) && !capable (CAP_SYS_RESOURCE )) -return -EPERM ; - -if (start & ~PAGE_MASK ) -return -EINVAL ; - -if (mode ==MPOL_DEFAULT ) - flags &= ~MPOL_MF_STRICT ; - - len = (len +PAGE_SIZE -1 ) &PAGE_MASK ; - end = start + len; - -if (end < start) -return -EINVAL ; -if (end == start) -return 0 ; - -if (mpol_check_policy (mode, nmask)) -return -EINVAL ; - - new =mpol_new (mode, nmask); -if (IS_ERR (new)) -return PTR_ERR (new); - -/* -* If we are using the default policy then operation -* on discontinuous address spaces is okay after all -*/ -if (!new) - flags |=MPOL_MF_DISCONTIG_OK ; - -PDprintk ("mbind %lx-%lx mode:%ld nodes:%lx\n" ,start,start+len, - mode,nodes_addr (nodes)[0 ]); - -down_write (&mm->mmap_sem); - vma =check_range (mm, start, end, nmask, - flags |MPOL_MF_INVERT , &pagelist); - - err =PTR_ERR (vma); -if (!IS_ERR (vma)) { -int nr_failed =0 ; - - err =mbind_range (vma, start, end, new); -if (!list_empty (&pagelist)) - nr_failed =swap_pages (&pagelist); - -if (!err && nr_failed && (flags &MPOL_MF_STRICT )) - err = -EIO ; - } -if (!list_empty (&pagelist)) -putback_lru_pages (&pagelist); - -up_write (&mm->mmap_sem); -mpol_free (new); -return err; -} -/* Set the process memory policy */ long do_set_mempolicy (int mode,nodemask_t *nodes) {
@@ -652,6 +522,71 @@ long do_get_mempolicy(int *policy, nodem }/* +* page migration +*/ + +/* Check if we are the only process mapping the page in question */ +static inline int single_mm_mapping (struct mm_struct *mm, +struct address_space *mapping) +{ +struct vm_area_struct *vma; +struct prio_tree_iter iter; +int rc =1 ; + +spin_lock (&mapping->i_mmap_lock); + vma_prio_tree_foreach(vma, &iter, &mapping->i_mmap, 0, ULONG_MAX) +if (mm != vma->vm_mm) { + rc =0 ; + goto out; + } +list_for_each_entry (vma, &mapping->i_mmap_nonlinear, shared.vm_set.list) +if (mm != vma->vm_mm) { + rc =0 ; + goto out; + } +out: +spin_unlock (&mapping->i_mmap_lock); +return rc; +} + +/* +* Add a page to be migrated to the pagelist +*/ +static void migrate_page_add (struct vm_area_struct *vma, +struct page *page,struct list_head *pagelist,unsigned long flags) +{ +/* +* Avoid migrating a page that is shared by others and not writable. +*/ +if ((flags &MPOL_MF_MOVE_ALL ) || !page->mapping ||PageAnon (page) || +mapping_writably_mapped (page->mapping) || +single_mm_mapping (vma->vm_mm, page->mapping)) { +int rc =isolate_lru_page (page); + +if (rc ==1 ) +list_add (&page->lru, pagelist); +/* +* If the isolate attempt was not successful then we just +* encountered an unswappable page. Something must be wrong. +*/ +WARN_ON (rc ==0 ); + } +} + +static int swap_pages (struct list_head *pagelist) +{ +LIST_HEAD (moved); +LIST_HEAD (failed); +int n; + + n =migrate_pages (pagelist,NULL , &moved, &failed); +putback_lru_pages (&failed); +putback_lru_pages (&moved); + +return n; +} + +/* * For now migrate_pages simply swaps out the pages from nodes that are in * the source set but not in the target set. In the future, we would * want a function that moves pages between the two nodesets in such
@@ -681,6 +616,77 @@ int do_migrate_pages(struct mm_struct *mreturn count; } +long do_mbind (unsigned long start,unsigned long len, +unsigned long mode,nodemask_t *nmask,unsigned long flags) +{ +struct vm_area_struct *vma; +struct mm_struct *mm = current->mm; +struct mempolicy *new; +unsigned long end; +int err; +LIST_HEAD (pagelist); + +if ((flags & ~(unsigned long )(MPOL_MF_STRICT | +MPOL_MF_MOVE |MPOL_MF_MOVE_ALL )) + || mode >MPOL_MAX ) +return -EINVAL ; +if ((flags &MPOL_MF_MOVE_ALL ) && !capable (CAP_SYS_RESOURCE )) +return -EPERM ; + +if (start & ~PAGE_MASK ) +return -EINVAL ; + +if (mode ==MPOL_DEFAULT ) + flags &= ~MPOL_MF_STRICT ; + + len = (len +PAGE_SIZE -1 ) &PAGE_MASK ; + end = start + len; + +if (end < start) +return -EINVAL ; +if (end == start) +return 0 ; + +if (mpol_check_policy (mode, nmask)) +return -EINVAL ; + + new =mpol_new (mode, nmask); +if (IS_ERR (new)) +return PTR_ERR (new); + +/* +* If we are using the default policy then operation +* on discontinuous address spaces is okay after all +*/ +if (!new) + flags |=MPOL_MF_DISCONTIG_OK ; + +PDprintk ("mbind %lx-%lx mode:%ld nodes:%lx\n" ,start,start+len, + mode,nodes_addr (nodes)[0 ]); + +down_write (&mm->mmap_sem); + vma =check_range (mm, start, end, nmask, + flags |MPOL_MF_INVERT , &pagelist); + + err =PTR_ERR (vma); +if (!IS_ERR (vma)) { +int nr_failed =0 ; + + err =mbind_range (vma, start, end, new); +if (!list_empty (&pagelist)) + nr_failed =swap_pages (&pagelist); + +if (!err && nr_failed && (flags &MPOL_MF_STRICT )) + err = -EIO ; + } +if (!list_empty (&pagelist)) +putback_lru_pages (&pagelist); + +up_write (&mm->mmap_sem); +mpol_free (new); +return err; +} +/* * User space interface with variable sized bitmaps for nodelists. */