mm/mpol: Add MPOL_MF_LAZY ...
Subsystems (4)
Commit-ID: 68d9661d42bf59830ae4ebae9e26d7ec6d288519 Gitweb: http://git.kernel.org/tip/68d9661d42bf59830ae4ebae9e26d7ec6d288519 Author: Lee Schermerhorn <[email protected]> AuthorDate: Thu, 12 Jan 2012 12:37:17 +0100 Committer: Ingo Molnar <[email protected]> CommitDate: Fri, 18 May 2012 08:16:13 +0200
mm/mpol: Add MPOL_MF_LAZY ...
This patch adds another mbind() flag to request "lazy migration". The flag, MPOL_MF_LAZY, modifies MPOL_MF_MOVE* such that the selected pages are simply unmapped from the calling task's page table ['_MOVE] or from all referencing page tables [_MOVE_ALL]. Anon pages will first be added to the swap [or migration?] cache, if necessary. The pages will be migrated in the fault path on "first touch", if the policy dictates at that time.
"Lazy Migration" will allow testing of migrate-on-fault via mbind(). Also allows applications to specify that only subsequently touched pages be migrated to obey new policy, instead of all pages in range. This can be useful for multi-threaded applications working on a large shared data area that is initialized by an initial thread resulting in all pages on one [or a few, if overflowed] nodes. After unmap, the pages in regions assigned to the worker threads will be automatically migrated local to the threads on 1st touch.
Signed-off-by Lee Schermerhorn <[email protected]>
Signed-off-by Peter Zijlstra <[email protected]>
Cc Suresh Siddha <[email protected]>
Cc Paul Turner <[email protected]>
Cc Dan Smith <[email protected]>
Cc Bharata B Rao <[email protected]>
Cc Lee Schermerhorn <[email protected]>
Cc Christoph Lameter <[email protected]>
Cc Rik van Riel <[email protected]>
Cc Andrea Arcangeli <[email protected]>
Cc Andrew Morton <[email protected]>
Cc Linus Torvalds <[email protected]>
Signed-off-by Ingo Molnar <[email protected]>
--- include/linux/mempolicy.h | 13 +++++- include/linux/migrate.h | 7 +++ include/linux/rmap.h | 5 +- mm/mempolicy.c | 20 ++++++---- mm/migrate.c | 96 ++++++++++++++++++++++++++++++++++++++++++++- mm/rmap.c | 6 +- 6 files changed, 129 insertions(+), 18 deletions(-)
include/linux/mempolicy.h +10 -3
@@ -48,9 +48,16 @@ enum mpol_rebind_step {/* Flags for mbind */ #define MPOL_MF_STRICT (1<<0)/* Verify existing pages in the mapping */ -#define MPOL_MF_MOVE (1<<1)/* Move pages owned by this process to conform to mapping */ -#define MPOL_MF_MOVE_ALL (1<<2)/* Move every page to conform to mapping */ -#define MPOL_MF_INTERNAL (1<<3)/* Internal flags start here */ +#define MPOL_MF_MOVE (1<<1)/* Move pages owned by this process to conform +to policy */ +#define MPOL_MF_MOVE_ALL (1<<2)/* Move every page to conform to policy */ +#define MPOL_MF_LAZY (1<<3)/* Modifies '_MOVE: lazy migrate on fault */ +#define MPOL_MF_INTERNAL (1<<4)/* Internal flags start here */ + +#define MPOL_MF_VALID (MPOL_MF_STRICT | \ + MPOL_MF_MOVE | \ + MPOL_MF_MOVE_ALL | \ + MPOL_MF_LAZY)/* * Internal flags that share the struct mempolicy flags word with
include/linux/migrate.h +7 -0
@@ -30,6 +30,8 @@ extern int migrate_vmas(struct mm_struct *mm,extern void migrate_page_copy (struct page *newpage,struct page *page);extern int migrate_huge_page_move_mapping (struct address_space *mapping,struct page *newpage,struct page *page); + +extern int migrate_pages_unmap_only (struct list_head *);#else static inline void putback_lru_pages (struct list_head *l) {}
@@ -59,6 +61,11 @@ static inline int migrate_huge_page_move_mapping(struct address_space *mapping,return -ENOSYS ; } +static inline int migrate_pages_unmap_only (struct list_head *l) +{ +return -ENOSYS ; +} +/* Possible settings for the migrate_page() method in address_operations */ #define migrate_page NULL#define fail_migrate_page NULL
include/linux/rmap.h +3 -2
@@ -163,8 +163,9 @@ int page_referenced_one(struct page *, struct vm_area_struct *,enum ttu_flags {TTU_UNMAP =0 ,/* unmap mode */ -TTU_MIGRATION =1 ,/* migration mode */ -TTU_MUNLOCK =2 ,/* munlock mode */ +TTU_MIGRATE_DIRECT =1 ,/* direct migration mode */ +TTU_MIGRATE_DEFERRED =2 ,/* deferred [lazy] migration mode */ +TTU_MUNLOCK =4 ,/* munlock mode */ TTU_ACTION_MASK =0xff ,TTU_IGNORE_MLOCK = (1 <<8 ),/* ignore mlock */
mm/mempolicy.c +12 -8
@@ -1048,8 +1048,7 @@ static long do_mbind(unsigned long start, unsigned long len,int err;LIST_HEAD (pagelist); -if (flags & ~(unsigned long )(MPOL_MF_STRICT | -MPOL_MF_MOVE |MPOL_MF_MOVE_ALL )) +if (flags & ~(unsigned long )MPOL_MF_VALID )return -EINVAL ;if ((flags &MPOL_MF_MOVE_ALL ) && !capable (CAP_SYS_NICE ))return -EPERM ;
@@ -1108,21 +1107,26 @@ static long do_mbind(unsigned long start, unsigned long len, vma =check_range (mm, start, end, nmask, flags |MPOL_MF_INVERT , &pagelist); - err =PTR_ERR (vma); -if (!IS_ERR (vma)) { -int nr_failed =0 ; - + err =PTR_ERR (vma);/* maybe ... */ +if (!IS_ERR (vma)) err =mbind_range (mm, start, end, new); +if (!err) { +int nr_failed =0 ; +if (!list_empty (&pagelist)) { - nr_failed =migrate_pages (&pagelist, new_vma_page, +if (flags &MPOL_MF_LAZY ) + nr_failed =migrate_pages_unmap_only (&pagelist); +else { + nr_failed =migrate_pages (&pagelist, new_vma_page, (unsigned long )vma, false, true); + }if (nr_failed)putback_lru_pages (&pagelist); } -if (!err && nr_failed && (flags &MPOL_MF_STRICT )) +if (nr_failed && (flags &MPOL_MF_STRICT )) err = -EIO ; }else putback_lru_pages (&pagelist);
mm/migrate.c +94 -2
114 lines not on this page · raw message
mm/rmap.c +3 -3
@@ -1305,13 +1305,13 @@ int try_to_unmap_one(struct page *page, struct vm_area_struct *vma,* pte. do_swap_page() will wait until the migration * pte is removed and then restart fault handling. */ -BUG_ON (TTU_ACTION (flags) !=TTU_MIGRATION ); +BUG_ON (TTU_ACTION (flags) !=TTU_MIGRATE_DIRECT ); entry =make_migration_entry (page,pte_write (pteval)); }set_pte_at (mm, address, pte,swp_entry_to_pte (entry));BUG_ON (pte_file (*pte)); }else if (IS_ENABLED (CONFIG_MIGRATION ) && - (TTU_ACTION (flags) ==TTU_MIGRATION )) { + (TTU_ACTION (flags) ==TTU_MIGRATE_DIRECT )) {/* Establish migration entry for a file page */ swp_entry_t entry; entry =make_migration_entry (page,pte_write (pteval));
@@ -1517,7 +1517,7 @@ static int try_to_unmap_anon(struct page *page, enum ttu_flags flags)* locking requirements of exec(), migration skips * temporary VMAs until after exec() completes. */ -if (IS_ENABLED (CONFIG_MIGRATION ) && (flags &TTU_MIGRATION ) && +if (IS_ENABLED (CONFIG_MIGRATION ) && (flags &TTU_MIGRATE_DIRECT ) &&is_vma_temporary_stack (vma))continue ;