cpuset,mm: update tasks' mems_allowed in time
Subsystems (5)
RFC PATCH 3/3 cpuset,mm: update tasks' mems_allowed in time
This patch fixes allocating page cache/slab object on the unallowed node when memory spread is set by updating tasks' mems_allowed after its cpuset's mems is changed.
In order to update tasks' mems_allowed in time, we must modify the code of memory policy. Because the memory policy is applied in the process's context originally. After applying this patch, one task directly manipulates anothers mems_allowed, and we use alloc_lock in the task_struct to protect mems_allowed and memory policy of the task.
But in the fast path, we didn't use lock to protect them, because adding a lock may lead to performance regression. But if we don't add a lock,the task might see no nodes when changing cpuset's mems_allowed to some non-overlapping set. In order to avoid it, we set all new allowed nodes, then clear newly disallowed ones.
Signed-off-by Miao Xie <[email protected]>
--- include/linux/cpuset.h | 13 +++- include/linux/sched.h | 8 +- init/main.c | 6 ++- kernel/cpuset.c | 184 ++++++++++++------------------------------------ kernel/kthread.c | 2 + mm/mempolicy.c | 134 +++++++++++++++++++++++++---------- mm/page_alloc.c | 5 +- 7 files changed, 162 insertions(+), 190 deletions(-)
include/linux/cpuset.h +9 -4
@@ -18,7 +18,6 @@extern int number_of_cpusets;/* How many cpusets are defined in system? */ -extern int cpuset_init_early (void );extern int cpuset_init (void );extern void cpuset_init_smp (void );extern void cpuset_cpus_allowed (struct task_struct *p,struct cpumask *mask);
@@ -27,7 +26,6 @@ extern void cpuset_cpus_allowed_locked(struct task_struct *p,extern nodemask_t cpuset_mems_allowed (struct task_struct *p);#define cpuset_current_mems_allowed (current->mems_allowed)void cpuset_init_current_mems_allowed (void ); -void cpuset_update_task_memory_state (void );int cpuset_nodemask_valid_mems_allowed (nodemask_t *nodemask);extern int __cpuset_node_allowed_softwall (int node,gfp_t gfp_mask);
@@ -92,9 +90,13 @@ extern void rebuild_sched_domains(void);extern void cpuset_print_task_mems_allowed (struct task_struct *p); +static inline void set_mems_allowed (nodemask_t nodemask) +{ + current->mems_allowed = nodemask; +} +#else /* !CONFIG_CPUSETS */ -static inline int cpuset_init_early (void ) {return 0 ; }static inline int cpuset_init (void ) {return 0 ; }static inline void cpuset_init_smp (void ) {}
@@ -116,7 +118,6 @@ static inline nodemask_t cpuset_mems_allowed(struct task_struct *p)#define cpuset_current_mems_allowed (node_states[N_HIGH_MEMORY])static inline void cpuset_init_current_mems_allowed (void ) {} -static inline void cpuset_update_task_memory_state (void ) {}static inline int cpuset_nodemask_valid_mems_allowed (nodemask_t *nodemask) {
@@ -188,6 +189,10 @@ static inline void cpuset_print_task_mems_allowed(struct task_struct *p) { } +static inline void set_mems_allowed (nodemask_t nodemask) +{ +} +#endif /* !CONFIG_CPUSETS */ #endif /* _LINUX_CPUSET_H */
include/linux/sched.h +4 -4
@@ -1300,7 +1300,8 @@ struct task_struct {/* Thread group tracking */ u32 parent_exec_id;u32 self_exec_id; -/* Protection of (de-)allocation: mm, files, fs, tty, keyrings */ +/* Protection of (de-)allocation: mm, files, fs, tty, keyrings, mems_allowed, +* mempolicy */ spinlock_t alloc_lock;/* Protection of the PI data structures: */
@@ -1363,8 +1364,7 @@ struct task_struct {cputime_t acct_timexpd;/* stime + utime since last update */ #endif #ifdef CONFIG_CPUSETS -nodemask_t mems_allowed; -int cpuset_mems_generation; +nodemask_t mems_allowed;/* Protected by alloc_lock */ int cpuset_mem_spread_rotor;#endif #ifdef CONFIG_CGROUPS
@@ -1382,7 +1382,7 @@ struct task_struct {struct futex_pi_state *pi_state_cache;#endif #ifdef CONFIG_NUMA -struct mempolicy *mempolicy; +struct mempolicy *mempolicy;/* Protected by alloc_lock */ short il_next;#endif atomic_t fs_excl;/* holding fs exclusive resources */
init/main.c +5 -1
@@ -650,7 +650,6 @@ asmlinkage void __init start_kernel(void)#endif vmalloc_init ();vfs_caches_init_early (); -cpuset_init_early ();page_cgroup_init ();mem_init ();enable_debug_pagealloc ();
@@ -857,6 +856,11 @@ static noinline int init_post(void)static int __init kernel_init (void * unused) {lock_kernel (); + +/* +* init can allocate pages on any node +*/ +set_mems_allowed (node_possible_map);/* * init can run on any cpu. */
kernel/cpuset.c +46 -138
304 lines not on this page · raw message
kernel/kthread.c +2 -0
@@ -13,6 +13,7 @@#include <linux/file.h> #include <linux/module.h> #include <linux/mutex.h> +#include <linux/cpuset.h> #include <trace/sched.h> #define KTHREAD_NICE_LEVEL (-5)
@@ -239,6 +240,7 @@ int kthreadd(void *unused)ignore_signals (tsk);set_user_nice (tsk,KTHREAD_NICE_LEVEL );set_cpus_allowed_ptr (tsk, cpu_all_mask); +set_mems_allowed (node_possible_map); current->flags |=PF_NOFREEZE |PF_FREEZER_NOSIG ;
mm/mempolicy.c +95 -39
240 lines not on this page · raw message
mm/page_alloc.c +1 -4
@@ -1632,10 +1632,7 @@ nofail_alloc:/* We now go into synchronous reclaim */ cpuset_memory_pressure_bump (); -/* -* The task's cpuset might have expanded its set of allowable nodes -*/ -cpuset_update_task_memory_state (); + p->flags |=PF_MEMALLOC ;lockdep_set_current_reclaim_state (gfp_mask);
Signature · 8 lines
1.6.0.3 -- To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to [email protected]. For more info on Linux MM, see: http://www.linux-mm.org/ . Don't email: <a href=mailto:"[email protected]"> [email protected] </a>