sched.h modernization -v2, phase #1: "Pre-splitup cleanups"
Subsystems (20)
PATCH 3/10 sched/headers: Make task_struct::wake_q an opaque pointer
In reply to Ingo MolnarTo be able to decouple wake_q functionality from <linux/sched.h> make the basic task_struct::wake_q pointer an opaque void *.
--- include/linux/sched.h | 2 +- kernel/fork.c | 2 +- kernel/sched/core.c | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-)
include/linux/sched.h +1 -1
@@ -1725,7 +1725,7 @@ struct task_struct {/* Protection of the PI data structures: */ raw_spinlock_t pi_lock; -struct wake_q_node wake_q; +void *wake_q;#ifdef CONFIG_RT_MUTEXES /* PI waiters blocked on a rt_mutex held by this task */
kernel/fork.c +1 -1
@@ -538,7 +538,7 @@ static struct task_struct *dup_task_struct(struct task_struct *orig, int node)#endif tsk->splice_pipe =NULL ; tsk->task_frag.page =NULL ; - tsk->wake_q.next =NULL ; + tsk->wake_q =NULL ;account_kernel_stack (tsk,1 );
kernel/sched/core.c +3 -3
@@ -426,7 +426,7 @@ static bool set_nr_if_polling(struct task_struct *p)void wake_q_add (struct wake_q_head *head,struct task_struct *task) { -struct wake_q_node *node = &task->wake_q; +struct wake_q_node *node = (void *)&task->wake_q;/* * Atomically grab the task, if ->wake_q is !nil already it means
@@ -455,11 +455,11 @@ void wake_up_q(struct wake_q_head *head)while (node !=WAKE_Q_TAIL ) {struct task_struct *task; - task = container_of(node, struct task_struct, wake_q); + task = container_of((void *)node, struct task_struct, wake_q);BUG_ON (!task);/* Task can safely be re-inserted now: */ node = node->next; - task->wake_q.next =NULL ; + task->wake_q =NULL ;/* * wake_up_process() implies a wmb() to pair with the queueing
Signature · 1 line
2.7.4
PATCH 5/10 sched/core: Remove the tsk_cpus_allowed() wrapper
In reply to Ingo MolnarSo the original intention of tsk_cpus_allowed() was to 'future-proof' the field - but it's pretty ineffectual at that, because half of the code uses ->cpus_allowed directly ...
Also, the wrapper makes the code longer than the original expression!
So just get rid of it. This also shrinks <linux/sched.h> a bit.
--- arch/powerpc/kernel/smp.c | 2 +- arch/powerpc/platforms/cell/spufs/sched.c | 2 +- arch/sparc/kernel/sysfs.c | 2 +- drivers/cpufreq/sparc-us2e-cpufreq.c | 4 ++-- drivers/cpufreq/sparc-us3-cpufreq.c | 4 ++-- drivers/infiniband/hw/hfi1/affinity.c | 2 +- drivers/infiniband/hw/hfi1/sdma.c | 2 +- include/linux/sched.h | 3 --- kernel/sched/core.c | 20 ++++++++++---------- kernel/sched/cpudeadline.c | 4 ++-- kernel/sched/cpupri.c | 4 ++-- kernel/sched/deadline.c | 7 +++---- kernel/sched/fair.c | 25 ++++++++++++------------- kernel/sched/rt.c | 5 ++--- lib/smp_processor_id.c | 2 +- samples/trace_events/trace-events-sample.c | 2 +- 16 files changed, 42 insertions(+), 48 deletions(-)
arch/powerpc/kernel/smp.c +1 -1
@@ -795,7 +795,7 @@ void __init smp_cpus_done(unsigned int max_cpus)* se we pin us down to CPU 0 for a short while */ alloc_cpumask_var (&old_mask,GFP_NOWAIT ); -cpumask_copy (old_mask,tsk_cpus_allowed (current)); +cpumask_copy (old_mask, ¤t->cpus_allowed);set_cpus_allowed_ptr (current,cpumask_of (boot_cpuid));if (smp_ops && smp_ops->setup_cpu)
arch/powerpc/platforms/cell/spufs/sched.c +1 -1
@@ -140,7 +140,7 @@ void __spu_update_sched_info(struct spu_context *ctx)* runqueue. The context will be rescheduled on the proper node * if it is timesliced or preempted. */ -cpumask_copy (&ctx->cpus_allowed,tsk_cpus_allowed (current)); +cpumask_copy (&ctx->cpus_allowed, ¤t->cpus_allowed);/* Save the current cpu id for spu interrupt routing. */ ctx->last_ran =raw_smp_processor_id ();
arch/sparc/kernel/sysfs.c +1 -1
@@ -106,7 +106,7 @@ static unsigned long run_on_cpu(unsigned long cpu,cpumask_t old_affinity;unsigned long ret; -cpumask_copy (&old_affinity,tsk_cpus_allowed (current)); +cpumask_copy (&old_affinity, ¤t->cpus_allowed);/* should return -EINVAL to userspace */ if (set_cpus_allowed_ptr (current,cpumask_of (cpu)))return 0 ;
drivers/cpufreq/sparc-us2e-cpufreq.c +2 -2
@@ -234,7 +234,7 @@ static unsigned int us2e_freq_get(unsigned int cpu)cpumask_t cpus_allowed;unsigned long clock_tick, estar; -cpumask_copy (&cpus_allowed,tsk_cpus_allowed (current)); +cpumask_copy (&cpus_allowed, ¤t->cpus_allowed);set_cpus_allowed_ptr (current,cpumask_of (cpu)); clock_tick =sparc64_get_clock_tick (cpu) /1000 ;
@@ -252,7 +252,7 @@ static int us2e_freq_target(struct cpufreq_policy *policy, unsigned int index)unsigned long clock_tick, divisor, old_divisor, estar;cpumask_t cpus_allowed; -cpumask_copy (&cpus_allowed,tsk_cpus_allowed (current)); +cpumask_copy (&cpus_allowed, ¤t->cpus_allowed);set_cpus_allowed_ptr (current,cpumask_of (cpu)); new_freq = clock_tick =sparc64_get_clock_tick (cpu) /1000 ;
drivers/cpufreq/sparc-us3-cpufreq.c +2 -2
@@ -82,7 +82,7 @@ static unsigned int us3_freq_get(unsigned int cpu)unsigned long reg;unsigned int ret; -cpumask_copy (&cpus_allowed,tsk_cpus_allowed (current)); +cpumask_copy (&cpus_allowed, ¤t->cpus_allowed);set_cpus_allowed_ptr (current,cpumask_of (cpu)); reg =read_safari_cfg ();
@@ -99,7 +99,7 @@ static int us3_freq_target(struct cpufreq_policy *policy, unsigned int index)unsigned long new_bits, new_freq, reg;cpumask_t cpus_allowed; -cpumask_copy (&cpus_allowed,tsk_cpus_allowed (current)); +cpumask_copy (&cpus_allowed, ¤t->cpus_allowed);set_cpus_allowed_ptr (current,cpumask_of (cpu)); new_freq =sparc64_get_clock_tick (cpu) /1000 ;
drivers/infiniband/hw/hfi1/affinity.c +1 -1
@@ -576,7 +576,7 @@ int hfi1_get_proc_affinity(int node)struct hfi1_affinity_node *entry;cpumask_var_t diff, hw_thread_mask, available_mask, intrs_mask;const struct cpumask *node_mask, - *proc_mask =tsk_cpus_allowed (current); + *proc_mask = ¤t->cpus_allowed;struct hfi1_affinity_node_list *affinity = &node_affinity;struct cpu_mask_set *set = &affinity->proc;
drivers/infiniband/hw/hfi1/sdma.c +1 -1
@@ -856,7 +856,7 @@ struct sdma_engine *sdma_select_user_engine(struct hfi1_devdata *dd, {struct sdma_rht_node *rht_node;struct sdma_engine *sde =NULL ; -const struct cpumask *current_mask =tsk_cpus_allowed (current); +const struct cpumask *current_mask = ¤t->cpus_allowed;unsigned long cpu_id;/*
include/linux/sched.h +0 -3
@@ -1999,9 +1999,6 @@ static inline struct vm_struct *task_stack_vm_area(const struct task_struct *t) }#endif -/* Future-safe accessor for struct task_struct's cpus_allowed. */ -#define tsk_cpus_allowed (tsk) (&(tsk)->cpus_allowed) -static inline int tsk_nr_cpus_allowed (struct task_struct *p) {return p->nr_cpus_allowed;
kernel/sched/core.c +10 -10
@@ -978,7 +978,7 @@ static struct rq *__migrate_task(struct rq *rq, struct task_struct *p, int dest_return rq;/* Affinity changed (again). */ -if (!cpumask_test_cpu (dest_cpu,tsk_cpus_allowed (p))) +if (!cpumask_test_cpu (dest_cpu, &p->cpus_allowed))return rq; rq =move_queued_task (rq, p, dest_cpu);
@@ -1255,10 +1255,10 @@ static int migrate_swap_stop(void *data)if (task_cpu (arg->src_task) != arg->src_cpu) goto unlock; -if (!cpumask_test_cpu (arg->dst_cpu,tsk_cpus_allowed (arg->src_task))) +if (!cpumask_test_cpu (arg->dst_cpu, &arg->src_task->cpus_allowed)) goto unlock; -if (!cpumask_test_cpu (arg->src_cpu,tsk_cpus_allowed (arg->dst_task))) +if (!cpumask_test_cpu (arg->src_cpu, &arg->dst_task->cpus_allowed)) goto unlock;__migrate_swap_task (arg->src_task, arg->dst_cpu);
@@ -1299,10 +1299,10 @@ int migrate_swap(struct task_struct *cur, struct task_struct *p)if (!cpu_active (arg.src_cpu) || !cpu_active (arg.dst_cpu)) goto out; -if (!cpumask_test_cpu (arg.dst_cpu,tsk_cpus_allowed (arg.src_task))) +if (!cpumask_test_cpu (arg.dst_cpu, &arg.src_task->cpus_allowed)) goto out; -if (!cpumask_test_cpu (arg.src_cpu,tsk_cpus_allowed (arg.dst_task))) +if (!cpumask_test_cpu (arg.src_cpu, &arg.dst_task->cpus_allowed)) goto out;trace_sched_swap_numa (cur, arg.src_cpu, p, arg.dst_cpu);
@@ -1486,14 +1486,14 @@ static int select_fallback_rq(int cpu, struct task_struct *p)for_each_cpu (dest_cpu, nodemask) {if (!cpu_active (dest_cpu))continue ; -if (cpumask_test_cpu (dest_cpu,tsk_cpus_allowed (p))) +if (cpumask_test_cpu (dest_cpu, &p->cpus_allowed))return dest_cpu; } }for (;;) {/* Any allowed, online CPU? */ -for_each_cpu (dest_cpu,tsk_cpus_allowed (p)) { +for_each_cpu (dest_cpu, &p->cpus_allowed) {if (!(p->flags &PF_KTHREAD ) && !cpu_active (dest_cpu))continue ;if (!cpu_online (dest_cpu))
@@ -1548,7 +1548,7 @@ int select_task_rq(struct task_struct *p, int cpu, int sd_flags, int wake_flags)if (tsk_nr_cpus_allowed (p) >1 ) cpu = p->sched_class->select_task_rq (p, cpu, sd_flags, wake_flags);else - cpu =cpumask_any (tsk_cpus_allowed (p)); + cpu =cpumask_any (&p->cpus_allowed);/* * In order not to call set_task_cpu() on a blocking task we need
@@ -1560,7 +1560,7 @@ int select_task_rq(struct task_struct *p, int cpu, int sd_flags, int wake_flags)* [ this allows ->select_task() to simply return task_cpu(p) and * not worry about this generic constraint ] */ -if (unlikely (!cpumask_test_cpu (cpu,tsk_cpus_allowed (p)) || +if (unlikely (!cpumask_test_cpu (cpu, &p->cpus_allowed) || !cpu_online (cpu))) cpu =select_fallback_rq (task_cpu (p), p);
@@ -5469,7 +5469,7 @@ int migrate_task_to(struct task_struct *p, int target_cpu)if (curr_cpu == target_cpu)return 0 ; -if (!cpumask_test_cpu (target_cpu,tsk_cpus_allowed (p))) +if (!cpumask_test_cpu (target_cpu, &p->cpus_allowed))return -EINVAL ;/* TODO: This is not properly updating schedstats */
kernel/sched/cpudeadline.c +2 -2
@@ -128,10 +128,10 @@ int cpudl_find(struct cpudl *cp, struct task_struct *p,const struct sched_dl_entity *dl_se = &p->dl;if (later_mask && -cpumask_and (later_mask, cp->free_cpus,tsk_cpus_allowed (p))) { +cpumask_and (later_mask, cp->free_cpus, &p->cpus_allowed)) { best_cpu =cpumask_any (later_mask); goto out; - }else if (cpumask_test_cpu (cpudl_maximum (cp),tsk_cpus_allowed (p)) && + }else if (cpumask_test_cpu (cpudl_maximum (cp), &p->cpus_allowed) &&dl_time_before (dl_se->deadline, cp->elements[0 ].dl)) { best_cpu =cpudl_maximum (cp);if (later_mask)
kernel/sched/cpupri.c +2 -2
@@ -103,11 +103,11 @@ int cpupri_find(struct cpupri *cp, struct task_struct *p,if (skip)continue ; -if (cpumask_any_and (tsk_cpus_allowed (p), vec->mask) >= nr_cpu_ids) +if (cpumask_any_and (&p->cpus_allowed, vec->mask) >= nr_cpu_ids)continue ;if (lowest_mask) { -cpumask_and (lowest_mask,tsk_cpus_allowed (p), vec->mask); +cpumask_and (lowest_mask, &p->cpus_allowed, vec->mask);/* * We have to ensure that we have at least one bit
kernel/sched/deadline.c +3 -4
@@ -252,7 +252,7 @@ static struct rq *dl_task_offline_migration(struct rq *rq, struct task_struct *p* If we cannot preempt any rq, fall back to pick any * online cpu. */ - cpu =cpumask_any_and (cpu_active_mask,tsk_cpus_allowed (p)); + cpu =cpumask_any_and (cpu_active_mask, &p->cpus_allowed);if (cpu >= nr_cpu_ids) {/* * Fail to find any suitable cpu.
@@ -1235,7 +1235,7 @@ static void set_curr_task_dl(struct rq *rq)static int pick_dl_task (struct rq *rq,struct task_struct *p,int cpu) {if (!task_running (rq, p) && -cpumask_test_cpu (cpu,tsk_cpus_allowed (p))) +cpumask_test_cpu (cpu, &p->cpus_allowed))return 1 ;return 0 ; }
@@ -1384,8 +1384,7 @@ static struct rq *find_lock_later_rq(struct task_struct *task, struct rq *rq)/* Retry if something changed. */ if (double_lock_balance (rq, later_rq)) {if (unlikely (task_rq (task) != rq || - !cpumask_test_cpu (later_rq->cpu, -tsk_cpus_allowed (task)) || + !cpumask_test_cpu (later_rq->cpu, &task->cpus_allowed) ||task_running (rq, task) || !dl_task (task) || !task_on_rq_queued (task))) {
kernel/sched/fair.c +12 -13
@@ -1551,7 +1551,7 @@ static void task_numa_compare(struct task_numa_env *env,*/ if (cur) {/* Skip this swap candidate if cannot move to the source cpu */ -if (!cpumask_test_cpu (env->src_cpu,tsk_cpus_allowed (cur))) +if (!cpumask_test_cpu (env->src_cpu, &cur->cpus_allowed)) goto unlock;/*
@@ -1661,7 +1661,7 @@ static void task_numa_find_cpu(struct task_numa_env *env,for_each_cpu (cpu,cpumask_of_node (env->dst_nid)) {/* Skip this CPU if the source task cannot migrate */ -if (!cpumask_test_cpu (cpu,tsk_cpus_allowed (env->p))) +if (!cpumask_test_cpu (cpu, &env->p->cpus_allowed))continue ; env->dst_cpu = cpu;
@@ -5458,7 +5458,7 @@ find_idlest_group(struct sched_domain *sd, struct task_struct *p,/* Skip over this group if it has no CPUs allowed */ if (!cpumask_intersects (sched_group_cpus (group), -tsk_cpus_allowed (p))) + &p->cpus_allowed))continue ; local_group = cpumask_test_cpu(this_cpu,
@@ -5578,7 +5578,7 @@ find_idlest_cpu(struct sched_group *group, struct task_struct *p, int this_cpu)return cpumask_first (sched_group_cpus (group));/* Traverse only the allowed CPUs */ -for_each_cpu_and (i,sched_group_cpus (group),tsk_cpus_allowed (p)) { +for_each_cpu_and (i,sched_group_cpus (group), &p->cpus_allowed) {if (idle_cpu (i)) {struct rq *rq =cpu_rq (i);struct cpuidle_state *idle =idle_get_state (rq);
@@ -5717,7 +5717,7 @@ static int select_idle_core(struct task_struct *p, struct sched_domain *sd, intif (!test_idle_cores (target, false))return -1 ; -cpumask_and (cpus,sched_domain_span (sd),tsk_cpus_allowed (p)); +cpumask_and (cpus,sched_domain_span (sd), &p->cpus_allowed);for_each_cpu_wrap (core, cpus, target, wrap) {bool idle = true;
@@ -5751,7 +5751,7 @@ static int select_idle_smt(struct task_struct *p, struct sched_domain *sd, int treturn -1 ;for_each_cpu (cpu,cpu_smt_mask (target)) { -if (!cpumask_test_cpu (cpu,tsk_cpus_allowed (p))) +if (!cpumask_test_cpu (cpu, &p->cpus_allowed))continue ;if (idle_cpu (cpu))return cpu;
@@ -5803,7 +5803,7 @@ static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, int t time =local_clock ();for_each_cpu_wrap (cpu,sched_domain_span (sd), target, wrap) { -if (!cpumask_test_cpu (cpu,tsk_cpus_allowed (p))) +if (!cpumask_test_cpu (cpu, &p->cpus_allowed))continue ;if (idle_cpu (cpu))break ;
@@ -5958,7 +5958,7 @@ select_task_rq_fair(struct task_struct *p, int prev_cpu, int sd_flag, int wake_fif (sd_flag &SD_BALANCE_WAKE ) {record_wakee (p); want_affine = !wake_wide (p) && !wake_cap (p, cpu, prev_cpu) - &&cpumask_test_cpu (cpu,tsk_cpus_allowed (p)); + &&cpumask_test_cpu (cpu, &p->cpus_allowed); }rcu_read_lock ();
@@ -6698,7 +6698,7 @@ int can_migrate_task(struct task_struct *p, struct lb_env *env)if (throttled_lb_pair (task_group (p), env->src_cpu, env->dst_cpu))return 0 ; -if (!cpumask_test_cpu (env->dst_cpu,tsk_cpus_allowed (p))) { +if (!cpumask_test_cpu (env->dst_cpu, &p->cpus_allowed)) {int cpu;schedstat_inc (p->se.statistics.nr_failed_migrations_affine);
@@ -6718,7 +6718,7 @@ int can_migrate_task(struct task_struct *p, struct lb_env *env)/* Prevent to re-select dst_cpu via env's cpus */ for_each_cpu_and (cpu, env->dst_grpmask, env->cpus) { -if (cpumask_test_cpu (cpu,tsk_cpus_allowed (p))) { +if (cpumask_test_cpu (cpu, &p->cpus_allowed)) { env->flags |=LBF_DST_PINNED ; env->new_dst_cpu = cpu;break ;
@@ -7252,7 +7252,7 @@ check_cpu_capacity(struct rq *rq, struct sched_domain *sd)/* * Group imbalance indicates (and tries to solve) the problem where balancing -* groups is inadequate due to tsk_cpus_allowed() constraints. +* groups is inadequate due to ->cpus_allowed constraints. * * Imagine a situation of two groups of 4 cpus each and 4 tasks each with a * cpumask covering 1 cpu of the first group and 3 cpus of the second group.
@@ -8211,8 +8211,7 @@ static int load_balance(int this_cpu, struct rq *this_rq,* if the curr task on busiest cpu can't be * moved to this_cpu */ -if (!cpumask_test_cpu (this_cpu, -tsk_cpus_allowed (busiest->curr))) { +if (!cpumask_test_cpu (this_cpu, &busiest->curr->cpus_allowed)) {raw_spin_unlock_irqrestore (&busiest->lock, flags); env.flags |=LBF_ALL_PINNED ;
kernel/sched/rt.c +2 -3
@@ -1591,7 +1591,7 @@ static void put_prev_task_rt(struct rq *rq, struct task_struct *p)static int pick_rt_task (struct rq *rq,struct task_struct *p,int cpu) {if (!task_running (rq, p) && -cpumask_test_cpu (cpu,tsk_cpus_allowed (p))) +cpumask_test_cpu (cpu, &p->cpus_allowed))return 1 ;return 0 ; }
@@ -1726,8 +1726,7 @@ static struct rq *find_lock_lowest_rq(struct task_struct *task, struct rq *rq)* Also make sure that it wasn't scheduled on its rq. */ if (unlikely (task_rq (task) != rq || - !cpumask_test_cpu (lowest_rq->cpu, -tsk_cpus_allowed (task)) || + !cpumask_test_cpu (lowest_rq->cpu, &task->cpus_allowed) ||task_running (rq, task) || !rt_task (task) || !task_on_rq_queued (task))) {
lib/smp_processor_id.c +1 -1
@@ -22,7 +22,7 @@ notrace static unsigned int check_preemption_disabled(const char *what1,* Kernel threads bound to a single CPU can safely use * smp_processor_id(): */ -if (cpumask_equal (tsk_cpus_allowed (current),cpumask_of (this_cpu))) +if (cpumask_equal (¤t->cpus_allowed,cpumask_of (this_cpu))) goto out;/*
samples/trace_events/trace-events-sample.c +1 -1
@@ -33,7 +33,7 @@ static void simple_thread_func(int cnt)/* Silly tracepoints */ trace_foo_bar ("hello" , cnt, array, random_strings[len], -tsk_cpus_allowed (current)); + ¤t->cpus_allowed);trace_foo_with_template_simple ("HELLO" , cnt);
Signature · 1 line
2.7.4
PATCH 6/10 sched/core: Remove the tsk_nr_cpus_allowed() wrapper
In reply to Ingo Molnartsk_nr_cpus_allowed() too is a pretty pointless wrapper that is not used consistently and which makes the code both harder to read and longer as well.
So remove it - this also shrinks <linux/sched.h> a bit.
--- include/linux/sched.h | 5 ----- kernel/sched/core.c | 2 +- kernel/sched/deadline.c | 28 ++++++++++++++-------------- kernel/sched/rt.c | 24 ++++++++++++------------ 4 files changed, 27 insertions(+), 32 deletions(-)
include/linux/sched.h +0 -5
@@ -1999,11 +1999,6 @@ static inline struct vm_struct *task_stack_vm_area(const struct task_struct *t) }#endif -static inline int tsk_nr_cpus_allowed (struct task_struct *p) -{ -return p->nr_cpus_allowed; -} -#define TNF_MIGRATED 0x01#define TNF_NO_GROUP 0x02#define TNF_SHARED 0x04
kernel/sched/core.c +1 -1
@@ -1545,7 +1545,7 @@ int select_task_rq(struct task_struct *p, int cpu, int sd_flags, int wake_flags) {lockdep_assert_held (&p->pi_lock); -if (tsk_nr_cpus_allowed (p) >1 ) +if (p->nr_cpus_allowed >1 ) cpu = p->sched_class->select_task_rq (p, cpu, sd_flags, wake_flags);else cpu =cpumask_any (&p->cpus_allowed);
kernel/sched/deadline.c +14 -14
@@ -134,7 +134,7 @@ static void inc_dl_migration(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq) {struct task_struct *p =dl_task_of (dl_se); -if (tsk_nr_cpus_allowed (p) >1 ) +if (p->nr_cpus_allowed >1 ) dl_rq->dl_nr_migratory++;update_dl_migration (dl_rq);
@@ -144,7 +144,7 @@ static void dec_dl_migration(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq) {struct task_struct *p =dl_task_of (dl_se); -if (tsk_nr_cpus_allowed (p) >1 ) +if (p->nr_cpus_allowed >1 ) dl_rq->dl_nr_migratory--;update_dl_migration (dl_rq);
@@ -958,7 +958,7 @@ static void enqueue_task_dl(struct rq *rq, struct task_struct *p, int flags)enqueue_dl_entity (&p->dl, pi_se, flags); -if (!task_current (rq, p) &&tsk_nr_cpus_allowed (p) >1 ) +if (!task_current (rq, p) && p->nr_cpus_allowed >1 )enqueue_pushable_dl_task (rq, p); }
@@ -1032,9 +1032,9 @@ select_task_rq_dl(struct task_struct *p, int cpu, int sd_flag, int flags)* try to make it stay here, it might be important. */ if (unlikely (dl_task (curr)) && - (tsk_nr_cpus_allowed (curr) <2 || + (curr->nr_cpus_allowed <2 || !dl_entity_preempt (&p->dl, &curr->dl)) && - (tsk_nr_cpus_allowed (p) >1 )) { + (p->nr_cpus_allowed >1 )) {int target =find_later_rq (p); if (target != -1 &&
@@ -1055,7 +1055,7 @@ static void check_preempt_equal_dl(struct rq *rq, struct task_struct *p)* Current can't be migrated, useless to reschedule, * let's hope p can move out. */ -if (tsk_nr_cpus_allowed (rq->curr) ==1 || +if (rq->curr->nr_cpus_allowed ==1 ||cpudl_find (&rq->rd->cpudl, rq->curr,NULL ) ==-1 )return ;
@@ -1063,7 +1063,7 @@ static void check_preempt_equal_dl(struct rq *rq, struct task_struct *p)* p is migratable, so let's not schedule it and * see if it is pushed or pulled somewhere else. */ -if (tsk_nr_cpus_allowed (p) !=1 && +if (p->nr_cpus_allowed !=1 &&cpudl_find (&rq->rd->cpudl, p,NULL ) !=-1 )return ;
@@ -1178,7 +1178,7 @@ static void put_prev_task_dl(struct rq *rq, struct task_struct *p) {update_curr_dl (rq); -if (on_dl_rq (&p->dl) &&tsk_nr_cpus_allowed (p) >1 ) +if (on_dl_rq (&p->dl) && p->nr_cpus_allowed >1 )enqueue_pushable_dl_task (rq, p); }
@@ -1279,7 +1279,7 @@ static int find_later_rq(struct task_struct *task)if (unlikely (!later_mask))return -1 ; -if (tsk_nr_cpus_allowed (task) ==1 ) +if (task->nr_cpus_allowed ==1 )return -1 ;/*
@@ -1424,7 +1424,7 @@ static struct task_struct *pick_next_pushable_dl_task(struct rq *rq)BUG_ON (rq->cpu !=task_cpu (p));BUG_ON (task_current (rq, p)); -BUG_ON (tsk_nr_cpus_allowed (p) <=1 ); +BUG_ON (p->nr_cpus_allowed <=1 );BUG_ON (!task_on_rq_queued (p));BUG_ON (!dl_task (p));
@@ -1463,7 +1463,7 @@ static int push_dl_task(struct rq *rq)*/ if (dl_task (rq->curr) &&dl_time_before (next_task->dl.deadline, rq->curr->dl.deadline) && -tsk_nr_cpus_allowed (rq->curr) >1 ) { + rq->curr->nr_cpus_allowed >1 ) {resched_curr (rq);return 0 ; }
@@ -1610,9 +1610,9 @@ static void task_woken_dl(struct rq *rq, struct task_struct *p) {if (!task_running (rq, p) && !test_tsk_need_resched (rq->curr) && -tsk_nr_cpus_allowed (p) >1 && + p->nr_cpus_allowed >1 &&dl_task (rq->curr) && - (tsk_nr_cpus_allowed (rq->curr) <2 || + (rq->curr->nr_cpus_allowed <2 || !dl_entity_preempt (&p->dl, &rq->curr->dl))) {push_dl_tasks (rq); }
@@ -1726,7 +1726,7 @@ static void switched_to_dl(struct rq *rq, struct task_struct *p)if (rq->curr != p) {#ifdef CONFIG_SMP -if (tsk_nr_cpus_allowed (p) >1 && rq->dl.overloaded) +if (p->nr_cpus_allowed >1 && rq->dl.overloaded)queue_push_tasks (rq);#endif if (dl_task (rq->curr))
kernel/sched/rt.c +12 -12
@@ -335,7 +335,7 @@ static void inc_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) rt_rq = &rq_of_rt_rq (rt_rq)->rt; rt_rq->rt_nr_total++; -if (tsk_nr_cpus_allowed (p) >1 ) +if (p->nr_cpus_allowed >1 ) rt_rq->rt_nr_migratory++;update_rt_migration (rt_rq);
@@ -352,7 +352,7 @@ static void dec_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) rt_rq = &rq_of_rt_rq (rt_rq)->rt; rt_rq->rt_nr_total--; -if (tsk_nr_cpus_allowed (p) >1 ) +if (p->nr_cpus_allowed >1 ) rt_rq->rt_nr_migratory--;update_rt_migration (rt_rq);
@@ -1324,7 +1324,7 @@ enqueue_task_rt(struct rq *rq, struct task_struct *p, int flags)enqueue_rt_entity (rt_se, flags); -if (!task_current (rq, p) &&tsk_nr_cpus_allowed (p) >1 ) +if (!task_current (rq, p) && p->nr_cpus_allowed >1 )enqueue_pushable_task (rq, p); }
@@ -1413,7 +1413,7 @@ select_task_rq_rt(struct task_struct *p, int cpu, int sd_flag, int flags)* will have to sort it out. */ if (curr &&unlikely (rt_task (curr)) && - (tsk_nr_cpus_allowed (curr) <2 || + (curr->nr_cpus_allowed <2 || curr->prio <= p->prio)) {int target =find_lowest_rq (p);
@@ -1437,7 +1437,7 @@ static void check_preempt_equal_prio(struct rq *rq, struct task_struct *p)* Current can't be migrated, useless to reschedule, * let's hope p can move out. */ -if (tsk_nr_cpus_allowed (rq->curr) ==1 || +if (rq->curr->nr_cpus_allowed ==1 || !cpupri_find (&rq->rd->cpupri, rq->curr,NULL ))return ;
@@ -1445,7 +1445,7 @@ static void check_preempt_equal_prio(struct rq *rq, struct task_struct *p)* p is migratable, so let's not schedule it and * see if it is pushed or pulled somewhere else. */ -if (tsk_nr_cpus_allowed (p) !=1 +if (p->nr_cpus_allowed !=1 &&cpupri_find (&rq->rd->cpupri, p,NULL ))return ;
@@ -1579,7 +1579,7 @@ static void put_prev_task_rt(struct rq *rq, struct task_struct *p)* The previous task needs to be made eligible for pushing * if it is still active */ -if (on_rt_rq (&p->rt) &&tsk_nr_cpus_allowed (p) >1 ) +if (on_rt_rq (&p->rt) && p->nr_cpus_allowed >1 )enqueue_pushable_task (rq, p); }
@@ -1629,7 +1629,7 @@ static int find_lowest_rq(struct task_struct *task)if (unlikely (!lowest_mask))return -1 ; -if (tsk_nr_cpus_allowed (task) ==1 ) +if (task->nr_cpus_allowed ==1 )return -1 ;/* No other targets possible */ if (!cpupri_find (&task_rq (task)->rd->cpupri, task, lowest_mask))
@@ -1761,7 +1761,7 @@ static struct task_struct *pick_next_pushable_task(struct rq *rq)BUG_ON (rq->cpu !=task_cpu (p));BUG_ON (task_current (rq, p)); -BUG_ON (tsk_nr_cpus_allowed (p) <=1 ); +BUG_ON (p->nr_cpus_allowed <=1 );BUG_ON (!task_on_rq_queued (p));BUG_ON (!rt_task (p));
@@ -2121,9 +2121,9 @@ static void task_woken_rt(struct rq *rq, struct task_struct *p) {if (!task_running (rq, p) && !test_tsk_need_resched (rq->curr) && -tsk_nr_cpus_allowed (p) >1 && + p->nr_cpus_allowed >1 && (dl_task (rq->curr) ||rt_task (rq->curr)) && - (tsk_nr_cpus_allowed (rq->curr) <2 || + (rq->curr->nr_cpus_allowed <2 || rq->curr->prio <= p->prio))push_rt_tasks (rq); }
@@ -2196,7 +2196,7 @@ static void switched_to_rt(struct rq *rq, struct task_struct *p)*/ if (task_on_rq_queued (p) && rq->curr != p) {#ifdef CONFIG_SMP -if (tsk_nr_cpus_allowed (p) >1 && rq->rt.overloaded) +if (p->nr_cpus_allowed >1 && rq->rt.overloaded)queue_push_tasks (rq);#endif /* CONFIG_SMP */ if (p->prio < rq->curr->prio)
Signature · 1 line
2.7.4
PATCH 1/10 sched/headers: Make all include/linux/sched/*.h headers build standalone
In reply to Ingo MolnarMake each header self-sufficient, so that it can be built successfully both in an allnoconfig and allyesconfig kernel.
Also standardize the naming of their header guards.
--- include/linux/sched/deadline.h | 8 +++++--- include/linux/sched/prio.h | 6 +++--- include/linux/sched/rt.h | 10 ++++++---- include/linux/sched/sysctl.h | 10 +++++++--- 4 files changed, 21 insertions(+), 13 deletions(-)
include/linux/sched/deadline.h +5 -3
@@ -1,5 +1,7 @@ -#ifndef _SCHED_DEADLINE_H -#define _SCHED_DEADLINE_H +#ifndef _LINUX_SCHED_DEADLINE_H +#define _LINUX_SCHED_DEADLINE_H + +#include <linux/sched.h> /* * SCHED_DEADLINE tasks has negative priorities, reflecting
@@ -26,4 +28,4 @@ static inline bool dl_time_before(u64 a, u64 b) return (s64)(a - b) < 0; } -#endif /* _SCHED_DEADLINE_H */ +#endif /* _LINUX_SCHED_DEADLINE_H */
include/linux/sched/prio.h +3 -3
@@ -1,5 +1,5 @@ -#ifndef _SCHED_PRIO_H -#define _SCHED_PRIO_H +#ifndef _LINUX_SCHED_PRIO_H +#define _LINUX_SCHED_PRIO_H#define MAX_NICE 19#define MIN_NICE -20
@@ -57,4 +57,4 @@ static inline long rlimit_to_nice(long prio) return (MAX_NICE - prio + 1); } -#endif /* _SCHED_PRIO_H */ +#endif /* _LINUX_SCHED_PRIO_H */
include/linux/sched/rt.h +6 -4
@@ -1,7 +1,9 @@ -#ifndef _SCHED_RT_H -#define _SCHED_RT_H +#ifndef _LINUX_SCHED_RT_H +#define _LINUX_SCHED_RT_H -#include <linux/sched/prio.h> +#include <linux/sched.h> + +struct task_struct ;static inline int rt_prio (int prio) {
@@ -57,4 +59,4 @@ extern void normalize_rt_tasks(void);*/ #define RR_TIMESLICE (100 * HZ / 1000) -#endif /* _SCHED_RT_H */ +#endif /* _LINUX_SCHED_RT_H */
include/linux/sched/sysctl.h +7 -3
@@ -1,5 +1,9 @@ -#ifndef _SCHED_SYSCTL_H -#define _SCHED_SYSCTL_H +#ifndef _LINUX_SCHED_SYSCTL_H +#define _LINUX_SCHED_SYSCTL_H + +#include <linux/types.h> + +struct ctl_table ;#ifdef CONFIG_DETECT_HUNG_TASK extern int sysctl_hung_task_check_count;
@@ -78,4 +82,4 @@ extern int sysctl_schedstats(struct ctl_table *table, int write, void __user *buffer, size_t *lenp, loff_t *ppos); -#endif /* _SCHED_SYSCTL_H */ +#endif /* _LINUX_SCHED_SYSCTL_H */
Signature · 1 line
2.7.4
cover sched.h modernization -v2, phase #1: "Pre-splitup cleanups"
This is the second version of the sched.h modernization series I posted two days ago:
https://lkml.org/lkml/2017/2/6/468
The point of the series is to significantly reduce the <linux/sched.h> footprint, to speed up the kernel build and to have a cleaner header structure. The new <linux/sched.h>'s typical preprocessed size goes down from ~0.68 MB (~22K lines) to ~0.45 MB (~15K lines) and is around 40% faster to build on typical configs.
The Git tree can be found at:
git git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git WIP.sched/core
Changes in -v2:
As per the suggestions from Linus, -v2 has a reworked patch split-up structure that should be easier to review, validate, test and bisect. I've further split up the patches into 148 component patches and reordered them into the following five groups/phases:
phase #1: "Pre-splitup cleanups": # 0721da143d99~1..582c7529a1bd, 10 patches
phase #2: "Prepare header dependencies": # 9017f600c534~1..d1eb6f0b580d, 48 patches phase #3: "Move definitions between headers": # 3d01f4b6e32d~1..fd88c1773dd6, 53 patches phase #4: "Remove header dependencies": # 2d246f83c5d3~1..2205323e0ca1, 35 patches
phase #5: "Post-splitup cleanups": # 55cf8bfb3af4~1..bcc6be9d66b2, 2 patches
The "pre-splitup cleanups" phase does a handful of preparatory changes that make the subsequent sched.h restructuring easier. Some of these patches have (mostly minor) semantic side-effects.
The middle phases are structured in a way to make it relatively easy to review and verify that they have no semantic side-effects:
"Prepare header dependencies":
- These patches just add trivial placeholder headers and update various .c and .h files with the expected future dependencies. These patches increase the header dependency graph of the kernel monotonically.
"Move definitions between headers":
- These are the meat of the code movement, but they don't actually change the header dependency graph: all code movement is between pre-existing headers.
"Remove header dependencies":
- These remove header #include lines and remove prototypes and thus shrink the header dependency graph monotonically to the desired end result. These are small patches but most breakage will likely bisect to these, so their finegrained split-up is useful as well.
The "post-splitup cleanups" do cleanups on top of the finished split-up header structure. It's two patches for now, more cleanups will be done later.
I've done a bisection step by step build test of all 148 steps on x86-defconfig, and have performed a fair amount of all[yes|mod|no]config tests, cross-build tests and boot tests on the final tree. Fixes to any bugs will be merged back into the series to preserve bisectability.
As a first step the "pre-splitup" patches are sent a reply to this 0/10 email.
I'll send the other phases in separate email threads, to make them easier to review.
As usual, any feedback is welcome!
Thanks,
Ingo
Ingo Molnar (10):
sched/headers: Make all include/linux/sched/*.h headers build standalone
sched/core: Convert ___assert_task_state() link time assert to BUILD_BUG_ON() sched/headers: Make task_struct::wake_q an opaque pointer
sched/core: Move the get_preempt_disable_ip() inline to sched/core.c
sched/core: Remove the tsk_cpus_allowed() wrapper
sched/core: Remove the tsk_nr_cpus_allowed() wrapper
rcu: Separate the RCU synchronization types and APIs into <linux/rcupdate_wait.h> sched/headers, cgroups: Remove the threadgroup_change_*() wrappery
mm/vmacache, sched/headers: Introduce 'struct vmacache' and move it from <linux/sched.h> to <linux/mm_types> kasan, sched/headers: Uninline kasan_enable/disable_current()
arch/powerpc/kernel/smp.c | 2 +- arch/powerpc/platforms/cell/spufs/sched.c | 2 +- arch/sparc/kernel/sysfs.c | 2 +- drivers/cpufreq/sparc-us2e-cpufreq.c | 4 ++-- drivers/cpufreq/sparc-us3-cpufreq.c | 4 ++-- drivers/infiniband/hw/hfi1/affinity.c | 2 +- drivers/infiniband/hw/hfi1/sdma.c | 2 +- fs/autofs4/autofs_i.h | 1 + fs/exec.c | 6 ++--- include/linux/cgroup-defs.h | 13 ++++++----- include/linux/dcache.h | 1 + include/linux/kasan.h | 10 ++------- include/linux/mm_types.h | 12 ++++++++++ include/linux/rcupdate.h | 40 --------------------------------- include/linux/rcupdate_wait.h | 50 +++++++++++++++++++++++++++++++++++++++++ include/linux/rcutiny.h | 11 ++------- include/linux/sched.h | 61 +++++--------------------------------------------- include/linux/sched/deadline.h | 8 ++++--- include/linux/sched/prio.h | 6 ++--- include/linux/sched/rt.h | 10 +++++---- include/linux/sched/sysctl.h | 10 ++++++--- include/linux/vmacache.h | 2 +- kernel/cgroup_pids.c | 2 +- kernel/debug/debug_core.c | 4 ++-- kernel/fork.c | 8 +++---- kernel/rcu/srcu.c | 2 +- kernel/rcu/tiny.c | 14 +++++++++++- kernel/rcu/tree.c | 2 +- kernel/rcu/update.c | 1 + kernel/sched/core.c | 41 +++++++++++++++++++++------------ kernel/sched/cpudeadline.c | 4 ++-- kernel/sched/cpupri.c | 4 ++-- kernel/sched/deadline.c | 35 ++++++++++++++--------------- kernel/sched/fair.c | 25 ++++++++++----------- kernel/sched/rt.c | 29 ++++++++++++------------ kernel/signal.c | 6 ++--- lib/sbitmap.c | 1 + lib/smp_processor_id.c | 2 +- mm/kasan/kasan.c | 10 +++++++++ mm/nommu.c | 2 +- mm/vmacache.c | 10 ++++----- samples/trace_events/trace-events-sample.c | 2 +- 42 files changed, 234 insertions(+), 229 deletions(-) create mode 100644 include/linux/rcupdate_wait.h
Signature · 1 line
2.7.4
PATCH 4/10 sched/core: Move the get_preempt_disable_ip() inline to sched/core.c
It's defined in <linux/sched.h>, but nothing outside the scheduler uses it - so move it to the sched/core.c usage site.
--- include/linux/sched.h | 9 --------- kernel/sched/core.c | 9 +++++++++ 2 files changed, 9 insertions(+), 9 deletions(-)
include/linux/sched.h +0 -9
@@ -3377,15 +3377,6 @@ static inline void cond_resched_rcu(void)#endif } -static inline unsigned long get_preempt_disable_ip (struct task_struct *p) -{ -#ifdef CONFIG_DEBUG_PREEMPT -return p->preempt_disable_ip; -#else -return 0 ; -#endif -} -/* * Does a critical section need to be broken due to another * task waiting?: (technically does not depend on CONFIG_PREEMPT,
kernel/sched/core.c +9 -0
@@ -3207,6 +3207,15 @@ static inline void preempt_latency_start(int val) { }static inline void preempt_latency_stop (int val) { }#endif +static inline unsigned long get_preempt_disable_ip (struct task_struct *p) +{ +#ifdef CONFIG_DEBUG_PREEMPT +return p->preempt_disable_ip; +#else +return 0 ; +#endif +} +/* * Print scheduling while atomic bug: */
Signature · 1 line
2.7.4
PATCH 2/10 sched/core: Convert ___assert_task_state() link time assert to BUILD_BUG_ON()
In reply to Ingo MolnarThe length of TASK_STATE_TO_CHAR_STR was still checked using the old link-time manual error method - convert it to BUILD_BUG_ON(). This has a couple of advantages:
- it's more obvious what's going on
- it reduces the size and complexity of <linux/sched.h>
- BUILD_BUG_ON() will fail during compilation, with a clearer error message than the link time assert.
--- include/linux/sched.h | 3 --- kernel/sched/core.c | 3 +++ 2 files changed, 3 insertions(+), 3 deletions(-)
include/linux/sched.h +0 -3
@@ -223,9 +223,6 @@ extern void proc_sched_set_task(struct task_struct *p);#define TASK_STATE_TO_CHAR_STR "RSDTtXZxKWPNn" -extern char ___assert_task_state[1 -2 *!!( -sizeof (TASK_STATE_TO_CHAR_STR )-1 !=ilog2 (TASK_STATE_MAX )+1 )]; -/* Convenience macros for the sake of set_current_state */ #define TASK_KILLABLE (TASK_WAKEKILL | TASK_UNINTERRUPTIBLE)#define TASK_STOPPED (TASK_WAKEKILL | __TASK_STOPPED)
kernel/sched/core.c +3 -0
@@ -5229,6 +5229,9 @@ void sched_show_task(struct task_struct *p)int ppid;unsigned long state = p->state; +/* Make sure the string lines up properly with the number of task states: */ +BUILD_BUG_ON (sizeof (TASK_STATE_TO_CHAR_STR )-1 !=ilog2 (TASK_STATE_MAX )+1 ); +if (!try_get_task_stack (p))return ;if (state)
Signature · 1 line
2.7.4
PATCH 8/10 sched/headers, cgroups: Remove the threadgroup_change_*() wrappery
In reply to Ingo Molnarthreadgroup_change_begin()/end() is a pointless wrapper around cgroup_threadgroup_change_begin()/end(), minus a might_sleep() in the !CONFIG_CGROUPS=y case.
Remove the wrappery, move the might_sleep() (the down_read() already has a might_sleep() check).
This debloats <linux/sched.h> a bit and simplifies this API.
Update all call sites.
No change in functionality.
--- fs/exec.c | 6 +++--- include/linux/cgroup-defs.h | 13 ++++++++----- include/linux/sched.h | 28 ---------------------------- kernel/cgroup_pids.c | 2 +- kernel/fork.c | 6 +++--- kernel/signal.c | 6 +++--- 6 files changed, 18 insertions(+), 43 deletions(-)
fs/exec.c +3 -3
@@ -1088,7 +1088,7 @@ static int de_thread(struct task_struct *tsk)struct task_struct *leader = tsk->group_leader;for (;;) { -threadgroup_change_begin (tsk); +cgroup_threadgroup_change_begin (tsk);write_lock_irq (&tasklist_lock);/* * Do this under tasklist_lock to ensure that
@@ -1099,7 +1099,7 @@ static int de_thread(struct task_struct *tsk)break ;__set_current_state (TASK_KILLABLE );write_unlock_irq (&tasklist_lock); -threadgroup_change_end (tsk); +cgroup_threadgroup_change_end (tsk);schedule ();if (unlikely (__fatal_signal_pending (tsk))) goto killed;
@@ -1157,7 +1157,7 @@ static int de_thread(struct task_struct *tsk)if (unlikely (leader->ptrace))__wake_up_parent (leader, leader->parent);write_unlock_irq (&tasklist_lock); -threadgroup_change_end (tsk); +cgroup_threadgroup_change_end (tsk);release_task (leader); }
include/linux/cgroup-defs.h +8 -5
@@ -528,8 +528,8 @@ extern struct percpu_rw_semaphore cgroup_threadgroup_rwsem;* cgroup_threadgroup_change_begin - threadgroup exclusion for cgroups * @tsk: target task * -* Called from threadgroup_change_begin() and allows cgroup operations to -* synchronize against threadgroup changes using a percpu_rw_semaphore. +* Allows cgroup operations to synchronize against threadgroup changes +* using a percpu_rw_semaphore. */ static inline void cgroup_threadgroup_change_begin (struct task_struct *tsk) {
@@ -540,8 +540,7 @@ static inline void cgroup_threadgroup_change_begin(struct task_struct *tsk)* cgroup_threadgroup_change_end - threadgroup exclusion for cgroups * @tsk: target task * -* Called from threadgroup_change_end(). Counterpart of -* cgroup_threadcgroup_change_begin(). +* Counterpart of cgroup_threadcgroup_change_begin(). */ static inline void cgroup_threadgroup_change_end (struct task_struct *tsk) {
@@ -552,7 +551,11 @@ static inline void cgroup_threadgroup_change_end(struct task_struct *tsk)#define CGROUP_SUBSYS_COUNT 0 -static inline void cgroup_threadgroup_change_begin (struct task_struct *tsk) {} +static inline void cgroup_threadgroup_change_begin (struct task_struct *tsk) +{ +might_sleep (); +} +static inline void cgroup_threadgroup_change_end (struct task_struct *tsk) {}#endif /* CONFIG_CGROUPS */
include/linux/sched.h +0 -28
@@ -3120,34 +3120,6 @@ static inline void unlock_task_sighand(struct task_struct *tsk,spin_unlock_irqrestore (&tsk->sighand->siglock, *flags); } -/** -* threadgroup_change_begin - mark the beginning of changes to a threadgroup -* @tsk: task causing the changes -* -* All operations which modify a threadgroup - a new thread joining the -* group, death of a member thread (the assertion of PF_EXITING) and -* exec(2) dethreading the process and replacing the leader - are wrapped -* by threadgroup_change_{begin|end}(). This is to provide a place which -* subsystems needing threadgroup stability can hook into for -* synchronization. -*/ -static inline void threadgroup_change_begin (struct task_struct *tsk) -{ -might_sleep (); -cgroup_threadgroup_change_begin (tsk); -} - -/** -* threadgroup_change_end - mark the end of changes to a threadgroup -* @tsk: task causing the changes -* -* See threadgroup_change_begin(). -*/ -static inline void threadgroup_change_end (struct task_struct *tsk) -{ -cgroup_threadgroup_change_end (tsk); -} -#ifdef CONFIG_THREAD_INFO_IN_TASK static inline struct thread_info *task_thread_info (struct task_struct *task)
kernel/cgroup_pids.c +1 -1
@@ -214,7 +214,7 @@ static void pids_cancel_attach(struct cgroup_taskset *tset)/* * task_css_check(true) in pids_can_fork() and pids_cancel_fork() relies -* on threadgroup_change_begin() held by the copy_process(). +* on cgroup_threadgroup_change_begin() held by the copy_process(). */ static int pids_can_fork (struct task_struct *task) {
kernel/fork.c +3 -3
@@ -1745,7 +1745,7 @@ static __latent_entropy struct task_struct *copy_process(INIT_LIST_HEAD (&p->thread_group); p->task_works =NULL ; -threadgroup_change_begin (current); +cgroup_threadgroup_change_begin (current);/* * Ensure that the cgroup subsystem policies allow the new process to be * forked. It should be noted the the new process's css_set can be changed
@@ -1835,7 +1835,7 @@ static __latent_entropy struct task_struct *copy_process(proc_fork_connector (p);cgroup_post_fork (p); -threadgroup_change_end (current); +cgroup_threadgroup_change_end (current);perf_event_fork (p);trace_task_newtask (p, clone_flags);
@@ -1846,7 +1846,7 @@ static __latent_entropy struct task_struct *copy_process( bad_fork_cancel_cgroup:cgroup_cancel_fork (p); bad_fork_free_pid: -threadgroup_change_end (current); +cgroup_threadgroup_change_end (current);if (pid != &init_struct_pid)free_pid (pid); bad_fork_cleanup_thread:
kernel/signal.c +3 -3
@@ -2395,11 +2395,11 @@ void exit_signals(struct task_struct *tsk)* @tsk is about to have PF_EXITING set - lock out users which * expect stable threadgroup. */ -threadgroup_change_begin (tsk); +cgroup_threadgroup_change_begin (tsk);if (thread_group_empty (tsk) ||signal_group_exit (tsk->signal)) { tsk->flags |=PF_EXITING ; -threadgroup_change_end (tsk); +cgroup_threadgroup_change_end (tsk);return ; }
@@ -2410,7 +2410,7 @@ void exit_signals(struct task_struct *tsk)*/ tsk->flags |=PF_EXITING ; -threadgroup_change_end (tsk); +cgroup_threadgroup_change_end (tsk);if (!signal_pending (tsk)) goto out;
Signature · 1 line
2.7.4
PATCH 10/10 kasan, sched/headers: Uninline kasan_enable/disable_current()
In reply to Ingo Molnar<linux/kasan.h> is a low level header that is included early in affected kernel headers. But it includes <linux/sched.h> which complicates the cleanup of sched.h dependencies.
But kasan.h has almost no need for sched.h: its only use of scheduler functionality is in two inline functions which are not used very frequently - so uninline kasan_enable_current() and kasan_disable_current().
Also add a <linux/sched.h> dependency to a .c file that depended on kasan.h including it.
This paves the way to remove the <linux/sched.h> include from kasan.h.
--- include/linux/kasan.h | 10 ++-------- lib/sbitmap.c | 1 + mm/kasan/kasan.c | 10 ++++++++++ 3 files changed, 13 insertions(+), 8 deletions(-)
include/linux/kasan.h +2 -8
@@ -30,16 +30,10 @@ static inline void *kasan_mem_to_shadow(const void *addr) }/* Enable reporting bugs after kasan_disable_current() */ -static inline void kasan_enable_current (void ) -{ - current->kasan_depth++; -} +extern void kasan_enable_current (void );/* Disable reporting bugs for current task */ -static inline void kasan_disable_current (void ) -{ - current->kasan_depth--; -} +extern void kasan_disable_current (void );void kasan_unpoison_shadow (const void *address,size_t size);
lib/sbitmap.c +1 -0
@@ -15,6 +15,7 @@* along with this program. If not, see <https://www.gnu.org/licenses/>. */ +#include <linux/sched.h> #include <linux/random.h> #include <linux/sbitmap.h>
mm/kasan/kasan.c +10 -0
@@ -39,6 +39,16 @@#include "kasan.h" #include "../slab.h" +void kasan_enable_current (void ) +{ + current->kasan_depth++; +} + +void kasan_disable_current (void ) +{ + current->kasan_depth--; +} +/* * Poisons the shadow memory for 'size' bytes starting from 'addr'. * Memory addresses should be aligned to KASAN_SHADOW_SCALE_SIZE.
Signature · 1 line
2.7.4
PATCH 7/10 rcu: Separate the RCU synchronization types and APIs into <linux/rcupdate_wait.h>
In reply to Ingo MolnarSo rcupdate.h is a pretty complex header, in particular it includes <linux/completion.h> which includes <linux/wait.h> - creating a dependency that includes <linux/wait.h> in <linux/sched.h>, which prevents the isolation of <linux/sched.h> from the derived <linux/wait.h> header.
Solve part of the problem by decoupling rcupdate.h from completions:
this can be done by separating out the rcu_synchronize types and APIs, and updating their usage sites.
Since this is a mostly RCU-internal types this will not just simplify <linux/sched.h>'s dependencies, but will make all the hundreds of .c files that include rcupdate.h but not completions or wait.h build faster.
( For rcutiny this means that two dependent APIs have to be uninlined,
but that shouldn't be much of a problem as they are rare variants. )
--- fs/autofs4/autofs_i.h | 1 + include/linux/dcache.h | 1 + include/linux/rcupdate.h | 40 ---------------------------------------- include/linux/rcupdate_wait.h | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++ include/linux/rcutiny.h | 11 ++--------- kernel/rcu/srcu.c | 2 +- kernel/rcu/tiny.c | 14 +++++++++++++- kernel/rcu/tree.c | 2 +- kernel/rcu/update.c | 1 + kernel/sched/core.c | 1 + 10 files changed, 71 insertions(+), 52 deletions(-)
fs/autofs4/autofs_i.h +1 -0
@@ -14,6 +14,7 @@#include <linux/mutex.h> #include <linux/spinlock.h> #include <linux/list.h> +#include <linux/completion.h> /* This is the range of ioctl() numbers we claim as ours */ #define AUTOFS_IOC_FIRST AUTOFS_IOC_READY
include/linux/dcache.h +1 -0
@@ -11,6 +11,7 @@#include <linux/rcupdate.h> #include <linux/lockref.h> #include <linux/stringhash.h> +#include <linux/wait.h> struct path ;struct vfsmount ;
include/linux/rcupdate.h +0 -40
@@ -40,7 +40,6 @@#include <linux/cpumask.h> #include <linux/seqlock.h> #include <linux/lockdep.h> -#include <linux/completion.h> #include <linux/debugobjects.h> #include <linux/bug.h> #include <linux/compiler.h>
@@ -226,45 +225,6 @@ void call_rcu_sched(struct rcu_head *head,void synchronize_sched (void ); -/* -* Structure allowing asynchronous waiting on RCU. -*/ -struct rcu_synchronize { -struct rcu_head head; -struct completion completion; -}; -void wakeme_after_rcu (struct rcu_head *head); - -void __wait_rcu_gp (bool checktiny,int n,call_rcu_func_t *crcu_array, -struct rcu_synchronize *rs_array); - -#define _wait_rcu_gp (checktiny, ...) \ -do { \ - call_rcu_func_t __crcu_array[] = { __VA_ARGS__ }; \ - struct rcu_synchronize __rs_array[ARRAY_SIZE(__crcu_array)]; \ - __wait_rcu_gp(checktiny, ARRAY_SIZE(__crcu_array), \ - __crcu_array, __rs_array); \ -} while (0) - -#define wait_rcu_gp (...) _wait_rcu_gp(false, __VA_ARGS__) - -/** -* synchronize_rcu_mult - Wait concurrently for multiple grace periods -* @...: List of call_rcu() functions for the flavors to wait on. -* -* This macro waits concurrently for multiple flavors of RCU grace periods. -* For example, synchronize_rcu_mult(call_rcu, call_rcu_bh) would wait -* on concurrent RCU and RCU-bh grace periods. Waiting on a give SRCU -* domain requires you to write a wrapper function for that SRCU domain's -* call_srcu() function, supplying the corresponding srcu_struct. -* -* If Tiny RCU, tell _wait_rcu_gp() not to bother waiting for RCU -* or RCU-bh, given that anywhere synchronize_rcu_mult() can be called -* is automatically a grace period. -*/ -#define synchronize_rcu_mult (...) \ - _wait_rcu_gp(IS_ENABLED(CONFIG_TINY_RCU), __VA_ARGS__) -/** * call_rcu_tasks() - Queue an RCU for invocation task-based grace period * @head: structure to be used for queueing the RCU updates.
include/linux/rcupdate_wait.h new file +50 -0
50 lines not on this page · raw message
include/linux/rcutiny.h +2 -9
@@ -47,15 +47,8 @@ static inline void cond_synchronize_sched(unsigned long oldstate)might_sleep (); } -static inline void rcu_barrier_bh (void ) -{ -wait_rcu_gp (call_rcu_bh); -} - -static inline void rcu_barrier_sched (void ) -{ -wait_rcu_gp (call_rcu_sched); -} +extern void rcu_barrier_bh (void ); +extern void rcu_barrier_sched (void );static inline void synchronize_rcu_expedited (void ) {
kernel/rcu/srcu.c +1 -1
@@ -30,7 +30,7 @@#include <linux/mutex.h> #include <linux/percpu.h> #include <linux/preempt.h> -#include <linux/rcupdate.h> +#include <linux/rcupdate_wait.h> #include <linux/sched.h> #include <linux/smp.h> #include <linux/delay.h>
kernel/rcu/tiny.c +13 -1
@@ -25,7 +25,7 @@#include <linux/completion.h> #include <linux/interrupt.h> #include <linux/notifier.h> -#include <linux/rcupdate.h> +#include <linux/rcupdate_wait.h> #include <linux/kernel.h> #include <linux/export.h> #include <linux/mutex.h>
@@ -49,6 +49,18 @@ static void __call_rcu(struct rcu_head *head,#include "tiny_plugin.h" +void rcu_barrier_bh (void ) +{ +wait_rcu_gp (call_rcu_bh); +} +EXPORT_SYMBOL (rcu_barrier_bh); + +void rcu_barrier_sched (void ) +{ +wait_rcu_gp (call_rcu_sched); +} +EXPORT_SYMBOL (rcu_barrier_sched); +#if defined(CONFIG_DEBUG_LOCK_ALLOC ) || defined(CONFIG_RCU_TRACE )/*
kernel/rcu/tree.c +1 -1
@@ -32,7 +32,7 @@#include <linux/init.h> #include <linux/spinlock.h> #include <linux/smp.h> -#include <linux/rcupdate.h> +#include <linux/rcupdate_wait.h> #include <linux/interrupt.h> #include <linux/sched.h> #include <linux/nmi.h>
kernel/rcu/update.c +1 -0
@@ -49,6 +49,7 @@#include <linux/moduleparam.h> #include <linux/kthread.h> #include <linux/tick.h> +#include <linux/rcupdate_wait.h> #define CREATE_TRACE_POINTS
kernel/sched/core.c +1 -0
@@ -10,6 +10,7 @@#include <linux/delayacct.h> #include <linux/init_task.h> #include <linux/context_tracking.h> +#include <linux/rcupdate_wait.h> #include <linux/blkdev.h> #include <linux/kprobes.h>
Signature · 1 line
2.7.4
PATCH 9/10 mm/vmacache, sched/headers: Introduce 'struct vmacache' and move it from <linux/sched.h> to <linux/mm_types>
In reply to Ingo MolnarThe <linux/sched.h> header includes various vmacache related defines, which are arguably misplaced.
Move them to mm_types.h and minimize the sched.h impact by putting all task vmacache state into a new 'struct vmacache' structure.
No change in functionality.
--- include/linux/mm_types.h | 12 ++++++++++++ include/linux/sched.h | 11 ++++------- include/linux/vmacache.h | 2 +- kernel/debug/debug_core.c | 4 ++-- mm/nommu.c | 2 +- mm/vmacache.c | 10 +++++----- 6 files changed, 25 insertions(+), 16 deletions(-)
include/linux/mm_types.h +12 -0
@@ -360,6 +360,18 @@ struct vm_area_struct {struct vm_userfaultfd_ctx vm_userfaultfd_ctx; }; +/* +* The per task VMA cache array: +*/ +#define VMACACHE_BITS 2 +#define VMACACHE_SIZE (1U << VMACACHE_BITS) +#define VMACACHE_MASK (VMACACHE_SIZE - 1) + +struct vmacache { +u32 seqnum; +struct vm_area_struct *vmas[VMACACHE_SIZE ]; +}; +struct core_thread {struct task_struct *task;struct core_thread *next;
include/linux/sched.h +4 -7
@@ -134,10 +134,6 @@ struct blk_plug;struct filename ;struct nameidata ; -#define VMACACHE_BITS 2 -#define VMACACHE_SIZE (1U << VMACACHE_BITS) -#define VMACACHE_MASK (VMACACHE_SIZE - 1) -/* * These are the constant used to fake the fixed-point load-average * counting. Some notes:
@@ -1554,9 +1550,10 @@ struct task_struct {#endif struct mm_struct *mm, *active_mm; -/* per-thread vma caching */ -u32 vmacache_seqnum; -struct vm_area_struct *vmacache[VMACACHE_SIZE ]; + +/* Per-thread vma caching: */ +struct vmacache vmacache; +#if defined(SPLIT_RSS_COUNTING )struct task_rss_stat rss_stat;#endif
include/linux/vmacache.h +1 -1
@@ -12,7 +12,7 @@static inline void vmacache_flush (struct task_struct *tsk) { -memset (tsk->vmacache,0 ,sizeof (tsk->vmacache)); +memset (tsk->vmacache.vmas,0 ,sizeof (tsk->vmacache.vmas)); }extern void vmacache_flush_all (struct mm_struct *mm);
kernel/debug/debug_core.c +2 -2
@@ -232,9 +232,9 @@ static void kgdb_flush_swbreak_addr(unsigned long addr)int i;for (i =0 ; i <VMACACHE_SIZE ; i++) { -if (!current->vmacache[i]) +if (!current->vmacache.vmas[i])continue ; -flush_cache_range (current->vmacache[i], +flush_cache_range (current->vmacache.vmas[i], addr, addr +BREAK_INSTR_SIZE ); } }
mm/nommu.c +1 -1
@@ -757,7 +757,7 @@ static void delete_vma_from_mm(struct vm_area_struct *vma) mm->map_count--;for (i =0 ; i <VMACACHE_SIZE ; i++) {/* if the vma is cached, invalidate the entire cache */ -if (curr->vmacache[i] == vma) { +if (curr->vmacache.vmas[i] == vma) {vmacache_invalidate (mm);break ; }
mm/vmacache.c +5 -5
@@ -60,7 +60,7 @@ static inline bool vmacache_valid_mm(struct mm_struct *mm)void vmacache_update (unsigned long addr,struct vm_area_struct *newvma) {if (vmacache_valid_mm (newvma->vm_mm)) - current->vmacache[VMACACHE_HASH (addr)] = newvma; + current->vmacache.vmas[VMACACHE_HASH (addr)] = newvma; }static bool vmacache_valid (struct mm_struct *mm)
@@ -71,12 +71,12 @@ static bool vmacache_valid(struct mm_struct *mm)return false; curr = current; -if (mm->vmacache_seqnum != curr->vmacache_seqnum) { +if (mm->vmacache_seqnum != curr->vmacache.seqnum) {/* * First attempt will always be invalid, initialize * the new cache for this task here. */ - curr->vmacache_seqnum = mm->vmacache_seqnum; + curr->vmacache.seqnum = mm->vmacache_seqnum;vmacache_flush (curr);return false; }
@@ -93,7 +93,7 @@ struct vm_area_struct *vmacache_find(struct mm_struct *mm, unsigned long addr)return NULL ;for (i =0 ; i <VMACACHE_SIZE ; i++) { -struct vm_area_struct *vma = current->vmacache[i]; +struct vm_area_struct *vma = current->vmacache.vmas[i];if (!vma)continue ;
@@ -121,7 +121,7 @@ struct vm_area_struct *vmacache_find_exact(struct mm_struct *mm,return NULL ;for (i =0 ; i <VMACACHE_SIZE ; i++) { -struct vm_area_struct *vma = current->vmacache[i]; +struct vm_area_struct *vma = current->vmacache.vmas[i];if (vma && vma->vm_start == start && vma->vm_end == end) {count_vm_vmacache_event (VMACACHE_FIND_HITS );
Signature · 1 line
2.7.4
Re: [PATCH 03/10] sched/headers: Make task_struct::wake_q an opaque pointer
In reply to Ingo MolnarQuoted 2 lines · Ingo Molnar
To be able to decouple
wake_qfunctionality from <linux/sched.h> make the basictask_struct::wake_qpointer an opaque void *.
Please don't use "void *" for opaque pointers.
You lose all typechecking, and this is just complete garbage:
+ struct wake_q_node *node = (void *)&task->wake_q;
What? You're casting a "void *" to "void *"? WTF?
The proper way to do opaque pointers is to declare them as pointers to a structure that hasn't been defined (ie use a "struct xyz;" forward declaration), and then that structure is only actually defined in the places that use the otherwise opaque pointer.
That way you
(a) never need to cast anything
(b) get proper (and strong) type checking for the pointers.
and the people who need to look into it automatically do the right thing, while anybody else who tries to dereference or otherwise use the struct pointer will get a compiler error.
Linus
Quoted 8 lines · Ingo Molnar
phase #1: "Pre-splitup cleanups": # 0721da143d99~1..582c7529a1bd, 10 patches
phase #2: "Prepare header dependencies": # 9017f600c534~1..d1eb6f0b580d, 48 patches phase #3: "Move definitions between headers": # 3d01f4b6e32d~1..fd88c1773dd6, 53 patches phase #4: "Remove header dependencies": # 2d246f83c5d3~1..2205323e0ca1, 35 patchesphase #5: "Post-splitup cleanups": # 55cf8bfb3af4~1..bcc6be9d66b2, 2 patches
So this seemed to be easier to go through, although I basically skipped all the one-liners so who knows what conflict markers might have been hiding there ;^)
I would personally have preferred phase 2/3 to be done in the reverse order (by having the "move definitions" phase just add an include in <sched.h> to still get them after the move), but it's not a huge issue and I doubt it's worth re-doing the series over it.
I noted a couple of nits I found, but in general I think you can just add my ack to this.
Linus
PATCH 1/1 sched/wake_q: Restore task_struct::wake_q type safety
In reply to Linus TorvaldsQuoted 11 lines · Linus Torvalds
Quoted 2 lines · Ingo Molnar
To be able to decouple
wake_qfunctionality from <linux/sched.h> make the basictask_struct::wake_qpointer an opaque void *.Please don't use "void *" for opaque pointers.
You lose all typechecking, and this is just complete garbage:
+ struct
wake_q_node*node = (void *)&task->wake_q;What? You're casting a "void *" to "void *"? WTF?
Well, I'm casting 'void **' to 'void *', so the cast is necessary in this specific scheme - but yeah, this is still pretty ugly.
Quoted 7 lines
The proper way to do opaque pointers is to declare them as pointers to a structure that hasn't been defined (ie use a "struct xyz;" forward declaration), and then that structure is only actually defined in the places that use the otherwise opaque pointer.
That way you
Quoted 7 lines
(a) never need to cast anything
(b) get proper (and strong) type checking for the pointers.
and the people who need to look into it automatically do the right thing, while anybody else who tries to dereference or otherwise use the struct pointer will get a compiler error.
So the problem here is that we don't really want an opaque pointer in a classic sense, but an opaque field.
struct wake_q is:
struct wake_q_node {
struct wake_q_node *next;
};i.e. it's really just a single linked list node, a single pointer. This line:
Quoted 1 line
+ struct
wake_q_node*node = (void *)&task->wake_q;
gets the pointer to the list node.
Couldn't think of a nicer way to do this - so let's not do this at all: we can just declare struct wake_q_node in sched.h and be done with it. We saved thousands of lines of code from it already, it's not worth doing it at the cost of strong type safety.
I.e. something like the patch attached below on top of WIP.sched/core?
Untested.
Thanks,
Ingo
==============================>
Quoted 1 line
From
fe7014cb9736f84f01f36003d72ed69d795ea82bMon Sep 17 00:00:00 2001
From: Ingo Molnar <[email protected]>
Date: Wed, 8 Feb 2017 22:32:31 +0100
Subject: [PATCH] sched/wake_q: Restore task_struct::wake_q type safety
Linus correctly observed that task_struct::wake_q is an opaque pointer in a rather ugly way, through which we lose not just readability but also type safety, for only marginal savings in sched.h.
Just restore the 'struct wake_q_node' type in sched.h and include sched.h in sched/wake_q.h.
--- include/linux/sched.h | 6 +++++- include/linux/sched/wake_q.h | 6 +----- ipc/msg.c | 1 - kernel/fork.c | 2 +- kernel/sched/core.c | 6 +++--- 5 files changed, 10 insertions(+), 11 deletions(-)
include/linux/sched.h +5 -1
@@ -476,6 +476,10 @@ enum perf_event_task_context { perf_nr_task_contexts, }; +struct wake_q_node { +struct wake_q_node *next; +}; +struct task_struct {#ifdef CONFIG_THREAD_INFO_IN_TASK /*
@@ -765,7 +769,7 @@ struct task_struct {/* Protection of the PI data structures: */ raw_spinlock_t pi_lock; -void *wake_q; +struct wake_q_node wake_q;#ifdef CONFIG_RT_MUTEXES /* PI waiters blocked on a rt_mutex held by this task: */
include/linux/sched/wake_q.h +1 -5
@@ -28,11 +28,7 @@* wakeup condition has in fact occurred. */ -struct task_struct ; - -struct wake_q_node { -struct wake_q_node *next; -}; +#include <linux/sched.h> struct wake_q_head {struct wake_q_node *first;
ipc/msg.c +0 -1
@@ -30,7 +30,6 @@#include <linux/proc_fs.h> #include <linux/list.h> #include <linux/security.h> -#include <linux/sched.h> #include <linux/sched/wake_q.h> #include <linux/syscalls.h> #include <linux/audit.h>
kernel/fork.c +1 -1
@@ -548,7 +548,7 @@ static struct task_struct *dup_task_struct(struct task_struct *orig, int node)#endif tsk->splice_pipe =NULL ; tsk->task_frag.page =NULL ; - tsk->wake_q =NULL ; + tsk->wake_q.next =NULL ;account_kernel_stack (tsk,1 );
kernel/sched/core.c +3 -3
@@ -431,7 +431,7 @@ static bool set_nr_if_polling(struct task_struct *p)void wake_q_add (struct wake_q_head *head,struct task_struct *task) { -struct wake_q_node *node = (void *)&task->wake_q; +struct wake_q_node *node = &task->wake_q;/* * Atomically grab the task, if ->wake_q is !nil already it means
@@ -460,11 +460,11 @@ void wake_up_q(struct wake_q_head *head)while (node !=WAKE_Q_TAIL ) {struct task_struct *task; - task = container_of((void *)node, struct task_struct, wake_q); + task = container_of(node, struct task_struct, wake_q);BUG_ON (!task);/* Task can safely be re-inserted now: */ node = node->next; - task->wake_q =NULL ; + task->wake_q.next =NULL ;/* * wake_up_process() implies a wmb() to pair with the queueing
Peter Zijlstra
maintainerRe: [PATCH 05/10] sched/core: Remove the tsk_cpus_allowed() wrapper
In reply to Ingo MolnarQuoted 5 lines · Ingo Molnar
So the original intention of
tsk_cpus_allowed()was to 'future-proof' the field - but it's pretty ineffectual at that, because half of the code uses ->cpus_allowed directly ...Also, the wrapper makes the code longer than the original expression!
I still object to taking this out without replacement. This leaves RT stranded.
If you want to implement it differently do that, but don't blindly wreck stuff for them.
Re: [PATCH 05/10] sched/core: Remove the tsk_cpus_allowed() wrapper
Quoted 9 lines · Peter Zijlstra
Quoted 5 lines · Ingo Molnar
So the original intention of
tsk_cpus_allowed()was to 'future-proof' the field - but it's pretty ineffectual at that, because half of the code uses ->cpus_allowed directly ...Also, the wrapper makes the code longer than the original expression!
I still object to taking this out without replacement.
Yeah, that would have been my next suggestion.
Quoted 1 line
This leaves RT stranded.
Well, no, it leaves -rt with slightly more patching work than it already has...
Because note how the wrappery is _already_ incomplete to a significant degree:
triton:~/tip> git grep -Ee '->cpus_allowed' | grep -vE 'tsk_|cpuset|core.c' | wc -l 27
triton:~/tip> git grep tsk_cpus_allowed | wc -l
43
I.e. around 40% of the places that use ->cpus_allowed in the upstream kernel are not properly wrapped. That fact already 'wrecks' -rt.
Thanks,
Ingo
Re: [PATCH 05/10] sched/core: Remove the tsk_cpus_allowed() wrapper
Quoted 21 lines · Ingo Molnar
Quoted 9 lines · Peter Zijlstra
Quoted 5 lines · Ingo Molnar
So the original intention of
tsk_cpus_allowed()was to 'future-proof' the field - but it's pretty ineffectual at that, because half of the code uses ->cpus_allowed directly ...Also, the wrapper makes the code longer than the original expression!
I still object to taking this out without replacement.
Yeah, that would have been my next suggestion.
Quoted 1 line
This leaves RT stranded.
Well, no, it leaves -rt with slightly more patching work than it already has...
Because note how the wrappery is _already_ incomplete to a significant degree:
Quoted 7 lines
triton:~/tip> git grep -Ee '->cpus_allowed' | grep -vE 'tsk_|cpuset|core.c' | wc -l 27
triton:~/tip> git greptsk_cpus_allowed| wc -l
43I.e. around 40% of the places that use ->cpus_allowed in the upstream kernel are not properly wrapped. That fact already 'wrecks' -rt.
Nope it does not. The places which use cpumask directly are not interfering with the decisions which are made by the scheduler whether migration can happen or not. All decision code pathes use the wrapper and we make sure on every update that this is the case.
I completely agree that your idea with the const *ptr is the better solution, but without that replacement RT is stranded and left alone with the mop up.
Thanks,
tglx
Re: [PATCH 05/10] sched/core: Remove the tsk_cpus_allowed() wrapper
Quoted 27 lines · Thomas Gleixner
Quoted 26 lines · Ingo Molnar
Quoted 9 lines · Peter Zijlstra
Quoted 5 lines · Ingo Molnar
So the original intention of
tsk_cpus_allowed()was to 'future-proof' the field - but it's pretty ineffectual at that, because half of the code uses ->cpus_allowed directly ...Also, the wrapper makes the code longer than the original expression!
I still object to taking this out without replacement.
Yeah, that would have been my next suggestion.
Quoted 1 line
This leaves RT stranded.
Well, no, it leaves -rt with slightly more patching work than it already has...
Because note how the wrappery is _already_ incomplete to a significant degree:
triton:~/tip> git grep -Ee '->cpus_allowed' | grep -vE 'tsk_|cpuset|core.c' | wc -l 27
triton:~/tip> git greptsk_cpus_allowed| wc -l
43
Quoted 7 lines
Quoted 2 lines
I.e. around 40% of the places that use ->cpus_allowed in the upstream kernel are not properly wrapped. That fact already 'wrecks' -rt.
Nope it does not. The places which use cpumask directly are not interfering with the decisions which are made by the scheduler whether migration can happen or not. All decision code pathes use the wrapper and we make sure on every update that this is the case.
Indeed!
Quoted 2 lines
I completely agree that your idea with the const *ptr is the better solution, but without that replacement RT is stranded and left alone with the mop up.
Ok, I'm convinced!
Thanks,
Ingo
Re: [PATCH 07/10] rcu: Separate the RCU synchronization types and APIs into <linux/rcupdate_wait.h>
In reply to Ingo MolnarQuoted 6 lines · Ingo Molnar
So
rcupdate.his a pretty complex header, in particular it includes <linux/completion.h> which includes <linux/wait.h> - creating a dependency that includes <linux/wait.h> in <linux/sched.h>, which prevents the isolation of <linux/sched.h> from the derived <linux/wait.h> header.
Quoted 8 lines
Solve part of the problem by decoupling
rcupdate.hfrom completions:
this can be done by separating out thercu_synchronizetypes and APIs, and updating their usage sites.Since this is a mostly RCU-internal types this will not just simplify <linux/sched.h>'s dependencies, but will make all the hundreds of .c files that include
rcupdate.hbut not completions orwait.hbuild faster.
Indeed, rcupdate.h is overdue for a more sweeping overhaul.
Quoted 2 lines
( For rcutiny this means that two dependent APIs have to be uninlined,
but that shouldn't be much of a problem as they are rare variants. )
Do people still care about Tiny kernel? If so, I can queue a patch that leaves rcu_barrier_bh() and rcu_barrier_sched() inline, but creates an rcu_barrier_generic() or some such, so that there is space taken up by only one EXPORT_SYMBOL() instead of two. (0day Test Robot yells at me every time I add one...)
Other than that, I don't see any problems with this. I will do some testing.
Thanx, Paul
Quoted 265 lines
Cc Linus Torvalds <[email protected]>Cc Mike Galbraith <[email protected]>Cc Paul E. McKenney <[email protected]>Cc Peter Zijlstra <[email protected]>Cc Thomas Gleixner <[email protected]>Signed-off-by Ingo Molnar <[email protected]>--- fs/autofs4/autofs_i.h | 1 + include/linux/dcache.h | 1 + include/linux/rcupdate.h | 40 ---------------------------------------- include/linux/rcupdate_wait.h | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++ include/linux/rcutiny.h | 11 ++--------- kernel/rcu/srcu.c | 2 +- kernel/rcu/tiny.c | 14 +++++++++++++- kernel/rcu/tree.c | 2 +- kernel/rcu/update.c | 1 + kernel/sched/core.c | 1 + 10 files changed, 71 insertions(+), 52 deletions(-)fs/autofs4/autofs_i.h +1
-0@@ -14,6 +14,7 @@#include <linux/mutex.h> #include <linux/spinlock.h> #include <linux/list.h> +#include <linux/completion.h> /* This is the range of ioctl() numbers we claim as ours */ #define AUTOFS_IOC_FIRST AUTOFS_IOC_READYinclude/linux/dcache.h +1
-0@@ -11,6 +11,7 @@#include <linux/rcupdate.h> #include <linux/lockref.h> #include <linux/stringhash.h> +#include <linux/wait.h> struct path ;struct vfsmount ;include/linux/rcupdate.h +0
-39@@ -40,7 +40,6 @@#include <linux/cpumask.h> #include <linux/seqlock.h> #include <linux/lockdep.h> -#include <linux/completion.h> #include <linux/debugobjects.h> #include <linux/bug.h> #include <linux/compiler.h> @@ -226,45 +225,6 @@ void call_rcu_sched(struct rcu_head *head,void synchronize_sched (void ); -/* -* Structure allowing asynchronous waiting on RCU. -*/ -struct rcu_synchronize { -struct rcu_head head; -struct completion completion; -}; -void wakeme_after_rcu(struct rcu_head *head);> - -void __wait_rcu_gp(bool checktiny, int n, call_rcu_func_t *crcu_array, - struct rcu_synchronize *rs_array); - -#define _wait_rcu_gp (checktiny, ...) \ -do { \ - call_rcu_func_t __crcu_array[] = { __VA_ARGS__ }; \ - struct rcu_synchronize __rs_array[ARRAY_SIZE(__crcu_array)]; \ - __wait_rcu_gp(checktiny, ARRAY_SIZE(__crcu_array), \ - __crcu_array, __rs_array); \ -} while (0) - -#define wait_rcu_gp (...) _wait_rcu_gp(false, __VA_ARGS__) - -/** -* synchronize_rcu_mult - Wait concurrently for multiple grace periods -* @...: List of call_rcu() functions for the flavors to wait on. -* -* This macro waits concurrently for multiple flavors of RCU grace periods. -* For example, synchronize_rcu_mult(call_rcu, call_rcu_bh) would wait -* on concurrent RCU and RCU-bh grace periods. Waiting on a give SRCU -* domain requires you to write a wrapper function for that SRCU domain's -* call_srcu() function, supplying the corresponding srcu_struct. -* -* If Tiny RCU, tell _wait_rcu_gp() not to bother waiting for RCU -* or RCU-bh, given that anywhere synchronize_rcu_mult() can be called -* is automatically a grace period. -*/ -#define synchronize_rcu_mult (...) \ - _wait_rcu_gp(IS_ENABLED(CONFIG_TINY_RCU), __VA_ARGS__) -/** * call_rcu_tasks() - Queue an RCU for invocation task-based grace period * @head: structure to be used for queueing the RCU updates. include/linux/rcupdate_wait.h new file +50
-0@@ -0,0 +1,50 @@ +#ifndef _LINUX_SCHED_RCUPDATE_WAIT_H +#define _LINUX_SCHED_RCUPDATE_WAIT_H + +/* +* RCU synchronization types and methods: +*/ + +#include <linux/rcupdate.h> +#include <linux/completion.h> + +/* +* Structure allowing asynchronous waiting on RCU. +*/ +struct rcu_synchronize { +struct rcu_head head; +struct completion completion; +}; +void wakeme_after_rcu (struct rcu_head *head); + +void __wait_rcu_gp (bool checktiny,int n,call_rcu_func_t *crcu_array, +struct rcu_synchronize *rs_array); + +#define _wait_rcu_gp (checktiny, ...) \ +do { \ + call_rcu_func_t __crcu_array[] = { __VA_ARGS__ }; \ + struct rcu_synchronize __rs_array[ARRAY_SIZE(__crcu_array)]; \ + __wait_rcu_gp(checktiny, ARRAY_SIZE(__crcu_array), \ + __crcu_array, __rs_array); \ +} while (0) + +#define wait_rcu_gp (...) _wait_rcu_gp(false, __VA_ARGS__) + +/** +* synchronize_rcu_mult - Wait concurrently for multiple grace periods +* @...: List of call_rcu() functions for the flavors to wait on. +* +* This macro waits concurrently for multiple flavors of RCU grace periods. +* For example, synchronize_rcu_mult(call_rcu, call_rcu_bh) would wait +* on concurrent RCU and RCU-bh grace periods. Waiting on a give SRCU +* domain requires you to write a wrapper function for that SRCU domain's +* call_srcu() function, supplying the corresponding srcu_struct. +* +* If Tiny RCU, tell _wait_rcu_gp() not to bother waiting for RCU +* or RCU-bh, given that anywhere synchronize_rcu_mult() can be called +* is automatically a grace period. +*/ +#define synchronize_rcu_mult (...) \ + _wait_rcu_gp(IS_ENABLED(CONFIG_TINY_RCU), __VA_ARGS__) + +#endif /* _LINUX_SCHED_RCUPDATE_WAIT_H */ include/linux/rcutiny.h +2
-9@@ -47,15 +47,8 @@ static inline void cond_synchronize_sched(unsigned long oldstate)might_sleep (); } -static inline void rcu_barrier_bh (void ) -{ -wait_rcu_gp (call_rcu_bh); -} - -static inline void rcu_barrier_sched (void ) -{ -wait_rcu_gp (call_rcu_sched); -} +extern void rcu_barrier_bh (void ); +extern void rcu_barrier_sched (void );static inline void synchronize_rcu_expedited (void ) {kernel/rcu/srcu.c +1
-1@@ -30,7 +30,7 @@#include <linux/mutex.h> #include <linux/percpu.h> #include <linux/preempt.h> -#include <linux/rcupdate.h> +#include <linux/rcupdate_wait.h> #include <linux/sched.h> #include <linux/smp.h> #include <linux/delay.h> kernel/rcu/tiny.c +13
-1@@ -25,7 +25,7 @@#include <linux/completion.h> #include <linux/interrupt.h> #include <linux/notifier.h> -#include <linux/rcupdate.h> +#include <linux/rcupdate_wait.h> #include <linux/kernel.h> #include <linux/export.h> #include <linux/mutex.h> @@ -49,6 +49,18 @@ static void __call_rcu(struct rcu_head *head,#include "tiny_plugin.h" +void rcu_barrier_bh (void ) +{ +wait_rcu_gp (call_rcu_bh); +} +EXPORT_SYMBOL (rcu_barrier_bh); + +void rcu_barrier_sched (void ) +{ +wait_rcu_gp (call_rcu_sched); +} +EXPORT_SYMBOL (rcu_barrier_sched); +#if defined(CONFIG_DEBUG_LOCK_ALLOC ) || defined(CONFIG_RCU_TRACE )/* kernel/rcu/tree.c +1
-1@@ -32,7 +32,7 @@#include <linux/init.h> #include <linux/spinlock.h> #include <linux/smp.h> -#include <linux/rcupdate.h> +#include <linux/rcupdate_wait.h> #include <linux/interrupt.h> #include <linux/sched.h> #include <linux/nmi.h> kernel/rcu/update.c +1
-0@@ -49,6 +49,7 @@#include <linux/moduleparam.h> #include <linux/kthread.h> #include <linux/tick.h> +#include <linux/rcupdate_wait.h> #define CREATE_TRACE_POINTS kernel/sched/core.c +1
-1@@ -10,6 +10,7 @@#include <linux/delayacct.h> #include <linux/init_task.h> #include <linux/context_tracking.h> +#include <linux/rcupdate_wait.h> #include <linux/blkdev.h> #include <linux/kprobes.h> --2.7.4