Cleanup task_struct::state
Subsystems (47)
Peter Zijlstra
maintainerv2 cover Cleanup task_struct::state
Hi!
The task_struct::state variable is a bit odd in a number of ways:
- it's declared 'volatile' (against current practises);
- it's 'unsigned long' which is a weird size;
- it's type is inconsistent when used for function arguments.
These patches clean that up by making it consistently 'unsigned int', and replace (almost) all accesses with READ_ONCE()/WRITE_ONCE(). In order to not miss any, the variable is renamed, ensuring a missed conversion results in a compile error.
The first few patches fix a number of pre-existing errors and introduce a few helpers to make the final conversion less painful.
This series applies on top of tip/master, and has been having the all-clear from the robots for the past week.
Since v1:
- fixed a whole bunch of compile fail on !x86
- collected tags
- (slightly) smaller Cc list
The plan is to stick them in tip/sched/core early next week, I'm hoping this won't cause too much pain for sfr.
Peter Zijlstra
maintainerPATCH v2 1/7 sched: Unbreak wakeups
Remove broken task->state references and let wake_up_process() DTRT.
The anti-pattern in these patches breaks the ordering of ->state vs COND as described in the comment near set_current_state() and can lead to missed wakeups:
(OoO load, observes RUNNING)<-.
for (;;) { |
t->state = UNINTERRUPTIBLE; |
smp_mb(); ,-----> | (observes !COND)
| /
if (COND) ---------' | COND = 1;
break; `- if (t->state != RUNNING)
wake_up_process(t); // not done
schedule(); // forever waiting
}
t->state = TASK_RUNNING;Signed-off-by Peter Zijlstra (Intel) <[email protected]>
Reviewed-by Davidlohr Bueso <[email protected]>
Acked-by Will Deacon <[email protected]>
--- drivers/net/ethernet/qualcomm/qca_spi.c | 6 ++---- drivers/usb/gadget/udc/max3420_udc.c | 15 +++++---------- drivers/usb/host/max3421-hcd.c | 3 +-- kernel/softirq.c | 2 +- 4 files changed, 9 insertions(+), 17 deletions(-)
drivers/net/ethernet/qualcomm/qca_spi.c +2 -4
@@ -653,8 +653,7 @@ qcaspi_intr_handler(int irq, void *data)struct qcaspi *qca = data; qca->intr_req++; -if (qca->spi_thread && - qca->spi_thread->state !=TASK_RUNNING ) +if (qca->spi_thread)wake_up_process (qca->spi_thread);return IRQ_HANDLED ;
@@ -777,8 +776,7 @@ qcaspi_netdev_xmit(struct sk_buff *skb,netif_trans_update (dev); -if (qca->spi_thread && - qca->spi_thread->state !=TASK_RUNNING ) +if (qca->spi_thread)wake_up_process (qca->spi_thread);return NETDEV_TX_OK ;
drivers/usb/gadget/udc/max3420_udc.c +5 -10
@@ -509,8 +509,7 @@ static irqreturn_t max3420_vbus_handler( ? USB_STATE_POWERED : USB_STATE_NOTATTACHED);spin_unlock_irqrestore (&udc->lock, flags); -if (udc->thread_task && - udc->thread_task->state !=TASK_RUNNING ) +if (udc->thread_task)wake_up_process (udc->thread_task);return IRQ_HANDLED ;
@@ -529,8 +528,7 @@ static irqreturn_t max3420_irq_handler(i }spin_unlock_irqrestore (&udc->lock, flags); -if (udc->thread_task && - udc->thread_task->state !=TASK_RUNNING ) +if (udc->thread_task)wake_up_process (udc->thread_task);return IRQ_HANDLED ;
@@ -1093,8 +1091,7 @@ static int max3420_wakeup(struct usb_gadspin_unlock_irqrestore (&udc->lock, flags); -if (udc->thread_task && - udc->thread_task->state !=TASK_RUNNING ) +if (udc->thread_task)wake_up_process (udc->thread_task);return ret; }
@@ -1117,8 +1114,7 @@ static int max3420_udc_start(struct usb_ udc->todo |=UDC_START ;spin_unlock_irqrestore (&udc->lock, flags); -if (udc->thread_task && - udc->thread_task->state !=TASK_RUNNING ) +if (udc->thread_task)wake_up_process (udc->thread_task);return 0 ;
@@ -1137,8 +1133,7 @@ static int max3420_udc_stop(struct usb_g udc->todo |=UDC_START ;spin_unlock_irqrestore (&udc->lock, flags); -if (udc->thread_task && - udc->thread_task->state !=TASK_RUNNING ) +if (udc->thread_task)wake_up_process (udc->thread_task);return 0 ;
drivers/usb/host/max3421-hcd.c +1 -2
@@ -1169,8 +1169,7 @@ max3421_irq_handler(int irq, void *dev_istruct spi_device *spi =to_spi_device (hcd->self.controller);struct max3421_hcd *max3421_hcd =hcd_to_max3421 (hcd); -if (max3421_hcd->spi_thread && - max3421_hcd->spi_thread->state !=TASK_RUNNING ) +if (max3421_hcd->spi_thread)wake_up_process (max3421_hcd->spi_thread);if (!test_and_set_bit (ENABLE_IRQ , &max3421_hcd->todo))disable_irq_nosync (spi->irq);
kernel/softirq.c +1 -1
@@ -76,7 +76,7 @@ static void wakeup_softirqd(void)/* Interrupts are disabled: no need to stop preemption */ struct task_struct *tsk =__this_cpu_read (ksoftirqd); -if (tsk && tsk->state !=TASK_RUNNING ) +if (tsk)wake_up_process (tsk); }
[tip: sched/core] sched: Unbreak wakeups
The following commit has been merged into the sched/core branch of tip:
The following commit has been merged into the sched/core branch of tip:
Commit-ID: 37aadc687ab441bbcb693ddae613acf9afcea1ab Gitweb: https://git.kernel.org/tip/37aadc687ab441bbcb693ddae613acf9afcea1ab Author: Peter Zijlstra <[email protected]> AuthorDate: Fri, 11 Jun 2021 10:28:11 +02:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Fri, 18 Jun 2021 11:43:06 +02:00
sched: Unbreak wakeups
Remove broken task->state references and let wake_up_process() DTRT.
The anti-pattern in these patches breaks the ordering of ->state vs COND as described in the comment near set_current_state() and can lead to missed wakeups:
(OoO load, observes RUNNING)<-.
for (;;) { |
t->state = UNINTERRUPTIBLE; |
smp_mb(); ,-----> | (observes !COND)
| /
if (COND) ---------' | COND = 1;
break; `- if (t->state != RUNNING)
wake_up_process(t); // not done
schedule(); // forever waiting
}
t->state = TASK_RUNNING;Signed-off-by Peter Zijlstra (Intel) <[email protected]>
Reviewed-by Davidlohr Bueso <[email protected]>
Acked-by Will Deacon <[email protected]>
--- drivers/net/ethernet/qualcomm/qca_spi.c | 6 ++---- drivers/usb/gadget/udc/max3420_udc.c | 15 +++++---------- drivers/usb/host/max3421-hcd.c | 3 +-- kernel/softirq.c | 2 +- 4 files changed, 9 insertions(+), 17 deletions(-)
drivers/net/ethernet/qualcomm/qca_spi.c +2 -4
@@ -653,8 +653,7 @@ qcaspi_intr_handler(int irq, void *data)struct qcaspi *qca = data; qca->intr_req++; -if (qca->spi_thread && - qca->spi_thread->state !=TASK_RUNNING ) +if (qca->spi_thread)wake_up_process (qca->spi_thread);return IRQ_HANDLED ;
@@ -777,8 +776,7 @@ qcaspi_netdev_xmit(struct sk_buff *skb, struct net_device *dev)netif_trans_update (dev); -if (qca->spi_thread && - qca->spi_thread->state !=TASK_RUNNING ) +if (qca->spi_thread)wake_up_process (qca->spi_thread);return NETDEV_TX_OK ;
drivers/usb/gadget/udc/max3420_udc.c +5 -10
@@ -509,8 +509,7 @@ static irqreturn_t max3420_vbus_handler(int irq, void *dev_id) ? USB_STATE_POWERED : USB_STATE_NOTATTACHED);spin_unlock_irqrestore (&udc->lock, flags); -if (udc->thread_task && - udc->thread_task->state !=TASK_RUNNING ) +if (udc->thread_task)wake_up_process (udc->thread_task);return IRQ_HANDLED ;
@@ -529,8 +528,7 @@ static irqreturn_t max3420_irq_handler(int irq, void *dev_id) }spin_unlock_irqrestore (&udc->lock, flags); -if (udc->thread_task && - udc->thread_task->state !=TASK_RUNNING ) +if (udc->thread_task)wake_up_process (udc->thread_task);return IRQ_HANDLED ;
@@ -1093,8 +1091,7 @@ static int max3420_wakeup(struct usb_gadget *gadget)spin_unlock_irqrestore (&udc->lock, flags); -if (udc->thread_task && - udc->thread_task->state !=TASK_RUNNING ) +if (udc->thread_task)wake_up_process (udc->thread_task);return ret; }
@@ -1117,8 +1114,7 @@ static int max3420_udc_start(struct usb_gadget *gadget, udc->todo |=UDC_START ;spin_unlock_irqrestore (&udc->lock, flags); -if (udc->thread_task && - udc->thread_task->state !=TASK_RUNNING ) +if (udc->thread_task)wake_up_process (udc->thread_task);return 0 ;
@@ -1137,8 +1133,7 @@ static int max3420_udc_stop(struct usb_gadget *gadget) udc->todo |=UDC_START ;spin_unlock_irqrestore (&udc->lock, flags); -if (udc->thread_task && - udc->thread_task->state !=TASK_RUNNING ) +if (udc->thread_task)wake_up_process (udc->thread_task);return 0 ;
drivers/usb/host/max3421-hcd.c +1 -2
@@ -1169,8 +1169,7 @@ max3421_irq_handler(int irq, void *dev_id)struct spi_device *spi =to_spi_device (hcd->self.controller);struct max3421_hcd *max3421_hcd =hcd_to_max3421 (hcd); -if (max3421_hcd->spi_thread && - max3421_hcd->spi_thread->state !=TASK_RUNNING ) +if (max3421_hcd->spi_thread)wake_up_process (max3421_hcd->spi_thread);if (!test_and_set_bit (ENABLE_IRQ , &max3421_hcd->todo))disable_irq_nosync (spi->irq);
kernel/softirq.c +1 -1
@@ -76,7 +76,7 @@ static void wakeup_softirqd(void)/* Interrupts are disabled: no need to stop preemption */ struct task_struct *tsk =__this_cpu_read (ksoftirqd); -if (tsk && tsk->state !=TASK_RUNNING ) +if (tsk)wake_up_process (tsk); }
Peter Zijlstra
maintainerPATCH v2 2/7 sched: Introduce task_is_running()
In reply to Peter ZijlstraReplace a bunch of 'p->state == TASK_RUNNING' with a new helper:task_is_running(p).
Signed-off-by Peter Zijlstra (Intel) <[email protected]>
Acked-by Davidlohr Bueso <[email protected]>
--- arch/alpha/kernel/process.c | 2 +- arch/arc/kernel/stacktrace.c | 2 +- arch/arm/kernel/process.c | 2 +- arch/arm64/kernel/process.c | 2 +- arch/csky/kernel/stacktrace.c | 2 +- arch/h8300/kernel/process.c | 2 +- arch/hexagon/kernel/process.c | 2 +- arch/ia64/kernel/process.c | 4 ++-- arch/m68k/kernel/process.c | 2 +- arch/mips/kernel/process.c | 2 +- arch/nds32/kernel/process.c | 2 +- arch/nios2/kernel/process.c | 2 +- arch/parisc/kernel/process.c | 4 ++-- arch/powerpc/kernel/process.c | 4 ++-- arch/riscv/kernel/stacktrace.c | 2 +- arch/s390/kernel/process.c | 2 +- arch/s390/mm/fault.c | 2 +- arch/sh/kernel/process_32.c | 2 +- arch/sparc/kernel/process_32.c | 3 +-- arch/sparc/kernel/process_64.c | 3 +-- arch/um/kernel/process.c | 2 +- arch/x86/kernel/process.c | 4 ++-- arch/xtensa/kernel/process.c | 2 +- block/blk-mq.c | 2 +- include/linux/sched.h | 2 ++ kernel/kcsan/report.c | 2 +- kernel/locking/lockdep.c | 2 +- kernel/rcu/tree_plugin.h | 2 +- kernel/sched/core.c | 6 +++--- kernel/sched/stats.h | 2 +- kernel/signal.c | 2 +- kernel/softirq.c | 3 +-- mm/compaction.c | 2 +- 33 files changed, 40 insertions(+), 41 deletions(-)
arch/alpha/kernel/process.c +1 -1
@@ -380,7 +380,7 @@ get_wchan(struct task_struct *p) {unsigned long schedule_frame;unsigned long pc; -if (!p || p == current || p->state ==TASK_RUNNING ) +if (!p || p == current ||task_is_running (p))return 0 ;/* * This one depends on the frame size of schedule(). Do a
arch/arc/kernel/stacktrace.c +1 -1
@@ -83,7 +83,7 @@ seed_unwind_frame_info(struct task_struc* is safe-kept and BLINK at a well known location in there */ -if (tsk->state ==TASK_RUNNING ) +if (task_is_running (tsk))return -1 ; frame_info->task = tsk;
arch/arm/kernel/process.c +1 -1
@@ -288,7 +288,7 @@ unsigned long get_wchan(struct task_strustruct stackframe frame;unsigned long stack_page;int count =0 ; -if (!p || p == current || p->state ==TASK_RUNNING ) +if (!p || p == current ||task_is_running (p))return 0 ; frame.fp =thread_saved_fp (p);
arch/arm64/kernel/process.c +1 -1
@@ -598,7 +598,7 @@ unsigned long get_wchan(struct task_strustruct stackframe frame;unsigned long stack_page, ret =0 ;int count =0 ; -if (!p || p == current || p->state ==TASK_RUNNING ) +if (!p || p == current ||task_is_running (p))return 0 ; stack_page = (unsigned long )try_get_task_stack (p);
arch/csky/kernel/stacktrace.c +1 -1
@@ -115,7 +115,7 @@ unsigned long get_wchan(struct task_stru {unsigned long pc =0 ; -if (likely (task && task != current && task->state !=TASK_RUNNING )) +if (likely (task && task != current && !task_is_running (task)))walk_stackframe (task,NULL , save_wchan, &pc);return pc; }
arch/h8300/kernel/process.c +1 -1
@@ -134,7 +134,7 @@ unsigned long get_wchan(struct task_struunsigned long stack_page;int count =0 ; -if (!p || p == current || p->state ==TASK_RUNNING ) +if (!p || p == current ||task_is_running (p))return 0 ; stack_page = (unsigned long )p;
arch/hexagon/kernel/process.c +1 -1
@@ -135,7 +135,7 @@ unsigned long get_wchan(struct task_struunsigned long fp, pc;unsigned long stack_page;int count =0 ; -if (!p || p == current || p->state ==TASK_RUNNING ) +if (!p || p == current ||task_is_running (p))return 0 ; stack_page = (unsigned long )task_stack_page (p);
arch/ia64/kernel/process.c +2 -2
@@ -529,7 +529,7 @@ get_wchan (struct task_struct *p)unsigned long ip;int count =0 ; -if (!p || p == current || p->state ==TASK_RUNNING ) +if (!p || p == current ||task_is_running (p))return 0 ;/*
@@ -542,7 +542,7 @@ get_wchan (struct task_struct *p)*/ unw_init_from_blocked_task (&info, p); do { - if (p->state == TASK_RUNNING) + if (task_is_running(p)) return 0; if (unw_unwind(&info) < 0) return 0;
arch/m68k/kernel/process.c +1 -1
@@ -268,7 +268,7 @@ unsigned long get_wchan(struct task_struunsigned long fp, pc;unsigned long stack_page;int count =0 ; -if (!p || p == current || p->state ==TASK_RUNNING ) +if (!p || p == current ||task_is_running (p))return 0 ; stack_page = (unsigned long )task_stack_page (p);
arch/mips/kernel/process.c +1 -1
@@ -662,7 +662,7 @@ unsigned long get_wchan(struct task_struunsigned long ra =0 ;#endif -if (!task || task == current || task->state ==TASK_RUNNING ) +if (!task || task == current ||task_is_running (task)) goto out;if (!task_stack_page (task)) goto out;
arch/nds32/kernel/process.c +1 -1
@@ -239,7 +239,7 @@ unsigned long get_wchan(struct task_struunsigned long stack_start, stack_end;int count =0 ; -if (!p || p == current || p->state ==TASK_RUNNING ) +if (!p || p == current ||task_is_running (p))return 0 ;if (IS_ENABLED (CONFIG_FRAME_POINTER )) {
arch/nios2/kernel/process.c +1 -1
@@ -223,7 +223,7 @@ unsigned long get_wchan(struct task_struunsigned long stack_page;int count =0 ; -if (!p || p == current || p->state ==TASK_RUNNING ) +if (!p || p == current ||task_is_running (p))return 0 ; stack_page = (unsigned long )p;
arch/parisc/kernel/process.c +2 -2
@@ -249,7 +249,7 @@ get_wchan(struct task_struct *p)unsigned long ip;int count =0 ; -if (!p || p == current || p->state ==TASK_RUNNING ) +if (!p || p == current ||task_is_running (p))return 0 ;/*
@@ -260,7 +260,7 @@ get_wchan(struct task_struct *p) do { if (unwind_once(&info) < 0) return 0; - if (p->state == TASK_RUNNING) + if (task_is_running(p)) return 0; ip = info.ip; if (!in_sched_functions(ip))
arch/powerpc/kernel/process.c +2 -2
@@ -2084,7 +2084,7 @@ static unsigned long __get_wchan(structunsigned long ip, sp;int count =0 ; -if (!p || p == current || p->state ==TASK_RUNNING ) +if (!p || p == current ||task_is_running (p))return 0 ; sp = p->thread.ksp;
@@ -2094,7 +2094,7 @@ static unsigned long __get_wchan(struct do { sp = *(unsigned long *)sp; if (!validate_sp(sp, p, STACK_FRAME_OVERHEAD) || - p->state == TASK_RUNNING) + task_is_running(p)) return 0; if (count > 0) { ip = ((unsigned long *)sp)[STACK_FRAME_LR_SAVE];
arch/riscv/kernel/stacktrace.c +1 -1
@@ -132,7 +132,7 @@ unsigned long get_wchan(struct task_stru {unsigned long pc =0 ; -if (likely (task && task != current && task->state !=TASK_RUNNING )) +if (likely (task && task != current && !task_is_running (task)))walk_stackframe (task,NULL , save_wchan, &pc);return pc; }
arch/s390/kernel/process.c +1 -1
@@ -180,7 +180,7 @@ unsigned long get_wchan(struct task_strustruct unwind_state state;unsigned long ip =0 ; -if (!p || p == current || p->state ==TASK_RUNNING || !task_stack_page (p)) +if (!p || p == current ||task_is_running (p) || !task_stack_page (p))return 0 ;if (!try_get_task_stack (p))
arch/s390/mm/fault.c +1 -1
@@ -702,7 +702,7 @@ static void pfault_interrupt(struct ext_* interrupt since it must be a leftover of a PFAULT * CANCEL operation which didn't remove all pending * completion interrupts. */ -if (tsk->state ==TASK_RUNNING ) +if (task_is_running (tsk)) tsk->thread.pfault_wait =-1 ; } }else {
arch/sh/kernel/process_32.c +1 -1
@@ -186,7 +186,7 @@ unsigned long get_wchan(struct task_stru {unsigned long pc; -if (!p || p == current || p->state ==TASK_RUNNING ) +if (!p || p == current ||task_is_running (p))return 0 ;/*
arch/sparc/kernel/process_32.c +1 -2
@@ -376,8 +376,7 @@ unsigned long get_wchan(struct task_strustruct reg_window32 *rw;int count =0 ; -if (!task || task == current || - task->state ==TASK_RUNNING ) +if (!task || task == current ||task_is_running (task)) goto out; fp =task_thread_info (task)->ksp + bias;
arch/sparc/kernel/process_64.c +1 -2
@@ -674,8 +674,7 @@ unsigned long get_wchan(struct task_struunsigned long ret =0 ;int count =0 ; -if (!task || task == current || - task->state ==TASK_RUNNING ) +if (!task || task == current ||task_is_running (task)) goto out; tp =task_thread_info (task);
arch/um/kernel/process.c +1 -1
@@ -369,7 +369,7 @@ unsigned long get_wchan(struct task_struunsigned long stack_page, sp, ip;bool seen_sched =0 ; -if ((p ==NULL ) || (p == current) || (p->state ==TASK_RUNNING )) +if ((p ==NULL ) || (p == current) ||task_is_running (p))return 0 ; stack_page = (unsigned long )task_stack_page (p);
arch/x86/kernel/process.c +2 -2
@@ -931,7 +931,7 @@ unsigned long get_wchan(struct task_struunsigned long start, bottom, top, sp, fp, ip, ret =0 ;int count =0 ; -if (p == current || p->state ==TASK_RUNNING ) +if (p == current ||task_is_running (p))return 0 ;if (!try_get_task_stack (p))
@@ -975,7 +975,7 @@ unsigned long get_wchan(struct task_stru goto out; } fp =READ_ONCE_NOCHECK (*(unsigned long *)fp); - }while (count++ <16 && p->state !=TASK_RUNNING ); + }while (count++ <16 && !task_is_running (p)); out:put_task_stack (p);
arch/xtensa/kernel/process.c +1 -1
@@ -304,7 +304,7 @@ unsigned long get_wchan(struct task_struunsigned long stack_page = (unsigned long )task_stack_page (p);int count =0 ; -if (!p || p == current || p->state ==TASK_RUNNING ) +if (!p || p == current ||task_is_running (p))return 0 ; sp = p->thread.sp;
block/blk-mq.c +1 -1
@@ -3926,7 +3926,7 @@ int blk_poll(struct request_queue *q, blif (signal_pending_state (state, current))__set_current_state (TASK_RUNNING ); -if (current->state ==TASK_RUNNING ) +if (task_is_running (current))return 1 ;if (ret <0 || !spin)break ;
include/linux/sched.h +2 -0
@@ -113,6 +113,8 @@ struct task_group; __TASK_TRACED | EXIT_DEAD | EXIT_ZOMBIE | \ TASK_PARKED) +#define task_is_running (task) (READ_ONCE((task)->state) == TASK_RUNNING) +#define task_is_traced (task) ((task->state & __TASK_TRACED) != 0)#define task_is_stopped (task) ((task->state & __TASK_STOPPED) != 0)
kernel/kcsan/report.c +1 -1
@@ -460,7 +460,7 @@ static void set_other_info_task_blocking* We may be instrumenting a code-path where current->state is already * something other than TASK_RUNNING. */ -const bool is_running = current->state ==TASK_RUNNING ; +const bool is_running =task_is_running (current);/* * To avoid deadlock in case we are in an interrupt here and this is a * race with a task on the same CPU (KCSAN_INTERRUPT_WATCHER), provide a
kernel/locking/lockdep.c +1 -1
@@ -760,7 +760,7 @@ static void lockdep_print_held_locks(str* It's not reliable to print a task's held locks if it's not sleeping * and it's not the current task. */ -if (p->state ==TASK_RUNNING && p != current) +if (p != current &&task_is_running (p))return ;for (i =0 ; i < depth; i++) {printk (" #%d: " , i);
kernel/rcu/tree_plugin.h +1 -1
@@ -2768,7 +2768,7 @@ EXPORT_SYMBOL_GPL(rcu_bind_current_to_no#ifdef CONFIG_SMP static char *show_rcu_should_be_on_cpu (struct task_struct *tsp) { -return tsp && tsp->state ==TASK_RUNNING && !tsp->on_cpu ?"!" :"" ; +return tsp &&task_is_running (tsp) && !tsp->on_cpu ?"!" :"" ; }#else // #ifdef CONFIG_SMP static char *show_rcu_should_be_on_cpu (struct task_struct *tsp)
kernel/sched/core.c +3 -3
@@ -5989,7 +5989,7 @@ static inline void sched_submit_work(str {unsigned int task_flags; -if (!tsk->state) +if (task_is_running (tsk))return ; task_flags = tsk->flags;
@@ -7964,7 +7964,7 @@ int __sched yield_to(struct task_structif (curr->sched_class != p->sched_class) goto out_unlock; -if (task_running (p_rq, p) || p->state) +if (task_running (p_rq, p) || !task_is_running (p)) goto out_unlock; yielded = curr->sched_class->yield_to_task (rq, p);
@@ -8167,7 +8167,7 @@ void sched_show_task(struct task_structpr_info ("task:%-15.15s state:%c" , p->comm,task_state_to_char (p)); -if (p->state ==TASK_RUNNING ) +if (task_is_running (p))pr_cont (" running task " );#ifdef CONFIG_DEBUG_STACK_USAGE free =stack_not_used (p);
kernel/sched/stats.h +1 -1
@@ -217,7 +217,7 @@ static inline void sched_info_depart(strrq_sched_info_depart (rq, delta); -if (t->state ==TASK_RUNNING ) +if (task_is_running (t))sched_info_enqueue (rq, t); }
kernel/signal.c +1 -1
@@ -4719,7 +4719,7 @@ void kdb_send_sig(struct task_struct *t, } new_t = kdb_prev_t != t; kdb_prev_t = t; -if (t->state !=TASK_RUNNING && new_t) { +if (!task_is_running (t) && new_t) {spin_unlock (&t->sighand->siglock);kdb_printf ("Process is not RUNNING, sending a signal from " "kdb risks deadlock\n"
kernel/softirq.c +1 -2
@@ -92,8 +92,7 @@ static bool ksoftirqd_running(unsigned lif (pending &SOFTIRQ_NOW_MASK )return false; -return tsk && (tsk->state ==TASK_RUNNING ) && - !__kthread_should_park (tsk); +return tsk &&task_is_running (tsk) && !__kthread_should_park (tsk); }#ifdef CONFIG_TRACE_IRQFLAGS
mm/compaction.c +1 -1
@@ -1955,7 +1955,7 @@ static inline bool is_via_compact_memorystatic bool kswapd_is_running (pg_data_t *pgdat) { -return pgdat->kswapd && (pgdat->kswapd->state ==TASK_RUNNING ); +return pgdat->kswapd &&task_is_running (pgdat->kswapd); }/*
Re: [PATCH v2 2/7] sched: Introduce task_is_running()
Hoi Peter,
Thanks for your patch!
Quoted 2 lines · Peter Zijlstra
Replace a bunch of '
p->state==TASK_RUNNING' with a new helper:task_is_running(p).
You're also sticking a READ_ONCE() in the helper, which wasn't done by any of the old implementations? Care to mention why?
Quoted 4 lines
Signed-off-by Peter Zijlstra (Intel) <[email protected]>Acked-by Davidlohr Bueso <[email protected]>arch/m68k/kernel/process.c | 2 +-
Regardless:
Gr{oetje,eeting}s,
Geert
Signature · 5 lines
Geert Uytterhoeven -- There's lots of Linux beyond ia32 -- [email protected] In personal conversations with technical people, I call myself a hacker. But when I'm talking to journalists I just say "programmer" or something like that. -- Linus Torvalds
Peter Zijlstra
maintainerRe: [PATCH v2 2/7] sched: Introduce task_is_running()
Quoted 10 lines · Geert Uytterhoeven
Hoi Peter,
Thanks for your patch!
Quoted 2 lines · Peter Zijlstra
Replace a bunch of '
p->state==TASK_RUNNING' with a new helper:task_is_running(p).You're also sticking a
READ_ONCE()in the helper, which wasn't done by any of the old implementations? Care to mention why?
Patch 7/7 should clarify. Arguably I should leave the READ_ONCE() off here and only add it there.
Re: [PATCH v2 2/7] sched: Introduce task_is_running()
In reply to Peter ZijlstraQuoted 21 lines · Peter Zijlstra
Replace a bunch of '
p->state==TASK_RUNNING' with a new helper:task_is_running(p).Signed-off-by Peter Zijlstra (Intel) <[email protected]>Acked-by Davidlohr Bueso <[email protected]>--- arch/alpha/kernel/process.c | 2 +- arch/arc/kernel/stacktrace.c | 2 +- arch/arm/kernel/process.c | 2 +- arch/arm64/kernel/process.c | 2 +- arch/csky/kernel/stacktrace.c | 2 +- arch/h8300/kernel/process.c | 2 +- arch/hexagon/kernel/process.c | 2 +- arch/ia64/kernel/process.c | 4 ++-- arch/m68k/kernel/process.c | 2 +- arch/mips/kernel/process.c | 2 +- arch/nds32/kernel/process.c | 2 +- arch/nios2/kernel/process.c | 2 +- arch/parisc/kernel/process.c | 4 ++-- arch/powerpc/kernel/process.c | 4 ++-- arch/riscv/kernel/stacktrace.c | 2 +-
Quoted 8 lines
arch/s390/kernel/process.c | 2 +- arch/s390/mm/fault.c | 2 +- arch/sh/kernel/process_32.c | 2 +- arch/sparc/kernel/process_32.c | 3 +-- arch/sparc/kernel/process_64.c | 3 +-- arch/um/kernel/process.c | 2 +- arch/x86/kernel/process.c | 4 ++-- arch/xtensa/kernel/process.c | 2 +-
Cheers for adding the missing arch bits:
Acked-by Will Deacon <[email protected]>
Will
[tip: sched/core] sched: Introduce task_is_running()
In reply to Peter ZijlstraThe following commit has been merged into the sched/core branch of tip:
The following commit has been merged into the sched/core branch of tip:
Commit-ID: b03fbd4ff24c5f075e58eb19261d5f8b3e40d7c6 Gitweb: https://git.kernel.org/tip/b03fbd4ff24c5f075e58eb19261d5f8b3e40d7c6 Author: Peter Zijlstra <[email protected]> AuthorDate: Fri, 11 Jun 2021 10:28:12 +02:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Fri, 18 Jun 2021 11:43:07 +02:00
sched: Introduce task_is_running()
Replace a bunch of 'p->state == TASK_RUNNING' with a new helper:task_is_running(p).
Signed-off-by Peter Zijlstra (Intel) <[email protected]>
Acked-by Davidlohr Bueso <[email protected]>
Acked-by Will Deacon <[email protected]>
--- arch/alpha/kernel/process.c | 2 +- arch/arc/kernel/stacktrace.c | 2 +- arch/arm/kernel/process.c | 2 +- arch/arm64/kernel/process.c | 2 +- arch/csky/kernel/stacktrace.c | 2 +- arch/h8300/kernel/process.c | 2 +- arch/hexagon/kernel/process.c | 2 +- arch/ia64/kernel/process.c | 4 ++-- arch/m68k/kernel/process.c | 2 +- arch/mips/kernel/process.c | 2 +- arch/nds32/kernel/process.c | 2 +- arch/nios2/kernel/process.c | 2 +- arch/parisc/kernel/process.c | 4 ++-- arch/powerpc/kernel/process.c | 4 ++-- arch/riscv/kernel/stacktrace.c | 2 +- arch/s390/kernel/process.c | 2 +- arch/s390/mm/fault.c | 2 +- arch/sh/kernel/process_32.c | 2 +- arch/sparc/kernel/process_32.c | 3 +-- arch/sparc/kernel/process_64.c | 3 +-- arch/um/kernel/process.c | 2 +- arch/x86/kernel/process.c | 4 ++-- arch/xtensa/kernel/process.c | 2 +- block/blk-mq.c | 2 +- include/linux/sched.h | 2 ++ kernel/kcsan/report.c | 2 +- kernel/locking/lockdep.c | 2 +- kernel/rcu/tree_plugin.h | 2 +- kernel/sched/core.c | 6 +++--- kernel/sched/stats.h | 2 +- kernel/signal.c | 2 +- kernel/softirq.c | 3 +-- mm/compaction.c | 2 +- 33 files changed, 40 insertions(+), 41 deletions(-)
arch/alpha/kernel/process.c +1 -1
@@ -380,7 +380,7 @@ get_wchan(struct task_struct *p) {unsigned long schedule_frame;unsigned long pc; -if (!p || p == current || p->state ==TASK_RUNNING ) +if (!p || p == current ||task_is_running (p))return 0 ;/* * This one depends on the frame size of schedule(). Do a
arch/arc/kernel/stacktrace.c +1 -1
@@ -83,7 +83,7 @@ seed_unwind_frame_info(struct task_struct *tsk, struct pt_regs *regs,* is safe-kept and BLINK at a well known location in there */ -if (tsk->state ==TASK_RUNNING ) +if (task_is_running (tsk))return -1 ; frame_info->task = tsk;
arch/arm/kernel/process.c +1 -1
@@ -288,7 +288,7 @@ unsigned long get_wchan(struct task_struct *p)struct stackframe frame;unsigned long stack_page;int count =0 ; -if (!p || p == current || p->state ==TASK_RUNNING ) +if (!p || p == current ||task_is_running (p))return 0 ; frame.fp =thread_saved_fp (p);
arch/arm64/kernel/process.c +1 -1
@@ -598,7 +598,7 @@ unsigned long get_wchan(struct task_struct *p)struct stackframe frame;unsigned long stack_page, ret =0 ;int count =0 ; -if (!p || p == current || p->state ==TASK_RUNNING ) +if (!p || p == current ||task_is_running (p))return 0 ; stack_page = (unsigned long )try_get_task_stack (p);
arch/csky/kernel/stacktrace.c +1 -1
@@ -115,7 +115,7 @@ unsigned long get_wchan(struct task_struct *task) {unsigned long pc =0 ; -if (likely (task && task != current && task->state !=TASK_RUNNING )) +if (likely (task && task != current && !task_is_running (task)))walk_stackframe (task,NULL , save_wchan, &pc);return pc; }
arch/h8300/kernel/process.c +1 -1
@@ -134,7 +134,7 @@ unsigned long get_wchan(struct task_struct *p)unsigned long stack_page;int count =0 ; -if (!p || p == current || p->state ==TASK_RUNNING ) +if (!p || p == current ||task_is_running (p))return 0 ; stack_page = (unsigned long )p;
arch/hexagon/kernel/process.c +1 -1
@@ -135,7 +135,7 @@ unsigned long get_wchan(struct task_struct *p)unsigned long fp, pc;unsigned long stack_page;int count =0 ; -if (!p || p == current || p->state ==TASK_RUNNING ) +if (!p || p == current ||task_is_running (p))return 0 ; stack_page = (unsigned long )task_stack_page (p);
arch/ia64/kernel/process.c +2 -2
@@ -529,7 +529,7 @@ get_wchan (struct task_struct *p)unsigned long ip;int count =0 ; -if (!p || p == current || p->state ==TASK_RUNNING ) +if (!p || p == current ||task_is_running (p))return 0 ;/*
@@ -542,7 +542,7 @@ get_wchan (struct task_struct *p)*/ unw_init_from_blocked_task (&info, p); do { - if (p->state == TASK_RUNNING) + if (task_is_running(p)) return 0; if (unw_unwind(&info) < 0) return 0;
arch/m68k/kernel/process.c +1 -1
@@ -268,7 +268,7 @@ unsigned long get_wchan(struct task_struct *p)unsigned long fp, pc;unsigned long stack_page;int count =0 ; -if (!p || p == current || p->state ==TASK_RUNNING ) +if (!p || p == current ||task_is_running (p))return 0 ; stack_page = (unsigned long )task_stack_page (p);
arch/mips/kernel/process.c +1 -1
@@ -662,7 +662,7 @@ unsigned long get_wchan(struct task_struct *task)unsigned long ra =0 ;#endif -if (!task || task == current || task->state ==TASK_RUNNING ) +if (!task || task == current ||task_is_running (task)) goto out;if (!task_stack_page (task)) goto out;
arch/nds32/kernel/process.c +1 -1
@@ -239,7 +239,7 @@ unsigned long get_wchan(struct task_struct *p)unsigned long stack_start, stack_end;int count =0 ; -if (!p || p == current || p->state ==TASK_RUNNING ) +if (!p || p == current ||task_is_running (p))return 0 ;if (IS_ENABLED (CONFIG_FRAME_POINTER )) {
arch/nios2/kernel/process.c +1 -1
@@ -223,7 +223,7 @@ unsigned long get_wchan(struct task_struct *p)unsigned long stack_page;int count =0 ; -if (!p || p == current || p->state ==TASK_RUNNING ) +if (!p || p == current ||task_is_running (p))return 0 ; stack_page = (unsigned long )p;
arch/parisc/kernel/process.c +2 -2
@@ -249,7 +249,7 @@ get_wchan(struct task_struct *p)unsigned long ip;int count =0 ; -if (!p || p == current || p->state ==TASK_RUNNING ) +if (!p || p == current ||task_is_running (p))return 0 ;/*
@@ -260,7 +260,7 @@ get_wchan(struct task_struct *p) do { if (unwind_once(&info) < 0) return 0; - if (p->state == TASK_RUNNING) + if (task_is_running(p)) return 0; ip = info.ip; if (!in_sched_functions(ip))
arch/powerpc/kernel/process.c +2 -2
@@ -2084,7 +2084,7 @@ static unsigned long __get_wchan(struct task_struct *p)unsigned long ip, sp;int count =0 ; -if (!p || p == current || p->state ==TASK_RUNNING ) +if (!p || p == current ||task_is_running (p))return 0 ; sp = p->thread.ksp;
@@ -2094,7 +2094,7 @@ static unsigned long __get_wchan(struct task_struct *p) do { sp = *(unsigned long *)sp; if (!validate_sp(sp, p, STACK_FRAME_OVERHEAD) || - p->state == TASK_RUNNING) + task_is_running(p)) return 0; if (count > 0) { ip = ((unsigned long *)sp)[STACK_FRAME_LR_SAVE];
arch/riscv/kernel/stacktrace.c +1 -1
@@ -132,7 +132,7 @@ unsigned long get_wchan(struct task_struct *task) {unsigned long pc =0 ; -if (likely (task && task != current && task->state !=TASK_RUNNING )) +if (likely (task && task != current && !task_is_running (task)))walk_stackframe (task,NULL , save_wchan, &pc);return pc; }
arch/s390/kernel/process.c +1 -1
@@ -180,7 +180,7 @@ unsigned long get_wchan(struct task_struct *p)struct unwind_state state;unsigned long ip =0 ; -if (!p || p == current || p->state ==TASK_RUNNING || !task_stack_page (p)) +if (!p || p == current ||task_is_running (p) || !task_stack_page (p))return 0 ;if (!try_get_task_stack (p))
arch/s390/mm/fault.c +1 -1
@@ -702,7 +702,7 @@ static void pfault_interrupt(struct ext_code ext_code,* interrupt since it must be a leftover of a PFAULT * CANCEL operation which didn't remove all pending * completion interrupts. */ -if (tsk->state ==TASK_RUNNING ) +if (task_is_running (tsk)) tsk->thread.pfault_wait =-1 ; } }else {
arch/sh/kernel/process_32.c +1 -1
@@ -186,7 +186,7 @@ unsigned long get_wchan(struct task_struct *p) {unsigned long pc; -if (!p || p == current || p->state ==TASK_RUNNING ) +if (!p || p == current ||task_is_running (p))return 0 ;/*
arch/sparc/kernel/process_32.c +1 -2
@@ -376,8 +376,7 @@ unsigned long get_wchan(struct task_struct *task)struct reg_window32 *rw;int count =0 ; -if (!task || task == current || - task->state ==TASK_RUNNING ) +if (!task || task == current ||task_is_running (task)) goto out; fp =task_thread_info (task)->ksp + bias;
arch/sparc/kernel/process_64.c +1 -2
@@ -674,8 +674,7 @@ unsigned long get_wchan(struct task_struct *task)unsigned long ret =0 ;int count =0 ; -if (!task || task == current || - task->state ==TASK_RUNNING ) +if (!task || task == current ||task_is_running (task)) goto out; tp =task_thread_info (task);
arch/um/kernel/process.c +1 -1
@@ -369,7 +369,7 @@ unsigned long get_wchan(struct task_struct *p)unsigned long stack_page, sp, ip;bool seen_sched =0 ; -if ((p ==NULL ) || (p == current) || (p->state ==TASK_RUNNING )) +if ((p ==NULL ) || (p == current) ||task_is_running (p))return 0 ; stack_page = (unsigned long )task_stack_page (p);
arch/x86/kernel/process.c +2 -2
@@ -931,7 +931,7 @@ unsigned long get_wchan(struct task_struct *p)unsigned long start, bottom, top, sp, fp, ip, ret =0 ;int count =0 ; -if (p == current || p->state ==TASK_RUNNING ) +if (p == current ||task_is_running (p))return 0 ;if (!try_get_task_stack (p))
@@ -975,7 +975,7 @@ unsigned long get_wchan(struct task_struct *p) goto out; } fp =READ_ONCE_NOCHECK (*(unsigned long *)fp); - }while (count++ <16 && p->state !=TASK_RUNNING ); + }while (count++ <16 && !task_is_running (p)); out:put_task_stack (p);
arch/xtensa/kernel/process.c +1 -1
@@ -304,7 +304,7 @@ unsigned long get_wchan(struct task_struct *p)unsigned long stack_page = (unsigned long )task_stack_page (p);int count =0 ; -if (!p || p == current || p->state ==TASK_RUNNING ) +if (!p || p == current ||task_is_running (p))return 0 ; sp = p->thread.sp;
block/blk-mq.c +1 -1
@@ -3926,7 +3926,7 @@ int blk_poll(struct request_queue *q, blk_qc_t cookie, bool spin)if (signal_pending_state (state, current))__set_current_state (TASK_RUNNING ); -if (current->state ==TASK_RUNNING ) +if (task_is_running (current))return 1 ;if (ret <0 || !spin)break ;
include/linux/sched.h +2 -0
@@ -113,6 +113,8 @@ struct task_group; __TASK_TRACED | EXIT_DEAD | EXIT_ZOMBIE | \ TASK_PARKED) +#define task_is_running (task) (READ_ONCE((task)->state) == TASK_RUNNING) +#define task_is_traced (task) ((task->state & __TASK_TRACED) != 0)#define task_is_stopped (task) ((task->state & __TASK_STOPPED) != 0)
kernel/kcsan/report.c +1 -1
@@ -460,7 +460,7 @@ static void set_other_info_task_blocking(unsigned long *flags,* We may be instrumenting a code-path where current->state is already * something other than TASK_RUNNING. */ -const bool is_running = current->state ==TASK_RUNNING ; +const bool is_running =task_is_running (current);/* * To avoid deadlock in case we are in an interrupt here and this is a * race with a task on the same CPU (KCSAN_INTERRUPT_WATCHER), provide a
kernel/locking/lockdep.c +1 -1
@@ -760,7 +760,7 @@ static void lockdep_print_held_locks(struct task_struct *p)* It's not reliable to print a task's held locks if it's not sleeping * and it's not the current task. */ -if (p->state ==TASK_RUNNING && p != current) +if (p != current &&task_is_running (p))return ;for (i =0 ; i < depth; i++) {printk (" #%d: " , i);
kernel/rcu/tree_plugin.h +1 -1
@@ -2768,7 +2768,7 @@ EXPORT_SYMBOL_GPL(rcu_bind_current_to_nocb);#ifdef CONFIG_SMP static char *show_rcu_should_be_on_cpu (struct task_struct *tsp) { -return tsp && tsp->state ==TASK_RUNNING && !tsp->on_cpu ?"!" :"" ; +return tsp &&task_is_running (tsp) && !tsp->on_cpu ?"!" :"" ; }#else // #ifdef CONFIG_SMP static char *show_rcu_should_be_on_cpu (struct task_struct *tsp)
kernel/sched/core.c +3 -3
@@ -5974,7 +5974,7 @@ static inline void sched_submit_work(struct task_struct *tsk) {unsigned int task_flags; -if (!tsk->state) +if (task_is_running (tsk))return ; task_flags = tsk->flags;
@@ -7949,7 +7949,7 @@ again:if (curr->sched_class != p->sched_class) goto out_unlock; -if (task_running (p_rq, p) || p->state) +if (task_running (p_rq, p) || !task_is_running (p)) goto out_unlock; yielded = curr->sched_class->yield_to_task (rq, p);
@@ -8152,7 +8152,7 @@ void sched_show_task(struct task_struct *p)pr_info ("task:%-15.15s state:%c" , p->comm,task_state_to_char (p)); -if (p->state ==TASK_RUNNING ) +if (task_is_running (p))pr_cont (" running task " );#ifdef CONFIG_DEBUG_STACK_USAGE free =stack_not_used (p);
kernel/sched/stats.h +1 -1
@@ -217,7 +217,7 @@ static inline void sched_info_depart(struct rq *rq, struct task_struct *t)rq_sched_info_depart (rq, delta); -if (t->state ==TASK_RUNNING ) +if (task_is_running (t))sched_info_enqueue (rq, t); }
kernel/signal.c +1 -1
@@ -4719,7 +4719,7 @@ void kdb_send_sig(struct task_struct *t, int sig) } new_t = kdb_prev_t != t; kdb_prev_t = t; -if (t->state !=TASK_RUNNING && new_t) { +if (!task_is_running (t) && new_t) {spin_unlock (&t->sighand->siglock);kdb_printf ("Process is not RUNNING, sending a signal from " "kdb risks deadlock\n"
kernel/softirq.c +1 -2
@@ -92,8 +92,7 @@ static bool ksoftirqd_running(unsigned long pending)if (pending &SOFTIRQ_NOW_MASK )return false; -return tsk && (tsk->state ==TASK_RUNNING ) && - !__kthread_should_park (tsk); +return tsk &&task_is_running (tsk) && !__kthread_should_park (tsk); }#ifdef CONFIG_TRACE_IRQFLAGS
mm/compaction.c +1 -1
@@ -1955,7 +1955,7 @@ static inline bool is_via_compact_memory(int order)static bool kswapd_is_running (pg_data_t *pgdat) { -return pgdat->kswapd && (pgdat->kswapd->state ==TASK_RUNNING ); +return pgdat->kswapd &&task_is_running (pgdat->kswapd); }/*
Peter Zijlstra
maintainerPATCH v2 3/7 sched,perf,kvm: Fix preemption condition
In reply to Peter ZijlstraWhen ran from the sched-out path (preempt_notifier or perf_event), p->state is irrelevant to determine preemption. You can get preempted with !task_is_running() just fine.
The right indicator for preemption is if the task is still on the runqueue in the sched-out path.
Signed-off-by Peter Zijlstra (Intel) <[email protected]>
Acked-by Mark Rutland <[email protected]>
--- kernel/events/core.c | 7 +++---- virt/kvm/kvm_main.c | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-)
kernel/events/core.c +3 -4
@@ -8568,13 +8568,12 @@ static void perf_event_switch(struct tas }, }; -if (!sched_in && task->state ==TASK_RUNNING ) +if (!sched_in && task->on_rq) { switch_event.event_id.header.misc |=PERF_RECORD_MISC_SWITCH_OUT_PREEMPT ; + } -perf_iterate_sb (perf_event_switch_output, - &switch_event, -NULL ); +perf_iterate_sb (perf_event_switch_output, &switch_event,NULL ); }/*
virt/kvm/kvm_main.c +1 -1
@@ -4869,7 +4869,7 @@ static void kvm_sched_out(struct preempt {struct kvm_vcpu *vcpu =preempt_notifier_to_vcpu (pn); -if (current->state ==TASK_RUNNING ) { +if (current->on_rq) {WRITE_ONCE (vcpu->preempted, true);WRITE_ONCE (vcpu->ready, true); }
[tip: sched/core] sched,perf,kvm: Fix preemption condition
The following commit has been merged into the sched/core branch of tip:
The following commit has been merged into the sched/core branch of tip:
Commit-ID: 3ba9f93b12361e005dd65fcc8072b42e3189f4f4 Gitweb: https://git.kernel.org/tip/3ba9f93b12361e005dd65fcc8072b42e3189f4f4 Author: Peter Zijlstra <[email protected]> AuthorDate: Fri, 11 Jun 2021 10:28:13 +02:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Fri, 18 Jun 2021 11:43:07 +02:00
sched,perf,kvm: Fix preemption condition
When ran from the sched-out path (preempt_notifier or perf_event), p->state is irrelevant to determine preemption. You can get preempted with !task_is_running() just fine.
The right indicator for preemption is if the task is still on the runqueue in the sched-out path.
Signed-off-by Peter Zijlstra (Intel) <[email protected]>
Acked-by Mark Rutland <[email protected]>
--- kernel/events/core.c | 7 +++---- virt/kvm/kvm_main.c | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-)
kernel/events/core.c +3 -4
@@ -8682,13 +8682,12 @@ static void perf_event_switch(struct task_struct *task, }, }; -if (!sched_in && task->state ==TASK_RUNNING ) +if (!sched_in && task->on_rq) { switch_event.event_id.header.misc |=PERF_RECORD_MISC_SWITCH_OUT_PREEMPT ; + } -perf_iterate_sb (perf_event_switch_output, - &switch_event, -NULL ); +perf_iterate_sb (perf_event_switch_output, &switch_event,NULL ); }/*
virt/kvm/kvm_main.c +1 -1
@@ -5025,7 +5025,7 @@ static void kvm_sched_out(struct preempt_notifier *pn, {struct kvm_vcpu *vcpu =preempt_notifier_to_vcpu (pn); -if (current->state ==TASK_RUNNING ) { +if (current->on_rq) {WRITE_ONCE (vcpu->preempted, true);WRITE_ONCE (vcpu->ready, true); }
Peter Zijlstra
maintainerPATCH v2 4/7 sched: Add get_current_state()
In reply to Peter ZijlstraRemove yet another few p->state accesses.
Signed-off-by Peter Zijlstra (Intel) <[email protected]>
Acked-by Will Deacon <[email protected]>
--- block/blk-mq.c | 2 +- include/linux/sched.h | 2 ++ kernel/freezer.c | 2 +- kernel/sched/core.c | 6 +++--- 4 files changed, 7 insertions(+), 5 deletions(-)
block/blk-mq.c +1 -1
@@ -3891,7 +3891,7 @@ int blk_poll(struct request_queue *q, bl hctx->poll_considered++; - state = current->state; + state =get_current_state (); do { int ret;
include/linux/sched.h +2 -0
@@ -212,6 +212,8 @@ struct task_group;#endif +#define get_current_state () READ_ONCE(current->state) +/* Task command name length: */ #define TASK_COMM_LEN 16
kernel/freezer.c +1 -1
@@ -58,7 +58,7 @@ bool __refrigerator(bool check_kthr_stop/* Hmm, should we be allowed to suspend when there are realtime processes around? */ bool was_frozen = false; -long save = current->state; +unsigned int save =get_current_state ();pr_debug ("%s entered refrigerator\n" , current->comm);
kernel/sched/core.c +3 -3
@@ -8273,15 +8273,15 @@ static inline int preempt_count_equals(ivoid __might_sleep (const char *file,int line,int preempt_offset) { +unsigned int state =get_current_state ();/* * Blocking primitives will set (and therefore destroy) current->state, * since we will exit with TASK_RUNNING make sure we enter with it, * otherwise we will destroy state. */ -WARN_ONCE (current->state !=TASK_RUNNING && current->task_state_change, +WARN_ONCE (state !=TASK_RUNNING && current->task_state_change,"do not call blocking ops when !TASK_RUNNING; " -"state=%lx set at [<%p>] %pS\n" , - current->state, +"state=%x set at [<%p>] %pS\n" , state, (void *)current->task_state_change, (void *)current->task_state_change);
[tip: sched/core] sched: Add get_current_state()
The following commit has been merged into the sched/core branch of tip:
The following commit has been merged into the sched/core branch of tip:
Commit-ID: d6c23bb3a2ad2f8f7dd46292b8bc54d27f2fb3f1 Gitweb: https://git.kernel.org/tip/d6c23bb3a2ad2f8f7dd46292b8bc54d27f2fb3f1 Author: Peter Zijlstra <[email protected]> AuthorDate: Fri, 11 Jun 2021 10:28:14 +02:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Fri, 18 Jun 2021 11:43:08 +02:00
sched: Add get_current_state()
Remove yet another few p->state accesses.
Signed-off-by Peter Zijlstra (Intel) <[email protected]>
Acked-by Will Deacon <[email protected]>
--- block/blk-mq.c | 2 +- include/linux/sched.h | 2 ++ kernel/freezer.c | 2 +- kernel/sched/core.c | 6 +++--- 4 files changed, 7 insertions(+), 5 deletions(-)
block/blk-mq.c +1 -1
@@ -3910,7 +3910,7 @@ int blk_poll(struct request_queue *q, blk_qc_t cookie, bool spin) hctx->poll_considered++; - state = current->state; + state =get_current_state (); do { int ret;
include/linux/sched.h +2 -0
@@ -213,6 +213,8 @@ struct task_group;#endif +#define get_current_state () READ_ONCE(current->state) +/* Task command name length: */ #define TASK_COMM_LEN 16
kernel/freezer.c +1 -1
@@ -58,7 +58,7 @@ bool __refrigerator(bool check_kthr_stop)/* Hmm, should we be allowed to suspend when there are realtime processes around? */ bool was_frozen = false; -long save = current->state; +unsigned int save =get_current_state ();pr_debug ("%s entered refrigerator\n" , current->comm);
kernel/sched/core.c +3 -3
@@ -9098,15 +9098,15 @@ static inline int preempt_count_equals(int preempt_offset)void __might_sleep (const char *file,int line,int preempt_offset) { +unsigned int state =get_current_state ();/* * Blocking primitives will set (and therefore destroy) current->state, * since we will exit with TASK_RUNNING make sure we enter with it, * otherwise we will destroy state. */ -WARN_ONCE (current->state !=TASK_RUNNING && current->task_state_change, +WARN_ONCE (state !=TASK_RUNNING && current->task_state_change,"do not call blocking ops when !TASK_RUNNING; " -"state=%lx set at [<%p>] %pS\n" , - current->state, +"state=%x set at [<%p>] %pS\n" , state, (void *)current->task_state_change, (void *)current->task_state_change);
Peter Zijlstra
maintainerPATCH v2 5/7 sched,timer: Use __set_current_state()
In reply to Peter ZijlstraThere's an existing helper for setting TASK_RUNNING; must've gotten lost last time we did this cleanup.
Signed-off-by Peter Zijlstra (Intel) <[email protected]>
Reviewed-by Davidlohr Bueso <[email protected]>
Acked-by Will Deacon <[email protected]>
--- kernel/time/timer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
kernel/time/timer.c +1 -1
@@ -1879,7 +1879,7 @@ signed long __sched schedule_timeout(sigprintk (KERN_ERR "schedule_timeout: wrong timeout " "value %lx\n" , timeout);dump_stack (); - current->state =TASK_RUNNING ; +__set_current_state (TASK_RUNNING ); goto out; } }
[tip: sched/core] sched,timer: Use __set_current_state()
The following commit has been merged into the sched/core branch of tip:
The following commit has been merged into the sched/core branch of tip:
Commit-ID: 600642ae9050a872055119ba09d0decc43f6c843 Gitweb: https://git.kernel.org/tip/600642ae9050a872055119ba09d0decc43f6c843 Author: Peter Zijlstra <[email protected]> AuthorDate: Fri, 11 Jun 2021 10:28:15 +02:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Fri, 18 Jun 2021 11:43:08 +02:00
sched,timer: Use __set_current_state()
There's an existing helper for setting TASK_RUNNING; must've gotten lost last time we did this cleanup.
Signed-off-by Peter Zijlstra (Intel) <[email protected]>
Reviewed-by Davidlohr Bueso <[email protected]>
Acked-by Will Deacon <[email protected]>
--- kernel/time/timer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
kernel/time/timer.c +1 -1
@@ -1879,7 +1879,7 @@ signed long __sched schedule_timeout(signed long timeout)printk (KERN_ERR "schedule_timeout: wrong timeout " "value %lx\n" , timeout);dump_stack (); - current->state =TASK_RUNNING ; +__set_current_state (TASK_RUNNING ); goto out; } }
Peter Zijlstra
maintainerPATCH v2 6/7 sched,arch: Remove unused TASK_STATE offsets
In reply to Peter ZijlstraAll 6 architectures define TASK_STATE in asm-offsets, but then never actually use it. Remove the definitions to make sure they never will.
Signed-off-by Peter Zijlstra (Intel) <[email protected]>
--- arch/csky/kernel/asm-offsets.c | 1 - arch/h8300/kernel/asm-offsets.c | 1 - arch/microblaze/kernel/asm-offsets.c | 1 - arch/mips/kernel/asm-offsets.c | 1 - arch/openrisc/kernel/asm-offsets.c | 1 - arch/parisc/kernel/asm-offsets.c | 1 - 6 files changed, 6 deletions(-)
arch/csky/kernel/asm-offsets.c +0 -1
@@ -9,7 +9,6 @@int main (void ) {/* offsets into the task struct */ -DEFINE (TASK_STATE , offsetof(struct task_struct , state));DEFINE (TASK_THREAD_INFO , offsetof(struct task_struct , stack));DEFINE (TASK_FLAGS , offsetof(struct task_struct , flags));DEFINE (TASK_PTRACE , offsetof(struct task_struct , ptrace));
arch/h8300/kernel/asm-offsets.c +0 -1
@@ -21,7 +21,6 @@int main (void ) {/* offsets into the task struct */ -OFFSET (TASK_STATE , task_struct, state);OFFSET (TASK_FLAGS , task_struct, flags);OFFSET (TASK_PTRACE , task_struct, ptrace);OFFSET (TASK_BLOCKED , task_struct, blocked);
arch/microblaze/kernel/asm-offsets.c +0 -1
@@ -70,7 +70,6 @@ int main(int argc, char *argv[])/* struct task_struct */ DEFINE (TS_THREAD_INFO , offsetof(struct task_struct , stack)); -DEFINE (TASK_STATE , offsetof(struct task_struct , state));DEFINE (TASK_FLAGS , offsetof(struct task_struct , flags));DEFINE (TASK_PTRACE , offsetof(struct task_struct , ptrace));DEFINE (TASK_BLOCKED , offsetof(struct task_struct , blocked));
arch/mips/kernel/asm-offsets.c +0 -1
@@ -78,7 +78,6 @@ void output_ptreg_defines(void)void output_task_defines (void ) {COMMENT ("MIPS task_struct offsets." ); -OFFSET (TASK_STATE , task_struct, state);OFFSET (TASK_THREAD_INFO , task_struct, stack);OFFSET (TASK_FLAGS , task_struct, flags);OFFSET (TASK_MM , task_struct, mm);
arch/openrisc/kernel/asm-offsets.c +0 -1
@@ -37,7 +37,6 @@int main (void ) {/* offsets into the task_struct */ -DEFINE (TASK_STATE , offsetof(struct task_struct , state));DEFINE (TASK_FLAGS , offsetof(struct task_struct , flags));DEFINE (TASK_PTRACE , offsetof(struct task_struct , ptrace));DEFINE (TASK_THREAD , offsetof(struct task_struct , thread));
arch/parisc/kernel/asm-offsets.c +0 -1
@@ -42,7 +42,6 @@int main (void ) {DEFINE (TASK_THREAD_INFO , offsetof(struct task_struct , stack)); -DEFINE (TASK_STATE , offsetof(struct task_struct , state));DEFINE (TASK_FLAGS , offsetof(struct task_struct , flags));DEFINE (TASK_SIGPENDING , offsetof(struct task_struct , pending));DEFINE (TASK_PTRACE , offsetof(struct task_struct , ptrace));
[tip: sched/core] sched,arch: Remove unused TASK_STATE offsets
The following commit has been merged into the sched/core branch of tip:
The following commit has been merged into the sched/core branch of tip:
Commit-ID: 7c3edd6d9cb4d8ea8db5b167dc2eee94d7e4667b Gitweb: https://git.kernel.org/tip/7c3edd6d9cb4d8ea8db5b167dc2eee94d7e4667b Author: Peter Zijlstra <[email protected]> AuthorDate: Fri, 11 Jun 2021 10:28:16 +02:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Fri, 18 Jun 2021 11:43:09 +02:00
sched,arch: Remove unused TASK_STATE offsets
All 6 architectures define TASK_STATE in asm-offsets, but then never actually use it. Remove the definitions to make sure they never will.
Signed-off-by Peter Zijlstra (Intel) <[email protected]>
--- arch/csky/kernel/asm-offsets.c | 1 - arch/h8300/kernel/asm-offsets.c | 1 - arch/microblaze/kernel/asm-offsets.c | 1 - arch/mips/kernel/asm-offsets.c | 1 - arch/openrisc/kernel/asm-offsets.c | 1 - arch/parisc/kernel/asm-offsets.c | 1 - 6 files changed, 6 deletions(-)
arch/csky/kernel/asm-offsets.c +0 -1
@@ -9,7 +9,6 @@int main (void ) {/* offsets into the task struct */ -DEFINE (TASK_STATE , offsetof(struct task_struct , state));DEFINE (TASK_THREAD_INFO , offsetof(struct task_struct , stack));DEFINE (TASK_FLAGS , offsetof(struct task_struct , flags));DEFINE (TASK_PTRACE , offsetof(struct task_struct , ptrace));
arch/h8300/kernel/asm-offsets.c +0 -1
@@ -21,7 +21,6 @@int main (void ) {/* offsets into the task struct */ -OFFSET (TASK_STATE , task_struct, state);OFFSET (TASK_FLAGS , task_struct, flags);OFFSET (TASK_PTRACE , task_struct, ptrace);OFFSET (TASK_BLOCKED , task_struct, blocked);
arch/microblaze/kernel/asm-offsets.c +0 -1
@@ -70,7 +70,6 @@ int main(int argc, char *argv[])/* struct task_struct */ DEFINE (TS_THREAD_INFO , offsetof(struct task_struct , stack)); -DEFINE (TASK_STATE , offsetof(struct task_struct , state));DEFINE (TASK_FLAGS , offsetof(struct task_struct , flags));DEFINE (TASK_PTRACE , offsetof(struct task_struct , ptrace));DEFINE (TASK_BLOCKED , offsetof(struct task_struct , blocked));
arch/mips/kernel/asm-offsets.c +0 -1
@@ -78,7 +78,6 @@ void output_ptreg_defines(void)void output_task_defines (void ) {COMMENT ("MIPS task_struct offsets." ); -OFFSET (TASK_STATE , task_struct, state);OFFSET (TASK_THREAD_INFO , task_struct, stack);OFFSET (TASK_FLAGS , task_struct, flags);OFFSET (TASK_MM , task_struct, mm);
arch/openrisc/kernel/asm-offsets.c +0 -1
@@ -37,7 +37,6 @@int main (void ) {/* offsets into the task_struct */ -DEFINE (TASK_STATE , offsetof(struct task_struct , state));DEFINE (TASK_FLAGS , offsetof(struct task_struct , flags));DEFINE (TASK_PTRACE , offsetof(struct task_struct , ptrace));DEFINE (TASK_THREAD , offsetof(struct task_struct , thread));
arch/parisc/kernel/asm-offsets.c +0 -1
@@ -42,7 +42,6 @@int main (void ) {DEFINE (TASK_THREAD_INFO , offsetof(struct task_struct , stack)); -DEFINE (TASK_STATE , offsetof(struct task_struct , state));DEFINE (TASK_FLAGS , offsetof(struct task_struct , flags));DEFINE (TASK_SIGPENDING , offsetof(struct task_struct , pending));DEFINE (TASK_PTRACE , offsetof(struct task_struct , ptrace));
Peter Zijlstra
maintainerPATCH v2 7/7 sched: Change task_struct::state
In reply to Peter ZijlstraChange the type and name of task_struct::state. Drop the volatile and shrink it to an 'unsigned int'. Rename it in order to find all uses such that we can use READ_ONCE/WRITE_ONCE as appropriate.
Signed-off-by Peter Zijlstra (Intel) <[email protected]>
Acked-by Will Deacon <[email protected]>
Acked-by Daniel Thompson <[email protected]>
--- arch/ia64/kernel/mca.c | 2 - arch/ia64/kernel/ptrace.c | 8 +++--- arch/powerpc/xmon/xmon.c | 13 +++++----- block/blk-mq.c | 2 - drivers/md/dm.c | 6 ++-- fs/binfmt_elf.c | 8 +++--- fs/binfmt_elf_fdpic.c | 4 ++- fs/userfaultfd.c | 4 +-- include/linux/sched.h | 31 +++++++++++------------ include/linux/sched/debug.h | 2 - include/linux/sched/signal.h | 2 - init/init_task.c | 2 - kernel/cgroup/cgroup-v1.c | 2 - kernel/debug/kdb/kdb_support.c | 18 +++++++------ kernel/fork.c | 4 +-- kernel/hung_task.c | 2 - kernel/kthread.c | 4 +-- kernel/locking/mutex.c | 6 ++-- kernel/locking/rtmutex.c | 4 +-- kernel/locking/rwsem.c | 2 - kernel/ptrace.c | 12 ++++----- kernel/rcu/rcutorture.c | 4 +-- kernel/rcu/tree_stall.h | 12 ++++----- kernel/sched/core.c | 53 +++++++++++++++++++++-------------------- kernel/sched/deadline.c | 10 +++---- kernel/sched/fair.c | 11 +++++--- lib/syscall.c | 4 +-- net/core/dev.c | 2 - 28 files changed, 123 insertions(+), 111 deletions(-)
arch/ia64/kernel/mca.c +1 -1
@@ -1788,7 +1788,7 @@ format_mca_init_stack(void *mca_data, un ti->task = p; ti->cpu = cpu; p->stack = ti; - p->state =TASK_UNINTERRUPTIBLE ; + p->__state =TASK_UNINTERRUPTIBLE ;cpumask_set_cpu (cpu, &p->cpus_mask);INIT_LIST_HEAD (&p->tasks); p->parent = p->real_parent = p->group_leader = p;
arch/ia64/kernel/ptrace.c +4 -4
@@ -641,11 +641,11 @@ ptrace_attach_sync_user_rbs (struct taskread_lock (&tasklist_lock);if (child->sighand) {spin_lock_irq (&child->sighand->siglock); -if (child->state ==TASK_STOPPED && +if (READ_ONCE (child->__state) ==TASK_STOPPED && !test_and_set_tsk_thread_flag (child,TIF_RESTORE_RSE )) {set_notify_resume (child); - child->state =TASK_TRACED ; +WRITE_ONCE (child->__state,TASK_TRACED ); stopped =1 ; }spin_unlock_irq (&child->sighand->siglock);
@@ -665,9 +665,9 @@ ptrace_attach_sync_user_rbs (struct taskread_lock (&tasklist_lock);if (child->sighand) {spin_lock_irq (&child->sighand->siglock); -if (child->state ==TASK_TRACED && +if (READ_ONCE (child->__state) ==TASK_TRACED && (child->signal->flags &SIGNAL_STOP_STOPPED )) { - child->state =TASK_STOPPED ; +WRITE_ONCE (child->__state,TASK_STOPPED ); }spin_unlock_irq (&child->sighand->siglock); }
arch/powerpc/xmon/xmon.c +7 -6
@@ -3162,6 +3162,7 @@ memzcan(void)static void show_task (struct task_struct *tsk) { +unsigned int p_state =READ_ONCE (tsk->__state);char state;/*
@@ -3169,14 +3170,14 @@ static void show_task(struct task_struct* appropriate for calling from xmon. This could be moved * to a common, generic, routine used by both. */ - state = (tsk->state ==0 ) ?'R' : - (tsk->state <0 ) ?'U' : - (tsk->state &TASK_UNINTERRUPTIBLE ) ?'D' : - (tsk->state &TASK_STOPPED ) ?'T' : - (tsk->state &TASK_TRACED ) ?'C' : + state = (p_state ==0 ) ?'R' : + (p_state <0 ) ?'U' : + (p_state &TASK_UNINTERRUPTIBLE ) ?'D' : + (p_state &TASK_STOPPED ) ?'T' : + (p_state &TASK_TRACED ) ?'C' : (tsk->exit_state &EXIT_ZOMBIE ) ?'Z' : (tsk->exit_state &EXIT_DEAD ) ?'E' : - (tsk->state &TASK_INTERRUPTIBLE ) ?'S' :'?' ; + (p_state &TASK_INTERRUPTIBLE ) ?'S' :'?' ; printf("%16px %16lx %16px %6d %6d %c %2d %s\n", tsk, tsk->thread.ksp, tsk->thread.regs,
block/blk-mq.c +1 -1
@@ -3886,7 +3886,7 @@ static bool blk_mq_poll_hybrid(struct reint blk_poll (struct request_queue *q,blk_qc_t cookie,bool spin) {struct blk_mq_hw_ctx *hctx; -long state; +unsigned int state;if (!blk_qc_t_valid (cookie) || !test_bit (QUEUE_FLAG_POLL , &q->queue_flags))
drivers/md/dm.c +3 -3
@@ -2328,7 +2328,7 @@ static bool md_in_flight_bios(struct mapreturn sum !=0 ; } -static int dm_wait_for_bios_completion (struct mapped_device *md,long task_state) +static int dm_wait_for_bios_completion (struct mapped_device *md,unsigned int task_state) {int r =0 ;DEFINE_WAIT (wait);
@@ -2351,7 +2351,7 @@ static int dm_wait_for_bios_completion(sreturn r; } -static int dm_wait_for_completion (struct mapped_device *md,long task_state) +static int dm_wait_for_completion (struct mapped_device *md,unsigned int task_state) {int r =0 ;
@@ -2478,7 +2478,7 @@ static void unlock_fs(struct mapped_devi* are being added to md->deferred list. */ static int __dm_suspend (struct mapped_device *md,struct dm_table *map, -unsigned suspend_flags,long task_state, +unsigned suspend_flags,unsigned int task_state,int dmf_suspended_flag) {bool do_lockfs = suspend_flags &DM_SUSPEND_LOCKFS_FLAG ;
fs/binfmt_elf.c +5 -3
@@ -1537,7 +1537,8 @@ static int fill_psinfo(struct elf_prpsin {const struct cred *cred;unsigned int i, len; - +unsigned int state; +/* first copy the parameters from user space */ memset (psinfo,0 ,sizeof (struct elf_prpsinfo ));
@@ -1559,7 +1560,8 @@ static int fill_psinfo(struct elf_prpsin psinfo->pr_pgrp =task_pgrp_vnr (p); psinfo->pr_sid =task_session_vnr (p); - i = p->state ?ffz (~p->state) +1 :0 ; + state =READ_ONCE (p->__state); + i = state ?ffz (~state) +1 :0 ; psinfo->pr_state = i; psinfo->pr_sname = (i >5 ) ?'.' :"RSDTZW" [i]; psinfo->pr_zomb = psinfo->pr_sname =='Z' ;
@@ -1571,7 +1573,7 @@ static int fill_psinfo(struct elf_prpsinSET_GID (psinfo->pr_gid,from_kgid_munged (cred->user_ns, cred->gid));rcu_read_unlock ();strncpy (psinfo->pr_fname, p->comm,sizeof (psinfo->pr_fname)); - +return 0 ; }
fs/binfmt_elf_fdpic.c +3 -1
@@ -1331,6 +1331,7 @@ static int fill_psinfo(struct elf_prpsin {const struct cred *cred;unsigned int i, len; +unsigned int state;/* first copy the parameters from user space */ memset (psinfo,0 ,sizeof (struct elf_prpsinfo ));
@@ -1353,7 +1354,8 @@ static int fill_psinfo(struct elf_prpsin psinfo->pr_pgrp =task_pgrp_vnr (p); psinfo->pr_sid =task_session_vnr (p); - i = p->state ?ffz (~p->state) +1 :0 ; + state =READ_ONCE (p->__state); + i = state ?ffz (~state) +1 :0 ; psinfo->pr_state = i; psinfo->pr_sname = (i >5 ) ?'.' :"RSDTZW" [i]; psinfo->pr_zomb = psinfo->pr_sname =='Z' ;
fs/userfaultfd.c +2 -2
@@ -337,7 +337,7 @@ static inline bool userfaultfd_must_waitreturn ret; } -static inline long userfaultfd_get_blocking_state (unsigned int flags) +static inline unsigned int userfaultfd_get_blocking_state (unsigned int flags) {if (flags &FAULT_FLAG_INTERRUPTIBLE )return TASK_INTERRUPTIBLE ;
@@ -370,7 +370,7 @@ vm_fault_t handle_userfault(struct vm_fastruct userfaultfd_wait_queue uwq;vm_fault_t ret =VM_FAULT_SIGBUS ;bool must_wait; -long blocking_state; +unsigned int blocking_state;/* * We don't do userfault handling for the final child pid update.
include/linux/sched.h +15 -16
@@ -113,13 +113,13 @@ struct task_group; __TASK_TRACED | EXIT_DEAD | EXIT_ZOMBIE | \ TASK_PARKED) -#define task_is_running (task) (READ_ONCE((task)->state) == TASK_RUNNING) +#define task_is_running (task) (READ_ONCE((task)->__state) == TASK_RUNNING) -#define task_is_traced (task) ((task->state & __TASK_TRACED) != 0) +#define task_is_traced (task) ((READ_ONCE(task->__state) & __TASK_TRACED) != 0) -#define task_is_stopped (task) ((task->state & __TASK_STOPPED) != 0) +#define task_is_stopped (task) ((READ_ONCE(task->__state) & __TASK_STOPPED) != 0) -#define task_is_stopped_or_traced (task) ((task->state & (__TASK_STOPPED | __TASK_TRACED)) != 0) +#define task_is_stopped_or_traced (task) ((READ_ONCE(task->__state) & (__TASK_STOPPED | __TASK_TRACED)) != 0)#ifdef CONFIG_DEBUG_ATOMIC_SLEEP
@@ -134,14 +134,14 @@ struct task_group;do { \WARN_ON_ONCE (is_special_task_state (state_value));\ current->task_state_change = _THIS_IP_; \ - current->state = (state_value); \ +WRITE_ONCE (current->__state, (state_value)); \ } while (0)#define set_current_state (state_value) \ do { \ WARN_ON_ONCE(is_special_task_state(state_value));\ current->task_state_change = _THIS_IP_; \ - smp_store_mb(current->state, (state_value)); \ + smp_store_mb(current->__state, (state_value)); \ } while (0)#define set_special_state (state_value) \
@@ -150,7 +150,7 @@ struct task_group;WARN_ON_ONCE (!is_special_task_state (state_value)); \raw_spin_lock_irqsave (¤t->pi_lock, flags); \ current->task_state_change = _THIS_IP_; \ - current->state = (state_value); \ +WRITE_ONCE (current->__state, (state_value)); \raw_spin_unlock_irqrestore (¤t->pi_lock, flags); \ } while (0) #else
@@ -192,10 +192,10 @@ struct task_group;* Also see the comments of try_to_wake_up(). */ #define __set_current_state (state_value) \ - current->state = (state_value) + WRITE_ONCE(current->__state, (state_value))#define set_current_state (state_value) \ - smp_store_mb(current->state, (state_value)) + smp_store_mb(current->__state, (state_value))/* * set_special_state() should be used for those states when the blocking task
@@ -207,13 +207,13 @@ struct task_group;do { \unsigned long flags;/* may shadow */ \raw_spin_lock_irqsave (¤t->pi_lock, flags); \ - current->state = (state_value); \ +WRITE_ONCE (current->__state, (state_value)); \raw_spin_unlock_irqrestore (¤t->pi_lock, flags); \ } while (0)#endif -#define get_current_state () READ_ONCE(current->state) +#define get_current_state () READ_ONCE(current->__state)/* Task command name length: */ #define TASK_COMM_LEN 16
@@ -658,8 +658,7 @@ struct task_struct {*/ struct thread_info thread_info;#endif -/* -1 unrunnable, 0 runnable, >0 stopped: */ -volatile long state; +unsigned int __state;/* * This begins the randomizable portion of task_struct. Only
@@ -1524,7 +1523,7 @@ static inline pid_t task_pgrp_nr(structstatic inline unsigned int task_state_index (struct task_struct *tsk) { -unsigned int tsk_state =READ_ONCE (tsk->state); +unsigned int tsk_state =READ_ONCE (tsk->__state);unsigned int state = (tsk_state | tsk->exit_state) &TASK_REPORT ;BUILD_BUG_ON_NOT_POWER_OF_2 (TASK_REPORT_MAX );
@@ -1832,10 +1831,10 @@ static __always_inline void scheduler_ip*/ preempt_fold_need_resched(); } -extern unsigned long wait_task_inactive (struct task_struct *,long match_state); +extern unsigned long wait_task_inactive (struct task_struct *,unsigned int match_state);#else static inline void scheduler_ipi (void ) { } -static inline unsigned long wait_task_inactive (struct task_struct *p,long match_state) +static inline unsigned long wait_task_inactive (struct task_struct *p,unsigned int match_state) {return 1 ; }
include/linux/sched/debug.h +1 -1
@@ -14,7 +14,7 @@ extern void dump_cpu_task(int cpu);/* * Only dump TASK_* tasks. (0 for all tasks) */ -extern void show_state_filter (unsigned long state_filter); +extern void show_state_filter (unsigned int state_filter);static inline void show_state (void ) {
include/linux/sched/signal.h +1 -1
@@ -382,7 +382,7 @@ static inline int fatal_signal_pending(sreturn task_sigpending (p) &&__fatal_signal_pending (p); } -static inline int signal_pending_state (long state,struct task_struct *p) +static inline int signal_pending_state (unsigned int state,struct task_struct *p) {if (!(state & (TASK_INTERRUPTIBLE |TASK_WAKEKILL )))return 0 ;
init/init_task.c +1 -1
@@ -71,7 +71,7 @@ struct task_struct init_task .thread_info = INIT_THREAD_INFO(init_task), .stack_refcount = REFCOUNT_INIT(1), #endif - .state = 0, + .__state = 0, .stack = init_stack, .usage = REFCOUNT_INIT(2), .flags = PF_KTHREAD,
kernel/cgroup/cgroup-v1.c +1 -1
@@ -713,7 +713,7 @@ int cgroupstats_build(struct cgroupstatscss_task_iter_start (&cgrp->self,0 , &it);while ((tsk =css_task_iter_next (&it))) { -switch (tsk->state) { +switch (READ_ONCE (tsk->__state)) {case TASK_RUNNING : stats->nr_running++;break ;
kernel/debug/kdb/kdb_support.c +10 -8
@@ -609,23 +609,25 @@ unsigned long kdb_task_state_string(cons*/ char kdb_task_state_char (const struct task_struct *p) { -int cpu; -char state; +unsigned int p_state;unsigned long tmp; +char state; +int cpu;if (!p ||copy_from_kernel_nofault (&tmp, (char *)p,sizeof (unsigned long )))return 'E' ; cpu =kdb_process_cpu (p); - state = (p->state ==0 ) ?'R' : - (p->state <0 ) ?'U' : - (p->state &TASK_UNINTERRUPTIBLE ) ?'D' : - (p->state &TASK_STOPPED ) ?'T' : - (p->state &TASK_TRACED ) ?'C' : + p_state =READ_ONCE (p->__state); + state = (p_state ==0 ) ?'R' : + (p_state <0 ) ?'U' : + (p_state &TASK_UNINTERRUPTIBLE ) ?'D' : + (p_state &TASK_STOPPED ) ?'T' : + (p_state &TASK_TRACED ) ?'C' : (p->exit_state &EXIT_ZOMBIE ) ?'Z' : (p->exit_state &EXIT_DEAD ) ?'E' : - (p->state &TASK_INTERRUPTIBLE ) ?'S' :'?' ; + (p_state &TASK_INTERRUPTIBLE ) ?'S' :'?' ;if (is_idle_task (p)) {/* Idle task. Is it really idle, apart from the kdb * interrupt? */
kernel/fork.c +2 -2
@@ -425,7 +425,7 @@ static int memcg_charge_kernel_stack(strstatic void release_task_stack (struct task_struct *tsk) { -if (WARN_ON (tsk->state !=TASK_DEAD )) +if (WARN_ON (READ_ONCE (tsk->__state) !=TASK_DEAD ))return ;/* Better to leak the stack than to free prematurely */ account_kernel_stack (tsk,-1 );
@@ -2392,7 +2392,7 @@ static __latent_entropy struct task_struatomic_dec (&p->cred->user->processes);exit_creds (p); bad_fork_free: - p->state =TASK_DEAD ; +WRITE_ONCE (p->__state,TASK_DEAD );put_task_stack (p);delayed_free_task (p); fork_out:
kernel/hung_task.c +1 -1
@@ -196,7 +196,7 @@ static void check_hung_uninterruptible_t last_break = jiffies; }/* use "==" to skip the TASK_KILLABLE tasks waiting on NFS */ -if (t->state ==TASK_UNINTERRUPTIBLE ) +if (READ_ONCE (t->__state) ==TASK_UNINTERRUPTIBLE )check_hung_task (t, timeout); } unlock:
kernel/kthread.c +2 -2
@@ -457,7 +457,7 @@ struct task_struct *kthread_create_on_no }EXPORT_SYMBOL (kthread_create_on_node); -static void __kthread_bind_mask (struct task_struct *p,const struct cpumask *mask,long state) +static void __kthread_bind_mask (struct task_struct *p,const struct cpumask *mask,unsigned int state) {unsigned long flags;
@@ -473,7 +473,7 @@ static void __kthread_bind_mask(struct traw_spin_unlock_irqrestore (&p->pi_lock, flags); } -static void __kthread_bind (struct task_struct *p,unsigned int cpu,long state) +static void __kthread_bind (struct task_struct *p,unsigned int cpu,unsigned int state) {__kthread_bind_mask (p,cpumask_of (cpu), state); }
kernel/locking/mutex.c +3 -3
@@ -923,7 +923,7 @@ __ww_mutex_add_waiter(struct mutex_waite* Lock a mutex (possibly interruptible), slowpath: */ static __always_inline int __sched -__mutex_lock_common (struct mutex *lock,long state,unsigned int subclass, +__mutex_lock_common (struct mutex *lock,unsigned int state,unsigned int subclass,struct lockdep_map *nest_lock,unsigned long ip,struct ww_acquire_ctx *ww_ctx,const bool use_ww_ctx) {
@@ -1098,14 +1098,14 @@ __mutex_lock_common(struct mutex *lock, } static int __sched -__mutex_lock (struct mutex *lock,long state,unsigned int subclass, +__mutex_lock (struct mutex *lock,unsigned int state,unsigned int subclass,struct lockdep_map *nest_lock,unsigned long ip) {return __mutex_lock_common (lock, state, subclass, nest_lock, ip,NULL , false); } static int __sched -__ww_mutex_lock (struct mutex *lock,long state,unsigned int subclass, +__ww_mutex_lock (struct mutex *lock,unsigned int state,unsigned int subclass,struct lockdep_map *nest_lock,unsigned long ip,struct ww_acquire_ctx *ww_ctx) {
kernel/locking/rtmutex.c +2 -2
@@ -1135,7 +1135,7 @@ void __sched rt_mutex_init_waiter(struct* * Must be called with lock->wait_lock held and interrupts disabled */ -static int __sched __rt_mutex_slowlock(struct rt_mutex *lock, int state, +static int __sched __rt_mutex_slowlock(struct rt_mutex *lock, unsigned int state,struct hrtimer_sleeper *timeout,struct rt_mutex_waiter *waiter) {
@@ -1190,7 +1190,7 @@ static void __sched rt_mutex_handle_dead/* * Slow path lock function: */ -static int __sched rt_mutex_slowlock(struct rt_mutex *lock, int state, +static int __sched rt_mutex_slowlock(struct rt_mutex *lock, unsigned int state,struct hrtimer_sleeper *timeout,enum rtmutex_chainwalk chwalk) {
kernel/locking/rwsem.c +1 -1
@@ -889,7 +889,7 @@ rwsem_spin_on_owner(struct rw_semaphore* Wait for the read lock to be granted */ static struct rw_semaphore __sched * -rwsem_down_read_slowpath (struct rw_semaphore *sem,long count,int state) +rwsem_down_read_slowpath (struct rw_semaphore *sem,long count,unsigned int state) {long adjustment = -RWSEM_READER_BIAS ;long rcnt = (count >>RWSEM_READER_SHIFT );
kernel/ptrace.c +6 -6
@@ -197,7 +197,7 @@ static bool ptrace_freeze_traced(structspin_lock_irq (&task->sighand->siglock);if (task_is_traced (task) && !looks_like_a_spurious_pid (task) && !__fatal_signal_pending (task)) { - task->state = __TASK_TRACED; +WRITE_ONCE (task->__state, __TASK_TRACED); ret = true; }spin_unlock_irq (&task->sighand->siglock);
@@ -207,7 +207,7 @@ static bool ptrace_freeze_traced(structstatic void ptrace_unfreeze_traced (struct task_struct *task) { -if (task->state != __TASK_TRACED) +if (READ_ONCE (task->__state) != __TASK_TRACED)return ;WARN_ON (!task->ptrace || task->parent != current);
@@ -217,11 +217,11 @@ static void ptrace_unfreeze_traced(struc* Recheck state under the lock to close this race. */ spin_lock_irq (&task->sighand->siglock); -if (task->state == __TASK_TRACED) { +if (READ_ONCE (task->__state) == __TASK_TRACED) {if (__fatal_signal_pending (task))wake_up_state (task, __TASK_TRACED);else - task->state =TASK_TRACED ; +WRITE_ONCE (task->__state,TASK_TRACED ); }spin_unlock_irq (&task->sighand->siglock); }
@@ -256,7 +256,7 @@ static int ptrace_check_attach(struct ta*/ read_lock (&tasklist_lock);if (child->ptrace && child->parent == current) { -WARN_ON (child->state == __TASK_TRACED); +WARN_ON (READ_ONCE (child->__state) == __TASK_TRACED);/* * child->sighand can't be NULL, release_task() * does ptrace_unlink() before __exit_signal().
@@ -273,7 +273,7 @@ static int ptrace_check_attach(struct ta* ptrace_stop() changes ->state back to TASK_RUNNING, * so we should not worry about leaking __TASK_TRACED. */ -WARN_ON (child->state == __TASK_TRACED); +WARN_ON (READ_ONCE (child->__state) == __TASK_TRACED); ret = -ESRCH ; } }
kernel/rcu/rcutorture.c +2 -2
@@ -1831,10 +1831,10 @@ rcu_torture_stats_print(void)srcutorture_get_gp_data (cur_ops->ttype, srcu_ctlp, &flags, &gp_seq); wtp =READ_ONCE (writer_task); -pr_alert ("??? Writer stall state %s(%d) g%lu f%#x ->state %#lx cpu %d\n" , +pr_alert ("??? Writer stall state %s(%d) g%lu f%#x ->state %#x cpu %d\n" ,rcu_torture_writer_state_getname (), rcu_torture_writer_state, gp_seq, flags, - wtp ==NULL ? ~0UL : wtp->state, + wtp ==NULL ? ~0U : wtp->__state, wtp ==NULL ?-1 : (int )task_cpu (wtp));if (!splatted && wtp) {sched_show_task (wtp);
kernel/rcu/tree_stall.h +6 -6
@@ -460,12 +460,12 @@ static void rcu_check_gp_kthread_starvatif (rcu_is_gp_kthread_starving (&j)) { cpu = gpk ?task_cpu (gpk) :-1 ; -pr_err ("%s kthread starved for %ld jiffies! g%ld f%#x %s(%d) ->state=%#lx ->cpu=%d\n" , +pr_err ("%s kthread starved for %ld jiffies! g%ld f%#x %s(%d) ->state=%#x ->cpu=%d\n" , rcu_state.name, j, (long )rcu_seq_current (&rcu_state.gp_seq),data_race (rcu_state.gp_flags),gp_state_getname (rcu_state.gp_state), rcu_state.gp_state, - gpk ? gpk->state : ~0 , cpu); + gpk ? gpk->__state : ~0 , cpu);if (gpk) {pr_err ("\tUnless %s kthread gets sufficient CPU time, OOM is now expected behavior.\n" , rcu_state.name);pr_err ("RCU grace-period kthread stack dump:\n" );
@@ -503,12 +503,12 @@ static void rcu_check_gp_kthread_expired time_after(jiffies, jiffies_fqs + RCU_STALL_MIGHT_MIN) && gpk && !READ_ONCE(gpk->on_rq)) { cpu =task_cpu (gpk); -pr_err ("%s kthread timer wakeup didn't happen for %ld jiffies! g%ld f%#x %s(%d) ->state=%#lx\n" , +pr_err ("%s kthread timer wakeup didn't happen for %ld jiffies! g%ld f%#x %s(%d) ->state=%#x\n" , rcu_state.name, (jiffies - jiffies_fqs), (long )rcu_seq_current (&rcu_state.gp_seq),data_race (rcu_state.gp_flags),gp_state_getname (RCU_GP_WAIT_FQS ),RCU_GP_WAIT_FQS , - gpk->state); + gpk->__state);pr_err ("\tPossible timer handling issue on cpu=%d timer-softirq=%u\n" , cpu,kstat_softirqs_cpu (TIMER_SOFTIRQ , cpu)); }
@@ -735,9 +735,9 @@ void show_rcu_gp_kthreads(void) ja = j -data_race (rcu_state.gp_activity); jr = j -data_race (rcu_state.gp_req_activity); jw = j -data_race (rcu_state.gp_wake_time); - pr_info("%s: wait state: %s(%d) ->state: %#lx delta ->gp_activity %lu ->gp_req_activity %lu ->gp_wake_time %lu ->gp_wake_seq %ld ->gp_seq %ld ->gp_seq_needed %ld ->gp_flags %#x\n", + pr_info("%s: wait state: %s(%d) ->state: %#x delta ->gp_activity %lu ->gp_req_activity %lu ->gp_wake_time %lu ->gp_wake_seq %ld ->gp_seq %ld ->gp_seq_needed %ld ->gp_flags %#x\n", rcu_state.name, gp_state_getname(rcu_state.gp_state), - rcu_state.gp_state, t ? t->state : 0x1ffffL, + rcu_state.gp_state, t ? t->__state : 0x1ffff, ja, jr, jw, (long)data_race(rcu_state.gp_wake_seq), (long)data_race(rcu_state.gp_seq), (long)data_race(rcu_get_root()->gp_seq_needed),
kernel/sched/core.c +28 -25
200 lines not on this page · raw message
kernel/sched/deadline.c +5 -5
@@ -348,10 +348,10 @@ static void task_non_contending(struct tif ((zerolag_time <0 ) ||hrtimer_active (&dl_se->inactive_timer)) {if (dl_task (p))sub_running_bw (dl_se, dl_rq); -if (!dl_task (p) || p->state ==TASK_DEAD ) { +if (!dl_task (p) ||READ_ONCE (p->__state) ==TASK_DEAD ) {struct dl_bw *dl_b =dl_bw_of (task_cpu (p)); -if (p->state ==TASK_DEAD ) +if (READ_ONCE (p->__state) ==TASK_DEAD )sub_rq_bw (&p->dl, &rq->dl);raw_spin_lock (&dl_b->lock);__dl_sub (dl_b, p->dl.dl_bw,dl_bw_cpus (task_cpu (p)));
@@ -1355,10 +1355,10 @@ static enum hrtimer_restart inactive_tassched_clock_tick ();update_rq_clock (rq); -if (!dl_task (p) || p->state ==TASK_DEAD ) { +if (!dl_task (p) ||READ_ONCE (p->__state) ==TASK_DEAD ) {struct dl_bw *dl_b =dl_bw_of (task_cpu (p)); -if (p->state ==TASK_DEAD && dl_se->dl_non_contending) { +if (READ_ONCE (p->__state) ==TASK_DEAD && dl_se->dl_non_contending) {sub_running_bw (&p->dl,dl_rq_of_se (&p->dl));sub_rq_bw (&p->dl,dl_rq_of_se (&p->dl)); dl_se->dl_non_contending =0 ;
@@ -1722,7 +1722,7 @@ static void migrate_task_rq_dl(struct ta {struct rq *rq; -if (p->state !=TASK_WAKING ) +if (READ_ONCE (p->__state) !=TASK_WAKING )return ; rq =task_rq (p);
kernel/sched/fair.c +7 -4
@@ -993,11 +993,14 @@ update_stats_dequeue(struct cfs_rq *cfs_if ((flags &DEQUEUE_SLEEP ) &&entity_is_task (se)) {struct task_struct *tsk =task_of (se); +unsigned int state; -if (tsk->state &TASK_INTERRUPTIBLE ) +/* XXX racy against TTWU */ + state =READ_ONCE (tsk->__state); +if (state &TASK_INTERRUPTIBLE )__schedstat_set (se->statistics.sleep_start,rq_clock (rq_of (cfs_rq))); -if (tsk->state &TASK_UNINTERRUPTIBLE ) +if (state &TASK_UNINTERRUPTIBLE )__schedstat_set (se->statistics.block_start,rq_clock (rq_of (cfs_rq))); }
@@ -6830,7 +6833,7 @@ static void migrate_task_rq_fair(struct* min_vruntime -- the latter is done by enqueue_entity() when placing * the task on the new runqueue. */ -if (p->state ==TASK_WAKING ) { +if (READ_ONCE (p->__state) ==TASK_WAKING ) {struct sched_entity *se = &p->se;struct cfs_rq *cfs_rq =cfs_rq_of (se);u64 min_vruntime;
@@ -11012,7 +11015,7 @@ static inline bool vruntime_normalized(s* waiting for actually being woken up by sched_ttwu_pending(). */ if (!se->sum_exec_runtime || - (p->state ==TASK_WAKING && p->sched_remote_wakeup)) + (READ_ONCE (p->__state) ==TASK_WAKING && p->sched_remote_wakeup))return true;return false;
lib/syscall.c +2 -2
@@ -68,13 +68,13 @@ static int collect_syscall(struct task_s*/ int task_current_syscall (struct task_struct *target,struct syscall_info *info) { -long state;unsigned long ncsw; +unsigned int state;if (target == current)return collect_syscall (target, info); - state = target->state; + state =READ_ONCE (target->__state);if (unlikely (!state))return -EAGAIN ;
net/core/dev.c +1 -1
@@ -4363,7 +4363,7 @@ static inline void ____napi_schedule(str* makes sure to proceed with napi polling * if the thread is explicitly woken from here. */ -if (READ_ONCE (thread->state) !=TASK_INTERRUPTIBLE ) +if (READ_ONCE (thread->__state) !=TASK_INTERRUPTIBLE )set_bit (NAPI_STATE_SCHED_THREADED , &napi->state);wake_up_process (thread);return ;
Re: [PATCH v2 7/7] sched: Change task_struct::state
Quoted 29 lines · Peter Zijlstra
kernel/sched/deadline.c +4
-4@@ -348,10 +348,10 @@ static void task_non_contending(struct tif ((zerolag_time <0 ) ||hrtimer_active (&dl_se->inactive_timer)) {if (dl_task (p))sub_running_bw (dl_se, dl_rq); -if (!dl_task (p) || p->state ==TASK_DEAD ) { +if (!dl_task (p) ||READ_ONCE (p->__state) ==TASK_DEAD ) {struct dl_bw *dl_b =dl_bw_of (task_cpu (p)); -if (p->state ==TASK_DEAD ) +if (READ_ONCE (p->__state) ==TASK_DEAD )sub_rq_bw (&p->dl, &rq->dl);raw_spin_lock (&dl_b->lock);__dl_sub (dl_b, p->dl.dl_bw,dl_bw_cpus (task_cpu (p)));@@ -1355,10 +1355,10 @@ static enum hrtimer_restart inactive_tassched_clock_tick ();update_rq_clock (rq); -if (!dl_task (p) || p->state ==TASK_DEAD ) { +if (!dl_task (p) ||READ_ONCE (p->__state) ==TASK_DEAD ) {struct dl_bw *dl_b =dl_bw_of (task_cpu (p)); -if (p->state ==TASK_DEAD && dl_se->dl_non_contending) { +if (READ_ONCE (p->__state) ==TASK_DEAD && dl_se->dl_non_contending) {sub_running_bw (&p->dl,dl_rq_of_se (&p->dl));sub_rq_bw (&p->dl,dl_rq_of_se (&p->dl)); dl_se->dl_non_contending =0 ;
kernel/sched/deadline.c static void migrate_task_rq_dl(struct ta · line 1728
Quoted 8 lines
kernel/sched/deadline.c +1
-1@@ -1722,7 +1722,7 @@ static void migrate_task_rq_dl(struct ta {struct rq *rq; -if (p->state !=TASK_WAKING ) +if (READ_ONCE (p->__state) !=TASK_WAKING )return ; rq =task_rq (p);
Reviewed-by Daniel Bristot de Oliveira <[email protected]>
Feel free to add it to the other patches as well.
Thanks!
-- Daniel
[tip: sched/core] sched: Change task_struct::state
In reply to Peter ZijlstraThe following commit has been merged into the sched/core branch of tip:
The following commit has been merged into the sched/core branch of tip:
Commit-ID: 2f064a59a11ff9bc22e52e9678bc601404c7cb34 Gitweb: https://git.kernel.org/tip/2f064a59a11ff9bc22e52e9678bc601404c7cb34 Author: Peter Zijlstra <[email protected]> AuthorDate: Fri, 11 Jun 2021 10:28:17 +02:00 Committer: Peter Zijlstra <[email protected]> CommitterDate: Fri, 18 Jun 2021 11:43:09 +02:00
sched: Change task_struct::state
Change the type and name of task_struct::state. Drop the volatile and shrink it to an 'unsigned int'. Rename it in order to find all uses such that we can use READ_ONCE/WRITE_ONCE as appropriate.
Signed-off-by Peter Zijlstra (Intel) <[email protected]>
Reviewed-by Daniel Bristot de Oliveira <[email protected]>
Acked-by Will Deacon <[email protected]>
Acked-by Daniel Thompson <[email protected]>
--- arch/ia64/kernel/mca.c | 2 +- arch/ia64/kernel/ptrace.c | 8 ++--- arch/powerpc/xmon/xmon.c | 13 ++++---- block/blk-mq.c | 2 +- drivers/md/dm.c | 6 ++-- fs/binfmt_elf.c | 8 +++-- fs/binfmt_elf_fdpic.c | 4 +- fs/userfaultfd.c | 4 +- include/linux/sched.h | 31 +++++++++---------- include/linux/sched/debug.h | 2 +- include/linux/sched/signal.h | 2 +- init/init_task.c | 2 +- kernel/cgroup/cgroup-v1.c | 2 +- kernel/debug/kdb/kdb_support.c | 18 ++++++----- kernel/fork.c | 4 +- kernel/hung_task.c | 2 +- kernel/kthread.c | 4 +- kernel/locking/mutex.c | 6 ++-- kernel/locking/rtmutex.c | 4 +- kernel/locking/rwsem.c | 2 +- kernel/ptrace.c | 12 +++---- kernel/rcu/rcutorture.c | 4 +- kernel/rcu/tree_stall.h | 12 +++---- kernel/sched/core.c | 53 +++++++++++++++++---------------- kernel/sched/deadline.c | 10 +++--- kernel/sched/fair.c | 11 ++++--- lib/syscall.c | 4 +- net/core/dev.c | 2 +- 28 files changed, 123 insertions(+), 111 deletions(-)
arch/ia64/kernel/mca.c +1 -1
@@ -1788,7 +1788,7 @@ format_mca_init_stack(void *mca_data, unsigned long offset, ti->task = p; ti->cpu = cpu; p->stack = ti; - p->state =TASK_UNINTERRUPTIBLE ; + p->__state =TASK_UNINTERRUPTIBLE ;cpumask_set_cpu (cpu, &p->cpus_mask);INIT_LIST_HEAD (&p->tasks); p->parent = p->real_parent = p->group_leader = p;
arch/ia64/kernel/ptrace.c +4 -4
@@ -641,11 +641,11 @@ ptrace_attach_sync_user_rbs (struct task_struct *child)read_lock (&tasklist_lock);if (child->sighand) {spin_lock_irq (&child->sighand->siglock); -if (child->state ==TASK_STOPPED && +if (READ_ONCE (child->__state) ==TASK_STOPPED && !test_and_set_tsk_thread_flag (child,TIF_RESTORE_RSE )) {set_notify_resume (child); - child->state =TASK_TRACED ; +WRITE_ONCE (child->__state,TASK_TRACED ); stopped =1 ; }spin_unlock_irq (&child->sighand->siglock);
@@ -665,9 +665,9 @@ ptrace_attach_sync_user_rbs (struct task_struct *child)read_lock (&tasklist_lock);if (child->sighand) {spin_lock_irq (&child->sighand->siglock); -if (child->state ==TASK_TRACED && +if (READ_ONCE (child->__state) ==TASK_TRACED && (child->signal->flags &SIGNAL_STOP_STOPPED )) { - child->state =TASK_STOPPED ; +WRITE_ONCE (child->__state,TASK_STOPPED ); }spin_unlock_irq (&child->sighand->siglock); }
arch/powerpc/xmon/xmon.c +7 -6
@@ -3162,6 +3162,7 @@ memzcan(void)static void show_task (struct task_struct *tsk) { +unsigned int p_state =READ_ONCE (tsk->__state);char state;/*
@@ -3169,14 +3170,14 @@ static void show_task(struct task_struct *tsk)* appropriate for calling from xmon. This could be moved * to a common, generic, routine used by both. */ - state = (tsk->state ==0 ) ?'R' : - (tsk->state <0 ) ?'U' : - (tsk->state &TASK_UNINTERRUPTIBLE ) ?'D' : - (tsk->state &TASK_STOPPED ) ?'T' : - (tsk->state &TASK_TRACED ) ?'C' : + state = (p_state ==0 ) ?'R' : + (p_state <0 ) ?'U' : + (p_state &TASK_UNINTERRUPTIBLE ) ?'D' : + (p_state &TASK_STOPPED ) ?'T' : + (p_state &TASK_TRACED ) ?'C' : (tsk->exit_state &EXIT_ZOMBIE ) ?'Z' : (tsk->exit_state &EXIT_DEAD ) ?'E' : - (tsk->state &TASK_INTERRUPTIBLE ) ?'S' :'?' ; + (p_state &TASK_INTERRUPTIBLE ) ?'S' :'?' ; printf("%16px %16lx %16px %6d %6d %c %2d %s\n", tsk, tsk->thread.ksp, tsk->thread.regs,
block/blk-mq.c +1 -1
@@ -3886,7 +3886,7 @@ static bool blk_mq_poll_hybrid(struct request_queue *q,int blk_poll (struct request_queue *q,blk_qc_t cookie,bool spin) {struct blk_mq_hw_ctx *hctx; -long state; +unsigned int state;if (!blk_qc_t_valid (cookie) || !test_bit (QUEUE_FLAG_POLL , &q->queue_flags))
drivers/md/dm.c +3 -3
@@ -2328,7 +2328,7 @@ static bool md_in_flight_bios(struct mapped_device *md)return sum !=0 ; } -static int dm_wait_for_bios_completion (struct mapped_device *md,long task_state) +static int dm_wait_for_bios_completion (struct mapped_device *md,unsigned int task_state) {int r =0 ;DEFINE_WAIT (wait);
@@ -2351,7 +2351,7 @@ static int dm_wait_for_bios_completion(struct mapped_device *md, long task_statereturn r; } -static int dm_wait_for_completion (struct mapped_device *md,long task_state) +static int dm_wait_for_completion (struct mapped_device *md,unsigned int task_state) {int r =0 ;
@@ -2478,7 +2478,7 @@ static void unlock_fs(struct mapped_device *md)* are being added to md->deferred list. */ static int __dm_suspend (struct mapped_device *md,struct dm_table *map, -unsigned suspend_flags,long task_state, +unsigned suspend_flags,unsigned int task_state,int dmf_suspended_flag) {bool do_lockfs = suspend_flags &DM_SUSPEND_LOCKFS_FLAG ;
fs/binfmt_elf.c +5 -3
@@ -1537,7 +1537,8 @@ static int fill_psinfo(struct elf_prpsinfo *psinfo, struct task_struct *p, {const struct cred *cred;unsigned int i, len; - +unsigned int state; +/* first copy the parameters from user space */ memset (psinfo,0 ,sizeof (struct elf_prpsinfo ));
@@ -1559,7 +1560,8 @@ static int fill_psinfo(struct elf_prpsinfo *psinfo, struct task_struct *p, psinfo->pr_pgrp =task_pgrp_vnr (p); psinfo->pr_sid =task_session_vnr (p); - i = p->state ?ffz (~p->state) +1 :0 ; + state =READ_ONCE (p->__state); + i = state ?ffz (~state) +1 :0 ; psinfo->pr_state = i; psinfo->pr_sname = (i >5 ) ?'.' :"RSDTZW" [i]; psinfo->pr_zomb = psinfo->pr_sname =='Z' ;
@@ -1571,7 +1573,7 @@ static int fill_psinfo(struct elf_prpsinfo *psinfo, struct task_struct *p,SET_GID (psinfo->pr_gid,from_kgid_munged (cred->user_ns, cred->gid));rcu_read_unlock ();strncpy (psinfo->pr_fname, p->comm,sizeof (psinfo->pr_fname)); - +return 0 ; }
fs/binfmt_elf_fdpic.c +3 -1
@@ -1331,6 +1331,7 @@ static int fill_psinfo(struct elf_prpsinfo *psinfo, struct task_struct *p, {const struct cred *cred;unsigned int i, len; +unsigned int state;/* first copy the parameters from user space */ memset (psinfo,0 ,sizeof (struct elf_prpsinfo ));
@@ -1353,7 +1354,8 @@ static int fill_psinfo(struct elf_prpsinfo *psinfo, struct task_struct *p, psinfo->pr_pgrp =task_pgrp_vnr (p); psinfo->pr_sid =task_session_vnr (p); - i = p->state ?ffz (~p->state) +1 :0 ; + state =READ_ONCE (p->__state); + i = state ?ffz (~state) +1 :0 ; psinfo->pr_state = i; psinfo->pr_sname = (i >5 ) ?'.' :"RSDTZW" [i]; psinfo->pr_zomb = psinfo->pr_sname =='Z' ;
fs/userfaultfd.c +2 -2
@@ -337,7 +337,7 @@ out:return ret; } -static inline long userfaultfd_get_blocking_state (unsigned int flags) +static inline unsigned int userfaultfd_get_blocking_state (unsigned int flags) {if (flags &FAULT_FLAG_INTERRUPTIBLE )return TASK_INTERRUPTIBLE ;
@@ -370,7 +370,7 @@ vm_fault_t handle_userfault(struct vm_fault *vmf, unsigned long reason)struct userfaultfd_wait_queue uwq;vm_fault_t ret =VM_FAULT_SIGBUS ;bool must_wait; -long blocking_state; +unsigned int blocking_state;/* * We don't do userfault handling for the final child pid update.
include/linux/sched.h +15 -16
@@ -113,13 +113,13 @@ struct task_group; __TASK_TRACED | EXIT_DEAD | EXIT_ZOMBIE | \ TASK_PARKED) -#define task_is_running (task) (READ_ONCE((task)->state) == TASK_RUNNING) +#define task_is_running (task) (READ_ONCE((task)->__state) == TASK_RUNNING) -#define task_is_traced (task) ((task->state & __TASK_TRACED) != 0) +#define task_is_traced (task) ((READ_ONCE(task->__state) & __TASK_TRACED) != 0) -#define task_is_stopped (task) ((task->state & __TASK_STOPPED) != 0) +#define task_is_stopped (task) ((READ_ONCE(task->__state) & __TASK_STOPPED) != 0) -#define task_is_stopped_or_traced (task) ((task->state & (__TASK_STOPPED | __TASK_TRACED)) != 0) +#define task_is_stopped_or_traced (task) ((READ_ONCE(task->__state) & (__TASK_STOPPED | __TASK_TRACED)) != 0)#ifdef CONFIG_DEBUG_ATOMIC_SLEEP
@@ -134,14 +134,14 @@ struct task_group;do { \WARN_ON_ONCE (is_special_task_state (state_value));\ current->task_state_change = _THIS_IP_; \ - current->state = (state_value); \ +WRITE_ONCE (current->__state, (state_value)); \ } while (0)#define set_current_state (state_value) \ do { \ WARN_ON_ONCE(is_special_task_state(state_value));\ current->task_state_change = _THIS_IP_; \ - smp_store_mb(current->state, (state_value)); \ + smp_store_mb(current->__state, (state_value)); \ } while (0)#define set_special_state (state_value) \
@@ -150,7 +150,7 @@ struct task_group;WARN_ON_ONCE (!is_special_task_state (state_value)); \raw_spin_lock_irqsave (¤t->pi_lock, flags); \ current->task_state_change = _THIS_IP_; \ - current->state = (state_value); \ +WRITE_ONCE (current->__state, (state_value)); \raw_spin_unlock_irqrestore (¤t->pi_lock, flags); \ } while (0) #else
@@ -192,10 +192,10 @@ struct task_group;* Also see the comments of try_to_wake_up(). */ #define __set_current_state (state_value) \ - current->state = (state_value) + WRITE_ONCE(current->__state, (state_value))#define set_current_state (state_value) \ - smp_store_mb(current->state, (state_value)) + smp_store_mb(current->__state, (state_value))/* * set_special_state() should be used for those states when the blocking task
@@ -207,13 +207,13 @@ struct task_group;do { \unsigned long flags;/* may shadow */ \raw_spin_lock_irqsave (¤t->pi_lock, flags); \ - current->state = (state_value); \ +WRITE_ONCE (current->__state, (state_value)); \raw_spin_unlock_irqrestore (¤t->pi_lock, flags); \ } while (0)#endif -#define get_current_state () READ_ONCE(current->state) +#define get_current_state () READ_ONCE(current->__state)/* Task command name length: */ #define TASK_COMM_LEN 16
@@ -666,8 +666,7 @@ struct task_struct {*/ struct thread_info thread_info;#endif -/* -1 unrunnable, 0 runnable, >0 stopped: */ -volatile long state; +unsigned int __state;/* * This begins the randomizable portion of task_struct. Only
@@ -1532,7 +1531,7 @@ static inline pid_t task_pgrp_nr(struct task_struct *tsk)static inline unsigned int task_state_index (struct task_struct *tsk) { -unsigned int tsk_state =READ_ONCE (tsk->state); +unsigned int tsk_state =READ_ONCE (tsk->__state);unsigned int state = (tsk_state | tsk->exit_state) &TASK_REPORT ;BUILD_BUG_ON_NOT_POWER_OF_2 (TASK_REPORT_MAX );
@@ -1840,10 +1839,10 @@ static __always_inline void scheduler_ipi(void)*/ preempt_fold_need_resched(); } -extern unsigned long wait_task_inactive (struct task_struct *,long match_state); +extern unsigned long wait_task_inactive (struct task_struct *,unsigned int match_state);#else static inline void scheduler_ipi (void ) { } -static inline unsigned long wait_task_inactive (struct task_struct *p,long match_state) +static inline unsigned long wait_task_inactive (struct task_struct *p,unsigned int match_state) {return 1 ; }
include/linux/sched/debug.h +1 -1
@@ -14,7 +14,7 @@ extern void dump_cpu_task(int cpu);/* * Only dump TASK_* tasks. (0 for all tasks) */ -extern void show_state_filter (unsigned long state_filter); +extern void show_state_filter (unsigned int state_filter);static inline void show_state (void ) {
include/linux/sched/signal.h +1 -1
@@ -382,7 +382,7 @@ static inline int fatal_signal_pending(struct task_struct *p)return task_sigpending (p) &&__fatal_signal_pending (p); } -static inline int signal_pending_state (long state,struct task_struct *p) +static inline int signal_pending_state (unsigned int state,struct task_struct *p) {if (!(state & (TASK_INTERRUPTIBLE |TASK_WAKEKILL )))return 0 ;
init/init_task.c +1 -1
@@ -71,7 +71,7 @@ struct task_struct init_task .thread_info = INIT_THREAD_INFO(init_task), .stack_refcount = REFCOUNT_INIT(1), #endif - .state = 0, + .__state = 0, .stack = init_stack, .usage = REFCOUNT_INIT(2), .flags = PF_KTHREAD,
kernel/cgroup/cgroup-v1.c +1 -1
@@ -713,7 +713,7 @@ int cgroupstats_build(struct cgroupstats *stats, struct dentry *dentry)css_task_iter_start (&cgrp->self,0 , &it);while ((tsk =css_task_iter_next (&it))) { -switch (tsk->state) { +switch (READ_ONCE (tsk->__state)) {case TASK_RUNNING : stats->nr_running++;break ;
kernel/debug/kdb/kdb_support.c +10 -8
@@ -609,23 +609,25 @@ unsigned long kdb_task_state_string(const char *s)*/ char kdb_task_state_char (const struct task_struct *p) { -int cpu; -char state; +unsigned int p_state;unsigned long tmp; +char state; +int cpu;if (!p ||copy_from_kernel_nofault (&tmp, (char *)p,sizeof (unsigned long )))return 'E' ; cpu =kdb_process_cpu (p); - state = (p->state ==0 ) ?'R' : - (p->state <0 ) ?'U' : - (p->state &TASK_UNINTERRUPTIBLE ) ?'D' : - (p->state &TASK_STOPPED ) ?'T' : - (p->state &TASK_TRACED ) ?'C' : + p_state =READ_ONCE (p->__state); + state = (p_state ==0 ) ?'R' : + (p_state <0 ) ?'U' : + (p_state &TASK_UNINTERRUPTIBLE ) ?'D' : + (p_state &TASK_STOPPED ) ?'T' : + (p_state &TASK_TRACED ) ?'C' : (p->exit_state &EXIT_ZOMBIE ) ?'Z' : (p->exit_state &EXIT_DEAD ) ?'E' : - (p->state &TASK_INTERRUPTIBLE ) ?'S' :'?' ; + (p_state &TASK_INTERRUPTIBLE ) ?'S' :'?' ;if (is_idle_task (p)) {/* Idle task. Is it really idle, apart from the kdb * interrupt? */
kernel/fork.c +2 -2
@@ -425,7 +425,7 @@ static int memcg_charge_kernel_stack(struct task_struct *tsk)static void release_task_stack (struct task_struct *tsk) { -if (WARN_ON (tsk->state !=TASK_DEAD )) +if (WARN_ON (READ_ONCE (tsk->__state) !=TASK_DEAD ))return ;/* Better to leak the stack than to free prematurely */ account_kernel_stack (tsk,-1 );
@@ -2392,7 +2392,7 @@ bad_fork_cleanup_count:atomic_dec (&p->cred->user->processes);exit_creds (p); bad_fork_free: - p->state =TASK_DEAD ; +WRITE_ONCE (p->__state,TASK_DEAD );put_task_stack (p);delayed_free_task (p); fork_out:
kernel/hung_task.c +1 -1
@@ -196,7 +196,7 @@ static void check_hung_uninterruptible_tasks(unsigned long timeout) last_break = jiffies; }/* use "==" to skip the TASK_KILLABLE tasks waiting on NFS */ -if (t->state ==TASK_UNINTERRUPTIBLE ) +if (READ_ONCE (t->__state) ==TASK_UNINTERRUPTIBLE )check_hung_task (t, timeout); } unlock:
kernel/kthread.c +2 -2
@@ -457,7 +457,7 @@ struct task_struct *kthread_create_on_node(int (*threadfn)(void *data), }EXPORT_SYMBOL (kthread_create_on_node); -static void __kthread_bind_mask (struct task_struct *p,const struct cpumask *mask,long state) +static void __kthread_bind_mask (struct task_struct *p,const struct cpumask *mask,unsigned int state) {unsigned long flags;
@@ -473,7 +473,7 @@ static void __kthread_bind_mask(struct task_struct *p, const struct cpumask *masraw_spin_unlock_irqrestore (&p->pi_lock, flags); } -static void __kthread_bind (struct task_struct *p,unsigned int cpu,long state) +static void __kthread_bind (struct task_struct *p,unsigned int cpu,unsigned int state) {__kthread_bind_mask (p,cpumask_of (cpu), state); }
kernel/locking/mutex.c +3 -3
@@ -923,7 +923,7 @@ __ww_mutex_add_waiter(struct mutex_waiter *waiter,* Lock a mutex (possibly interruptible), slowpath: */ static __always_inline int __sched -__mutex_lock_common (struct mutex *lock,long state,unsigned int subclass, +__mutex_lock_common (struct mutex *lock,unsigned int state,unsigned int subclass,struct lockdep_map *nest_lock,unsigned long ip,struct ww_acquire_ctx *ww_ctx,const bool use_ww_ctx) {
@@ -1098,14 +1098,14 @@ err_early_kill: } static int __sched -__mutex_lock (struct mutex *lock,long state,unsigned int subclass, +__mutex_lock (struct mutex *lock,unsigned int state,unsigned int subclass,struct lockdep_map *nest_lock,unsigned long ip) {return __mutex_lock_common (lock, state, subclass, nest_lock, ip,NULL , false); } static int __sched -__ww_mutex_lock (struct mutex *lock,long state,unsigned int subclass, +__ww_mutex_lock (struct mutex *lock,unsigned int state,unsigned int subclass,struct lockdep_map *nest_lock,unsigned long ip,struct ww_acquire_ctx *ww_ctx) {
kernel/locking/rtmutex.c +2 -2
@@ -1135,7 +1135,7 @@ void __sched rt_mutex_init_waiter(struct rt_mutex_waiter *waiter)* * Must be called with lock->wait_lock held and interrupts disabled */ -static int __sched __rt_mutex_slowlock(struct rt_mutex *lock, int state, +static int __sched __rt_mutex_slowlock(struct rt_mutex *lock, unsigned int state,struct hrtimer_sleeper *timeout,struct rt_mutex_waiter *waiter) {
@@ -1190,7 +1190,7 @@ static void __sched rt_mutex_handle_deadlock(int res, int detect_deadlock,/* * Slow path lock function: */ -static int __sched rt_mutex_slowlock(struct rt_mutex *lock, int state, +static int __sched rt_mutex_slowlock(struct rt_mutex *lock, unsigned int state,struct hrtimer_sleeper *timeout,enum rtmutex_chainwalk chwalk) {
kernel/locking/rwsem.c +1 -1
@@ -889,7 +889,7 @@ rwsem_spin_on_owner(struct rw_semaphore *sem)* Wait for the read lock to be granted */ static struct rw_semaphore __sched * -rwsem_down_read_slowpath (struct rw_semaphore *sem,long count,int state) +rwsem_down_read_slowpath (struct rw_semaphore *sem,long count,unsigned int state) {long adjustment = -RWSEM_READER_BIAS ;long rcnt = (count >>RWSEM_READER_SHIFT );
kernel/ptrace.c +6 -6
@@ -197,7 +197,7 @@ static bool ptrace_freeze_traced(struct task_struct *task)spin_lock_irq (&task->sighand->siglock);if (task_is_traced (task) && !looks_like_a_spurious_pid (task) && !__fatal_signal_pending (task)) { - task->state = __TASK_TRACED; +WRITE_ONCE (task->__state, __TASK_TRACED); ret = true; }spin_unlock_irq (&task->sighand->siglock);
@@ -207,7 +207,7 @@ static bool ptrace_freeze_traced(struct task_struct *task)static void ptrace_unfreeze_traced (struct task_struct *task) { -if (task->state != __TASK_TRACED) +if (READ_ONCE (task->__state) != __TASK_TRACED)return ;WARN_ON (!task->ptrace || task->parent != current);
@@ -217,11 +217,11 @@ static void ptrace_unfreeze_traced(struct task_struct *task)* Recheck state under the lock to close this race. */ spin_lock_irq (&task->sighand->siglock); -if (task->state == __TASK_TRACED) { +if (READ_ONCE (task->__state) == __TASK_TRACED) {if (__fatal_signal_pending (task))wake_up_state (task, __TASK_TRACED);else - task->state =TASK_TRACED ; +WRITE_ONCE (task->__state,TASK_TRACED ); }spin_unlock_irq (&task->sighand->siglock); }
@@ -256,7 +256,7 @@ static int ptrace_check_attach(struct task_struct *child, bool ignore_state)*/ read_lock (&tasklist_lock);if (child->ptrace && child->parent == current) { -WARN_ON (child->state == __TASK_TRACED); +WARN_ON (READ_ONCE (child->__state) == __TASK_TRACED);/* * child->sighand can't be NULL, release_task() * does ptrace_unlink() before __exit_signal().
@@ -273,7 +273,7 @@ static int ptrace_check_attach(struct task_struct *child, bool ignore_state)* ptrace_stop() changes ->state back to TASK_RUNNING, * so we should not worry about leaking __TASK_TRACED. */ -WARN_ON (child->state == __TASK_TRACED); +WARN_ON (READ_ONCE (child->__state) == __TASK_TRACED); ret = -ESRCH ; } }
kernel/rcu/rcutorture.c +2 -2
@@ -1831,10 +1831,10 @@ rcu_torture_stats_print(void)srcutorture_get_gp_data (cur_ops->ttype, srcu_ctlp, &flags, &gp_seq); wtp =READ_ONCE (writer_task); -pr_alert ("??? Writer stall state %s(%d) g%lu f%#x ->state %#lx cpu %d\n" , +pr_alert ("??? Writer stall state %s(%d) g%lu f%#x ->state %#x cpu %d\n" ,rcu_torture_writer_state_getname (), rcu_torture_writer_state, gp_seq, flags, - wtp ==NULL ? ~0UL : wtp->state, + wtp ==NULL ? ~0U : wtp->__state, wtp ==NULL ?-1 : (int )task_cpu (wtp));if (!splatted && wtp) {sched_show_task (wtp);
kernel/rcu/tree_stall.h +6 -6
@@ -460,12 +460,12 @@ static void rcu_check_gp_kthread_starvation(void)if (rcu_is_gp_kthread_starving (&j)) { cpu = gpk ?task_cpu (gpk) :-1 ; -pr_err ("%s kthread starved for %ld jiffies! g%ld f%#x %s(%d) ->state=%#lx ->cpu=%d\n" , +pr_err ("%s kthread starved for %ld jiffies! g%ld f%#x %s(%d) ->state=%#x ->cpu=%d\n" , rcu_state.name, j, (long )rcu_seq_current (&rcu_state.gp_seq),data_race (rcu_state.gp_flags),gp_state_getname (rcu_state.gp_state), rcu_state.gp_state, - gpk ? gpk->state : ~0 , cpu); + gpk ? gpk->__state : ~0 , cpu);if (gpk) {pr_err ("\tUnless %s kthread gets sufficient CPU time, OOM is now expected behavior.\n" , rcu_state.name);pr_err ("RCU grace-period kthread stack dump:\n" );
@@ -503,12 +503,12 @@ static void rcu_check_gp_kthread_expired_fqs_timer(void) time_after(jiffies, jiffies_fqs + RCU_STALL_MIGHT_MIN) && gpk && !READ_ONCE(gpk->on_rq)) { cpu =task_cpu (gpk); -pr_err ("%s kthread timer wakeup didn't happen for %ld jiffies! g%ld f%#x %s(%d) ->state=%#lx\n" , +pr_err ("%s kthread timer wakeup didn't happen for %ld jiffies! g%ld f%#x %s(%d) ->state=%#x\n" , rcu_state.name, (jiffies - jiffies_fqs), (long )rcu_seq_current (&rcu_state.gp_seq),data_race (rcu_state.gp_flags),gp_state_getname (RCU_GP_WAIT_FQS ),RCU_GP_WAIT_FQS , - gpk->state); + gpk->__state);pr_err ("\tPossible timer handling issue on cpu=%d timer-softirq=%u\n" , cpu,kstat_softirqs_cpu (TIMER_SOFTIRQ , cpu)); }
@@ -735,9 +735,9 @@ void show_rcu_gp_kthreads(void) ja = j -data_race (rcu_state.gp_activity); jr = j -data_race (rcu_state.gp_req_activity); jw = j -data_race (rcu_state.gp_wake_time); - pr_info("%s: wait state: %s(%d) ->state: %#lx delta ->gp_activity %lu ->gp_req_activity %lu ->gp_wake_time %lu ->gp_wake_seq %ld ->gp_seq %ld ->gp_seq_needed %ld ->gp_flags %#x\n", + pr_info("%s: wait state: %s(%d) ->state: %#x delta ->gp_activity %lu ->gp_req_activity %lu ->gp_wake_time %lu ->gp_wake_seq %ld ->gp_seq %ld ->gp_seq_needed %ld ->gp_flags %#x\n", rcu_state.name, gp_state_getname(rcu_state.gp_state), - rcu_state.gp_state, t ? t->state : 0x1ffffL, + rcu_state.gp_state, t ? t->__state : 0x1ffff, ja, jr, jw, (long)data_race(rcu_state.gp_wake_seq), (long)data_race(rcu_state.gp_seq), (long)data_race(rcu_get_root()->gp_seq_needed),
kernel/sched/core.c +28 -25
200 lines not on this page · raw message
kernel/sched/deadline.c +5 -5
@@ -348,10 +348,10 @@ static void task_non_contending(struct task_struct *p)if ((zerolag_time <0 ) ||hrtimer_active (&dl_se->inactive_timer)) {if (dl_task (p))sub_running_bw (dl_se, dl_rq); -if (!dl_task (p) || p->state ==TASK_DEAD ) { +if (!dl_task (p) ||READ_ONCE (p->__state) ==TASK_DEAD ) {struct dl_bw *dl_b =dl_bw_of (task_cpu (p)); -if (p->state ==TASK_DEAD ) +if (READ_ONCE (p->__state) ==TASK_DEAD )sub_rq_bw (&p->dl, &rq->dl);raw_spin_lock (&dl_b->lock);__dl_sub (dl_b, p->dl.dl_bw,dl_bw_cpus (task_cpu (p)));
@@ -1355,10 +1355,10 @@ static enum hrtimer_restart inactive_task_timer(struct hrtimer *timer)sched_clock_tick ();update_rq_clock (rq); -if (!dl_task (p) || p->state ==TASK_DEAD ) { +if (!dl_task (p) ||READ_ONCE (p->__state) ==TASK_DEAD ) {struct dl_bw *dl_b =dl_bw_of (task_cpu (p)); -if (p->state ==TASK_DEAD && dl_se->dl_non_contending) { +if (READ_ONCE (p->__state) ==TASK_DEAD && dl_se->dl_non_contending) {sub_running_bw (&p->dl,dl_rq_of_se (&p->dl));sub_rq_bw (&p->dl,dl_rq_of_se (&p->dl)); dl_se->dl_non_contending =0 ;
@@ -1722,7 +1722,7 @@ static void migrate_task_rq_dl(struct task_struct *p, int new_cpu __maybe_unused {struct rq *rq; -if (p->state !=TASK_WAKING ) +if (READ_ONCE (p->__state) !=TASK_WAKING )return ; rq =task_rq (p);
kernel/sched/fair.c +7 -4
@@ -993,11 +993,14 @@ update_stats_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)if ((flags &DEQUEUE_SLEEP ) &&entity_is_task (se)) {struct task_struct *tsk =task_of (se); +unsigned int state; -if (tsk->state &TASK_INTERRUPTIBLE ) +/* XXX racy against TTWU */ + state =READ_ONCE (tsk->__state); +if (state &TASK_INTERRUPTIBLE )__schedstat_set (se->statistics.sleep_start,rq_clock (rq_of (cfs_rq))); -if (tsk->state &TASK_UNINTERRUPTIBLE ) +if (state &TASK_UNINTERRUPTIBLE )__schedstat_set (se->statistics.block_start,rq_clock (rq_of (cfs_rq))); }
@@ -6888,7 +6891,7 @@ static void migrate_task_rq_fair(struct task_struct *p, int new_cpu)* min_vruntime -- the latter is done by enqueue_entity() when placing * the task on the new runqueue. */ -if (p->state ==TASK_WAKING ) { +if (READ_ONCE (p->__state) ==TASK_WAKING ) {struct sched_entity *se = &p->se;struct cfs_rq *cfs_rq =cfs_rq_of (se);u64 min_vruntime;
@@ -11053,7 +11056,7 @@ static inline bool vruntime_normalized(struct task_struct *p)* waiting for actually being woken up by sched_ttwu_pending(). */ if (!se->sum_exec_runtime || - (p->state ==TASK_WAKING && p->sched_remote_wakeup)) + (READ_ONCE (p->__state) ==TASK_WAKING && p->sched_remote_wakeup))return true;return false;
lib/syscall.c +2 -2
@@ -68,13 +68,13 @@ static int collect_syscall(struct task_struct *target, struct syscall_info *info*/ int task_current_syscall (struct task_struct *target,struct syscall_info *info) { -long state;unsigned long ncsw; +unsigned int state;if (target == current)return collect_syscall (target, info); - state = target->state; + state =READ_ONCE (target->__state);if (unlikely (!state))return -EAGAIN ;
net/core/dev.c +1 -1
@@ -4363,7 +4363,7 @@ static inline void ____napi_schedule(struct softnet_data *sd,* makes sure to proceed with napi polling * if the thread is explicitly woken from here. */ -if (READ_ONCE (thread->state) !=TASK_INTERRUPTIBLE ) +if (READ_ONCE (thread->__state) !=TASK_INTERRUPTIBLE )set_bit (NAPI_STATE_SCHED_THREADED , &napi->state);wake_up_process (thread);return ;