net: ip4_datagram_connect: Use correct form of statistics update
Subsystems (4)
A message that is not in the archive (sent off-list or not yet received): [email protected]
PATCH 1/6 net: ip4_datagram_connect: Use correct form of statistics update
ip4_datagram_connect is called with BH processing enabled. Therefore we cannot use IP_INC_STATS_BH but must use IP_INC_STATS which disables BH handling before incrementing the counter.
The following trace is triggered without this patch:
[ 9293.806634] __this_cpu_add operation in preemptible [00000000] code: ntpd/2150 [ 9293.813894] caller is __this_cpu_preempt_check+0x38/0x60 [ 9293.813896] CPU: 2 PID: 2150 Comm: ntpd Tainted: GF 3.12.0-rc4+ #178 [ 9293.813897] Hardware name: Hewlett-Packard HP Z230 SFF Workstation/1906, BIOS L51 v01.10 07/05/2013 [ 9293.813898] 0000000000000002 ffff8803f33fddc8 ffffffff816d57c4 ffff8803f33fdfd8 [ 9293.813903] ffff8803f33fddf8 ffffffff8137341c ffff8800366b0380 00000000ffffff9b [ 9293.813905] ffff8800366b0680 ffffffffffffff9b ffff8803f33fde38 ffffffff81373478 [ 9293.813916] Call Trace: [ 9293.813920] [<ffffffff816d57c4>] dump_stack+0x4e/0x82 [ 9293.813921] [<ffffffff8137341c>] check_preemption_disabled+0xec/0x110 [ 9293.813923] [<ffffffff81373478>] __this_cpu_preempt_check+0x38/0x60 [ 9293.813926] [<ffffffff8163cd5d>] ip4_datagram_connect+0x2cd/0x2f0 [ 9293.813929] [<ffffffff8164b73e>] inet_dgram_connect+0x2e/0x80 [ 9293.813934] [<ffffffff815c766b>] SYSC_connect+0xdb/0x100 [ 9293.813938] [<ffffffff81006e6f>] ? mask_8259A+0x2f/0x30 [ 9293.813940] [<ffffffff81010505>] ? syscall_trace_enter+0x155/0x270 [ 9293.813942] [<ffffffff815c7b2e>] SyS_connect+0xe/0x10 [ 9293.813944] [<ffffffff816e4d13>] tracesys+0xe1/0xe6
net/ipv4/datagram.c +1 -1
@@ -57,7 +57,7 @@ int ip4_datagram_connect(struct sock *skif (IS_ERR (rt)) { err =PTR_ERR (rt);if (err == -ENETUNREACH ) -IP_INC_STATS_BH (sock_net (sk),IPSTATS_MIB_OUTNOROUTES ); +IP_INC_STATS (sock_net (sk),IPSTATS_MIB_OUTNOROUTES ); goto out; }
Quoted 5 lines · Christoph Lameter
ip4_datagram_connectis called with BH processing enabled. Therefore we cannot useIP_INC_STATS_BHbut must useIP_INC_STATSwhich disables BH handling before incrementing the counter.The following trace is triggered without this patch:
...
Quoted 3 lines
Cc Eric Dumazet <[email protected]>Signed-off-by Christoph Lameter <[email protected]>
Patch is fine but was not CC to netdev and David Miller.
PATCH 2/6 percpu: Add raw_cpu_ops
In reply to a message not in the archiveThe following patches will add preemption checks to __this_cpu ops so we need to have an alternative way to use this cpu operations without preemption checks.
raw_cpu ops rely on the __this_cpu_xxx_1/2/4/8 operations having no preemption checks. We therefore do not have to duplicate these functions but can affort to only duplicate the higher level macros.
Also add raw_cpu_ptr for symmetries sake.
include/linux/percpu.h +74 -11
@@ -139,6 +139,9 @@ extern int __init pcpu_page_first_chunk( pcpu_fc_populate_pte_fn_t populate_pte_fn);#endif +/* For symmetries sake. */ +#define raw_cpu_ptr __this_cpu_ptr +/* * Use this to get to a cpu's version of the per-cpu object * dynamically allocated. Non-atomic access to the current CPU's
@@ -520,17 +523,7 @@ do { \/* * Generic percpu operations for context that are safe from preemption/interrupts. -* Either we do not care about races or the caller has the -* responsibility of handling preemption/interrupt issues. Arch code can still -* override these instructions since the arch per cpu code may be more -* efficient and may actually get race freeness for free (that is the -* case for x86 for example). -* -* If there is no other protection through preempt disable and/or -* disabling interupts then one of these RMW operations can show unexpected -* behavior because the execution thread was rescheduled on another processor -* or an interrupt occurred and the same percpu variable was modified from -* the interrupt context. +* These functions verify that preemption has been disabled. */ #ifndef __this_cpu_read# ifndef __this_cpu_read_1
@@ -755,4 +748,74 @@ do { \ __pcpu_double_call_return_bool(__this_cpu_cmpxchg_double_, (pcp1), (pcp2), (oval1), (oval2), (nval1), (nval2))#endif +/* +* Generic percpu operations for context where we do not want to do +* any checks for preemptiosn. +* +* If there is no other protection through preempt disable and/or +* disabling interupts then one of these RMW operations can show unexpected +* behavior because the execution thread was rescheduled on another processor +* or an interrupt occurred and the same percpu variable was modified from +* the interrupt context. +*/ +#ifndef raw_cpu_read +# define raw_cpu_read (pcp) __pcpu_size_call_return(__this_cpu_read_, (pcp)) +#endif + +#ifndef raw_cpu_write +# define raw_cpu_write (pcp, val) __pcpu_size_call(__this_cpu_write_, (pcp), (val)) +#endif + +#ifndef raw_cpu_add +# define raw_cpu_add (pcp, val) __pcpu_size_call(__this_cpu_add_, (pcp), (val)) +#endif + +#ifndef raw_cpu_sub +# define raw_cpu_sub (pcp, val) raw_cpu_add((pcp), -(val)) +#endif + +#ifndef raw_cpu_inc +# define raw_cpu_inc (pcp) raw_cpu_add((pcp), 1) +#endif + +#ifndef raw_cpu_dec +# define raw_cpu_dec (pcp) raw_cpu_sub((pcp), 1) +#endif + +#ifndef raw_cpu_and +# define raw_cpu_and (pcp, val) __pcpu_size_call(__this_cpu_and_, (pcp), (val)) +#endif + +#ifndef raw_cpu_or +# define raw_cpu_or (pcp, val) __pcpu_size_call(__this_cpu_or_, (pcp), (val)) +#endif + +#ifndef raw_cpu_xor +# define raw_cpu_xor (pcp, val) __pcpu_size_call(__this_cpu_xor_, (pcp), (val)) +#endif + +#ifndef raw_cpu_add_return +# define raw_cpu_add_return (pcp, val) \ + __pcpu_size_call_return2(__this_cpu_add_return_, pcp, val) +#endif + +#define raw_cpu_sub_return (pcp, val) raw_cpu_add_return(pcp, -(val)) +#define raw_cpu_inc_return (pcp) raw_cpu_add_return(pcp, 1) +#define raw_cpu_dec_return (pcp) raw_cpu_add_return(pcp, -1) + +#ifndef raw_cpu_xchg +# define raw_cpu_xchg (pcp, nval) \ + __pcpu_size_call_return2(__this_cpu_xchg_, (pcp), nval) +#endif + +#ifndef raw_cpu_cmpxchg +# define raw_cpu_cmpxchg (pcp, oval, nval) \ + __pcpu_size_call_return2(__this_cpu_cmpxchg_, pcp, oval, nval) +#endif + +#ifndef raw_cpu_cmpxchg_double +# define raw_cpu_cmpxchg_double (pcp1, pcp2, oval1, oval2, nval1, nval2) \ + __pcpu_double_call_return_bool(__this_cpu_cmpxchg_double_, (pcp1), (pcp2), (oval1), (oval2), (nval1), (nval2)) +#endif +#endif /* __LINUX_PERCPU_H */
PATCH 3/6 mm: Use raw_cpu ops for determining current NUMA node
In reply to a message not in the archiveWith the preempt checking logic for __this_cpu_ops we will get false positives from locations in the code that use numa_node_id.
Before the __this_cpu ops where introduced there were no checks for preemption present either. smp_raw_processor_id() was used. See http://www.spinics.net/lists/linux-numa/msg00641.html
Therefore we need to use raw_cpu_read here to avoid false postives.
Note that this issue has been discussed in prior years.
If the process changes nodes after retrieving the current numa node then that is acceptable since most uses of numa_node etc are for optimization and not for correctness.
Some sample traces:
__this_cpu_read operation in preemptible [00000000] code: login/1456 caller is __this_cpu_preempt_check+0x2b/0x2d
CPU: 0 PID: 1456 Comm: login Not tainted 3.12.0-rc4-cl-00062-g2fe80d3-dirty #185 Hardware name: Bochs Bochs, BIOS Bochs 01/01/2011 000000000000013c ffff88001f31ba58 ffffffff8147cf5e ffff88001f31bfd8 ffff88001f31ba88 ffffffff8127eea9 0000000000000000 ffff88001f3975c0 00000000f7707000 ffff88001f3975c0 ffff88001f31bac0 ffffffff8127eeef Call Trace: [<ffffffff8147cf5e>] dump_stack+0x4e/0x82 [<ffffffff8127eea9>] check_preemption_disabled+0xc5/0xe0 [<ffffffff8127eeef>] __this_cpu_preempt_check+0x2b/0x2d [<ffffffff81030ff5>] ? show_stack+0x3b/0x3d [<ffffffff810ebee3>] get_task_policy+0x1d/0x49 [<ffffffff810ed705>] get_vma_policy+0x14/0x76 [<ffffffff810ed8ff>] alloc_pages_vma+0x35/0xff [<ffffffff810dad97>] handle_mm_fault+0x290/0x73b [<ffffffff810503da>] __do_page_fault+0x3fe/0x44d [<ffffffff8109b360>] ? trace_hardirqs_on_caller+0x142/0x19e [<ffffffff8109b3c9>] ? trace_hardirqs_on+0xd/0xf [<ffffffff81278bed>] ? trace_hardirqs_off_thunk+0x3a/0x3c [<ffffffff810be97f>] ? find_get_pages_contig+0x18e/0x18e [<ffffffff810be97f>] ? find_get_pages_contig+0x18e/0x18e [<ffffffff81050451>] do_page_fault+0x9/0xc [<ffffffff81483602>] page_fault+0x22/0x30 [<ffffffff810be97f>] ? find_get_pages_contig+0x18e/0x18e [<ffffffff810be97f>] ? find_get_pages_contig+0x18e/0x18e [<ffffffff810be4c3>] ? file_read_actor+0x3a/0x15a [<ffffffff810be97f>] ? find_get_pages_contig+0x18e/0x18e [<ffffffff810bffab>] generic_file_aio_read+0x38e/0x624 [<ffffffff810f6d69>] do_sync_read+0x54/0x73 [<ffffffff810f7890>] vfs_read+0x9d/0x12a [<ffffffff810f7a59>] SyS_read+0x47/0x7e [<ffffffff81484f21>] cstar_dispatch+0x7/0x23
caller is __this_cpu_preempt_check+0x2b/0x2d
CPU: 0 PID: 1456 Comm: login Not tainted 3.12.0-rc4-cl-00062-g2fe80d3-dirty #185 Hardware name: Bochs Bochs, BIOS Bochs 01/01/2011 00000000000000e8 ffff88001f31bbf8 ffffffff8147cf5e ffff88001f31bfd8 ffff88001f31bc28 ffffffff8127eea9 ffffffff823c5c40 00000000000213da 0000000000000000 0000000000000000 ffff88001f31bc60 ffffffff8127eeef Call Trace: [<ffffffff8147cf5e>] dump_stack+0x4e/0x82 [<ffffffff8127eea9>] check_preemption_disabled+0xc5/0xe0 [<ffffffff8127eeef>] __this_cpu_preempt_check+0x2b/0x2d [<ffffffff810e006e>] ? install_special_mapping+0x11/0xe4 [<ffffffff810ec8a8>] alloc_pages_current+0x8f/0xbc [<ffffffff810bec6b>] __page_cache_alloc+0xb/0xd [<ffffffff810c7e90>] __do_page_cache_readahead+0xf4/0x219 [<ffffffff810c7e0e>] ? __do_page_cache_readahead+0x72/0x219 [<ffffffff810c827c>] ra_submit+0x1c/0x20 [<ffffffff810c850c>] ondemand_readahead+0x28c/0x2b4 [<ffffffff810c85e9>] page_cache_sync_readahead+0x38/0x3a [<ffffffff810bfe7e>] generic_file_aio_read+0x261/0x624 [<ffffffff810f6d69>] do_sync_read+0x54/0x73 [<ffffffff810f7890>] vfs_read+0x9d/0x12a [<ffffffff810f7a59>] SyS_read+0x47/0x7e [<ffffffff81484f21>] cstar_dispatch+0x7/0x23
include/linux/topology.h +2 -2
@@ -182,7 +182,7 @@ DECLARE_PER_CPU(int, numa_node);/* Returns the number of the current Node. */ static inline int numa_node_id (void ) { -return __this_cpu_read (numa_node); +return raw_cpu_read (numa_node); }#endif
@@ -239,7 +239,7 @@ static inline void set_numa_mem(int node/* Returns the number of the nearest Node with memory */ static inline int numa_mem_id (void ) { -return __this_cpu_read (_numa_mem_); +return raw_cpu_read (_numa_mem_); }#endif
PATCH 4/6 Use raw_cpu_write for initialization of per cpu refcount.
In reply to a message not in the archiveThe initialization of a structure is not subject to synchronization. The use of __this_cpu would trigger a false positive with the additional preemption checks for __this_cpu ops.
So simply disable the check through the use of raw_cpu ops.
Trace:
[ 0.668066] __this_cpu_write operation in preemptible [00000000] code: modprobe/286 [ 0.668108] caller is __this_cpu_preempt_check+0x38/0x60 [ 0.668111] CPU: 3 PID: 286 Comm: modprobe Tainted: GF 3.12.0-rc4+ #187 [ 0.668112] Hardware name: FUJITSU CELSIUS W530 Power/D3227-A1, BIOS V4.6.5.4 R1.10.0 for D3227-A1x 09/16/2013 [ 0.668113] 0000000000000003 ffff8807edda1d18 ffffffff816d5a57 ffff8807edda1fd8 [ 0.668117] ffff8807edda1d48 ffffffff8137359c ffff8807edda1ef8 ffffffffa0002178 [ 0.668121] ffffc90000067730 ffff8807edda1e48 ffff8807edda1d88 ffffffff813735f8 [ 0.668124] Call Trace: [ 0.668129] [<ffffffff816d5a57>] dump_stack+0x4e/0x82 [ 0.668132] [<ffffffff8137359c>] check_preemption_disabled+0xec/0x110 [ 0.668135] [<ffffffff813735f8>] __this_cpu_preempt_check+0x38/0x60 [ 0.668139] [<ffffffff810c24fd>] load_module+0xcfd/0x2650 [ 0.668143] [<ffffffff816dd922>] ? page_fault+0x22/0x30 [ 0.668146] [<ffffffff810c3ef6>] SyS_init_module+0xa6/0xd0 [ 0.668150] [<ffffffff816e4fd3>] tracesys+0xe1/0xe6
kernel/module.c +1 -1
@@ -643,7 +643,7 @@ static int module_unload_init(struct modINIT_LIST_HEAD (&mod->target_list);/* Hold reference count during initialization. */ -__this_cpu_write (mod->refptr->incs,1 ); +raw_cpu_write (mod->refptr->incs,1 );/* Backwards compatibility macros put refcount during init. */ mod->waiter = current;
PATCH 5/6 net: __this_cpu_inc in route.c
In reply to a message not in the archiveThe RT_CACHE_STAT_INC macro triggers the new preemption checks for __this_cpu ops
I do not see any other synchronization that would allow the use of a __this_cpu operation here however in commit dbd2915ce87e811165da0717f8e159276ebb803e Andrew justifies the use of raw_smp_processor_id() here because "we do not care" about races.
So lets use raw_cpu ops here and hope for the best. The use of __this_cpu op improves the situation already from what commit dbd2915ce87e811165da0717f8e159276ebb803e did since the single instruction emitted on x86 does not allow the race to occur anymore. However, non x86 platforms could still experience a race here.
Trace:
[ 1277.189084] __this_cpu_add operation in preemptible [00000000] code: avahi-daemon/1193 [ 1277.189085] caller is __this_cpu_preempt_check+0x38/0x60 [ 1277.189086] CPU: 1 PID: 1193 Comm: avahi-daemon Tainted: GF 3.12.0-rc4+ #187 [ 1277.189087] Hardware name: FUJITSU CELSIUS W530 Power/D3227-A1, BIOS V4.6.5.4 R1.10.0 for D3227-A1x 09/16/2013 [ 1277.189088] 0000000000000001 ffff8807ef78fa00 ffffffff816d5a57 ffff8807ef78ffd8 [ 1277.189089] ffff8807ef78fa30 ffffffff8137359c ffff8807ef78fba0 ffff88079f822b40 [ 1277.189091] 0000000020000000 ffff8807ee32c800 ffff8807ef78fa70 ffffffff813735f8 [ 1277.189093] Call Trace: [ 1277.189094] [<ffffffff816d5a57>] dump_stack+0x4e/0x82 [ 1277.189096] [<ffffffff8137359c>] check_preemption_disabled+0xec/0x110 [ 1277.189097] [<ffffffff813735f8>] __this_cpu_preempt_check+0x38/0x60 [ 1277.189098] [<ffffffff81610d65>] __ip_route_output_key+0x575/0x8c0 [ 1277.189100] [<ffffffff816110d7>] ip_route_output_flow+0x27/0x70 [ 1277.189101] [<ffffffff81616c80>] ? ip_copy_metadata+0x1a0/0x1a0 [ 1277.189102] [<ffffffff81640b15>] udp_sendmsg+0x825/0xa20 [ 1277.189104] [<ffffffff811b4aa9>] ? do_sys_poll+0x449/0x5d0 [ 1277.189105] [<ffffffff8164c695>] inet_sendmsg+0x85/0xc0 [ 1277.189106] [<ffffffff815c6e3c>] sock_sendmsg+0x9c/0xd0 [ 1277.189108] [<ffffffff813735f8>] ? __this_cpu_preempt_check+0x38/0x60 [ 1277.189109] [<ffffffff815c7550>] ? move_addr_to_kernel+0x40/0xa0 [ 1277.189111] [<ffffffff815c71ec>] ___sys_sendmsg+0x37c/0x390 [ 1277.189112] [<ffffffff8136613a>] ? string.isra.3+0x3a/0xd0 [ 1277.189113] [<ffffffff8136613a>] ? string.isra.3+0x3a/0xd0 [ 1277.189115] [<ffffffff81367b54>] ? vsnprintf+0x364/0x650 [ 1277.189116] [<ffffffff81367ee9>] ? snprintf+0x39/0x40 [ 1277.189118] [<ffffffff813735f8>] ? __this_cpu_preempt_check+0x38/0x60 [ 1277.189119] [<ffffffff815c7ff9>] __sys_sendmsg+0x49/0x90 [ 1277.189121] [<ffffffff815c8052>] SyS_sendmsg+0x12/0x20 [ 1277.189122] [<ffffffff816e4fd3>] tracesys+0xe1/0xe6
net/ipv4/route.c +1 -1
@@ -197,7 +197,7 @@ const __u8 ip_tos2prio[16] = {EXPORT_SYMBOL (ip_tos2prio); static DEFINE_PER_CPU(struct rt_cache_stat, rt_cache_stat); -#define RT_CACHE_STAT_INC (field) __this_cpu_inc(rt_cache_stat.field) +#define RT_CACHE_STAT_INC (field) raw_cpu_inc(rt_cache_stat.field)#ifdef CONFIG_PROC_FS static void *rt_cache_seq_start (struct seq_file *seq,loff_t *pos)
PATCH 6/6 percpu: Add preemption checks to __this_cpu ops
In reply to a message not in the archiveV3->V4:
- Drop CONFIG_DEBUG_THIS_CPU_OPERATIONS
- Add support for logging the exact operation that caused the issue.
We define a check function in order to avoid trouble with the include files. Then the higher level __this_cpu macros are modified to invoke the check before __this_cpu operation
include/linux/percpu.h +42 -10
@@ -175,6 +175,12 @@ extern phys_addr_t per_cpu_ptr_to_phys(vextern void __bad_size_call_parameter (void ); +#ifdef CONFIG_DEBUG_PREEMPT +extern void __this_cpu_preempt_check (const char *op); +#else +static inline void __this_cpu_preempt_check (const char *op) { } +#endif +#define __pcpu_size_call_return (stem, variable) \ ({ typeof(variable) pscr_ret__; \ __verify_pcpu_ptr(&(variable)); \
@@ -538,7 +544,8 @@ do { \# ifndef __this_cpu_read_8# define __this_cpu_read_8 (pcp) (*__this_cpu_ptr(&(pcp)))# endif -# define __this_cpu_read (pcp) __pcpu_size_call_return(__this_cpu_read_, (pcp)) +# define __this_cpu_read (pcp) \ + (__this_cpu_preempt_check("read"),__pcpu_size_call_return(__this_cpu_read_, (pcp)))#endif #define __this_cpu_generic_to_op (pcp, val, op) \
@@ -559,7 +566,12 @@ do { \# ifndef __this_cpu_write_8# define __this_cpu_write_8 (pcp, val) __this_cpu_generic_to_op((pcp), (val), =)# endif -# define __this_cpu_write (pcp, val) __pcpu_size_call(__this_cpu_write_, (pcp), (val)) + +# define __this_cpu_write (pcp, val) \ +do { __this_cpu_preempt_check("write"); \ + __pcpu_size_call(__this_cpu_write_, (pcp), (val)); \ +} while (0) +#endif #ifndef __this_cpu_add
@@ -575,7 +587,12 @@ do { \# ifndef __this_cpu_add_8# define __this_cpu_add_8 (pcp, val) __this_cpu_generic_to_op((pcp), (val), +=)# endif -# define __this_cpu_add (pcp, val) __pcpu_size_call(__this_cpu_add_, (pcp), (val)) + +# define __this_cpu_add (pcp, val) \ +do { __this_cpu_preempt_check("add"); \ + __pcpu_size_call(__this_cpu_add_, (pcp), (val)); \ +} while (0) +#endif #ifndef __this_cpu_sub
@@ -603,7 +620,12 @@ do { \# ifndef __this_cpu_and_8# define __this_cpu_and_8 (pcp, val) __this_cpu_generic_to_op((pcp), (val), &=)# endif -# define __this_cpu_and (pcp, val) __pcpu_size_call(__this_cpu_and_, (pcp), (val)) + +# define __this_cpu_and (pcp, val) \ +do { __this_cpu_preempt_check("and"); \ + __pcpu_size_call(__this_cpu_and_, (pcp), (val)); \ +} while (0) +#endif #ifndef __this_cpu_or
@@ -619,7 +641,12 @@ do { \# ifndef __this_cpu_or_8# define __this_cpu_or_8 (pcp, val) __this_cpu_generic_to_op((pcp), (val), |=)# endif -# define __this_cpu_or (pcp, val) __pcpu_size_call(__this_cpu_or_, (pcp), (val)) + +# define __this_cpu_or (pcp, val) \ +do { __this_cpu_preempt_check("or"); \ + __pcpu_size_call(__this_cpu_or_, (pcp), (val)); \ +} while (0) +#endif #ifndef __this_cpu_xor
@@ -635,7 +662,12 @@ do { \# ifndef __this_cpu_xor_8# define __this_cpu_xor_8 (pcp, val) __this_cpu_generic_to_op((pcp), (val), ^=)# endif -# define __this_cpu_xor (pcp, val) __pcpu_size_call(__this_cpu_xor_, (pcp), (val)) + +# define __this_cpu_xor (pcp, val) \ +do { __this_cpu_preempt_check("xor"); \ + __pcpu_size_call(__this_cpu_xor_, (pcp), (val)); \ +} while (0) +#endif #define __this_cpu_generic_add_return (pcp, val) \
@@ -658,7 +690,7 @@ do { \# define __this_cpu_add_return_8 (pcp, val) __this_cpu_generic_add_return(pcp, val)# endif # define __this_cpu_add_return (pcp, val) \ - __pcpu_size_call_return2(__this_cpu_add_return_, pcp, val) + (__this_cpu_preempt_check("add_return"),__pcpu_size_call_return2(__this_cpu_add_return_, pcp, val))#endif #define __this_cpu_sub_return (pcp, val) __this_cpu_add_return(pcp, -(val))
@@ -686,7 +718,7 @@ do { \# define __this_cpu_xchg_8 (pcp, nval) __this_cpu_generic_xchg(pcp, nval)# endif # define __this_cpu_xchg (pcp, nval) \ - __pcpu_size_call_return2(__this_cpu_xchg_, (pcp), nval) + (__this_cpu_preempt_check("xchg"),__pcpu_size_call_return2(__this_cpu_xchg_, (pcp), nval))#endif #define __this_cpu_generic_cmpxchg (pcp, oval, nval) \
@@ -712,7 +744,7 @@ do { \# define __this_cpu_cmpxchg_8 (pcp, oval, nval) __this_cpu_generic_cmpxchg(pcp, oval, nval)# endif # define __this_cpu_cmpxchg (pcp, oval, nval) \ - __pcpu_size_call_return2(__this_cpu_cmpxchg_, pcp, oval, nval) + (__this_cpu_preempt_check("cmpxchg"),__pcpu_size_call_return2(__this_cpu_cmpxchg_, pcp, oval, nval))#endif #define __this_cpu_generic_cmpxchg_double (pcp1, pcp2, oval1, oval2, nval1, nval2) \
@@ -745,7 +777,7 @@ do { \ __this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)# endif # define __this_cpu_cmpxchg_double (pcp1, pcp2, oval1, oval2, nval1, nval2) \ - __pcpu_double_call_return_bool(__this_cpu_cmpxchg_double_, (pcp1), (pcp2), (oval1), (oval2), (nval1), (nval2)) + (__this_cpu_preempt_check("cmpxchg_double"),__pcpu_double_call_return_bool(__this_cpu_cmpxchg_double_, (pcp1), (pcp2), (oval1), (oval2), (nval1), (nval2)))#endif /*
lib/smp_processor_id.c +17 -4
@@ -7,7 +7,7 @@#include <linux/kallsyms.h> #include <linux/sched.h> -notrace unsigned int debug_smp_processor_id(void) +notrace static unsigned int check_preemption_disabled(char *what) {unsigned long preempt_count =preempt_count ();int this_cpu =raw_smp_processor_id ();
@@ -39,9 +39,9 @@ notrace unsigned int debug_smp_processorif (!printk_ratelimit ()) goto out_enable; -printk (KERN_ERR "BUG: using smp_processor_id() in preemptible [%08x] " -"code: %s/%d\n" , -preempt_count () -1 , current->comm, current->pid); +printk (KERN_ERR "%s in preemptible [%08x] code: %s/%d\n" , + what,preempt_count () -1 , current->comm, current->pid); +print_symbol ("caller is %s\n" , (long )__builtin_return_address (0 ));dump_stack ();
@@ -51,5 +51,18 @@ out:return this_cpu; } +notrace unsigned int debug_smp_processor_id(void) +{ +return check_preemption_disabled ("BUG: using smp_processor_id()" ); +}EXPORT_SYMBOL (debug_smp_processor_id); +notrace void __this_cpu_preempt_check(const char *op) +{ +char text[40 ]; + +snprintf (text,sizeof (text),"__this_cpu_%s operation" , op); +check_preemption_disabled (text); +} +EXPORT_SYMBOL (__this_cpu_preempt_check); +