rcu/tree: Add a work to allocate pages from regular context
Subsystems (14)
Uladzislau Rezki (Sony)
maintainerPATCH 1/16 rcu/tree: Add a work to allocate pages from regular context
The current memmory-allocation interface presents to following difficulties that this patch is designed to overcome:
a) If built with CONFIG_PROVE_RAW_LOCK_NESTING, the lockdep will complain about violation("BUG: Invalid wait context") of the nesting rules. It does the raw_spinlock vs. spinlock nesting checks, i.e. it is not legal to acquire a spinlock_t while holding a raw_spinlock_t.
Internally the kfree_rcu() uses raw_spinlock_t whereas the "page allocator" internally deals with spinlock_t to access to its zones. The code also can be broken from higher level of view:
<snip>
raw_spin_lock(&some_lock);
kfree_rcu(some_pointer, some_field_offset);<snip>
b) If built with CONFIG_PREEMPT_RT. Please note, in that case spinlock_t is converted into sleepable variant. Invoking the page allocator from atomic contexts leads to "BUG: scheduling while atomic".
c) call_rcu() is invoked from raw atomic context and kfree_rcu() and kvfree_rcu() are expected to be called from atomic raw context as well.
Move out a page allocation from contexts which trigger kvfree_rcu() function to the separate worker. When a k[v]free_rcu() per-cpu page cache is empty a fallback mechanism is used and a special job is scheduled to refill the per-cpu cache.
--- kernel/rcu/tree.c | 109 ++++++++++++++++++++++++++++------------------ 1 file changed, 66 insertions(+), 43 deletions(-)
kernel/rcu/tree.c +66 -43
@@ -177,7 +177,7 @@ module_param(rcu_unlock_delay, int, 0444);* per-CPU. Object size is equal to one page. This value * can be changed at boot time. */ -static int rcu_min_cached_objs =2 ; +static int rcu_min_cached_objs =5 ;module_param (rcu_min_cached_objs, int,0444 );/* Retrieve RCU kthreads priority for rcutorture */
@@ -3084,6 +3084,9 @@ struct kfree_rcu_cpu_work {* In order to save some per-cpu space the list is singular. * Even though it is lockless an access has to be protected by the * per-cpu lock. +* @page_cache_work: A work to refill the cache when it is empty +* @work_in_progress: Indicates that page_cache_work is running +* @hrtimer: A hrtimer for scheduling a page_cache_work * @nr_bkv_objs: number of allocated objects at @bkvcache. * * This is a per-CPU structure. The reason that it is not included in
@@ -3100,6 +3103,11 @@ struct kfree_rcu_cpu {bool monitor_todo;bool initialized;int count; + +struct work_struct page_cache_work; +atomic_t work_in_progress; +struct hrtimer hrtimer; +struct llist_head bkvcache;int nr_bkv_objs; };
@@ -3217,10 +3225,10 @@ static void kfree_rcu_work(struct work_struct *work) }rcu_lock_release (&rcu_callback_map); - krcp =krc_this_cpu_lock (&flags); +raw_spin_lock_irqsave (&krcp->lock, flags);if (put_cached_bnode (krcp, bkvhead[i])) bkvhead[i] =NULL ; -krc_this_cpu_unlock (krcp, flags); +raw_spin_unlock_irqrestore (&krcp->lock, flags);if (bkvhead[i])free_page ((unsigned long ) bkvhead[i]);
@@ -3347,6 +3355,57 @@ static void kfree_rcu_monitor(struct work_struct *work)raw_spin_unlock_irqrestore (&krcp->lock, flags); } +static enum hrtimer_restart +schedule_page_work_fn (struct hrtimer *t) +{ +struct kfree_rcu_cpu *krcp = + container_of(t, struct kfree_rcu_cpu, hrtimer); + +queue_work (system_highpri_wq, &krcp->page_cache_work); +return HRTIMER_NORESTART ; +} + +static void fill_page_cache_func (struct work_struct *work) +{ +struct kvfree_rcu_bulk_data *bnode; +struct kfree_rcu_cpu *krcp = + container_of(work, struct kfree_rcu_cpu, + page_cache_work); +unsigned long flags; +bool pushed; +int i; + +for (i =0 ; i < rcu_min_cached_objs; i++) { + bnode = (struct kvfree_rcu_bulk_data *) +__get_free_page (GFP_KERNEL | __GFP_NOWARN); + +if (bnode) { +raw_spin_lock_irqsave (&krcp->lock, flags); + pushed =put_cached_bnode (krcp, bnode); +raw_spin_unlock_irqrestore (&krcp->lock, flags); + +if (!pushed) { +free_page ((unsigned long ) bnode); +break ; + } + } + } + +atomic_set (&krcp->work_in_progress,0 ); +} + +static void +run_page_cache_worker (struct kfree_rcu_cpu *krcp) +{ +if (rcu_scheduler_active ==RCU_SCHEDULER_RUNNING && + !atomic_xchg (&krcp->work_in_progress,1 )) { +hrtimer_init (&krcp->hrtimer,CLOCK_MONOTONIC , +HRTIMER_MODE_REL ); + krcp->hrtimer.function = schedule_page_work_fn; +hrtimer_start (&krcp->hrtimer,0 ,HRTIMER_MODE_REL ); + } +} +static inline bool kvfree_call_rcu_add_ptr_to_bulk (struct kfree_rcu_cpu *krcp,void *ptr) {
@@ -3363,32 +3422,8 @@ kvfree_call_rcu_add_ptr_to_bulk(struct kfree_rcu_cpu *krcp, void *ptr)if (!krcp->bkvhead[idx] || krcp->bkvhead[idx]->nr_records ==KVFREE_BULK_MAX_ENTR ) { bnode =get_cached_bnode (krcp); -if (!bnode) { -/* -* To keep this path working on raw non-preemptible -* sections, prevent the optional entry into the -* allocator as it uses sleeping locks. In fact, even -* if the caller of kfree_rcu() is preemptible, this -* path still is not, as krcp->lock is a raw spinlock. -* With additional page pre-allocation in the works, -* hitting this return is going to be much less likely. -*/ -if (IS_ENABLED (CONFIG_PREEMPT_RT )) -return false; - -/* -* NOTE: For one argument of kvfree_rcu() we can -* drop the lock and get the page in sleepable -* context. That would allow to maintain an array -* for the CONFIG_PREEMPT_RT as well if no cached -* pages are available. -*/ - bnode = (struct kvfree_rcu_bulk_data *) -__get_free_page (GFP_NOWAIT | __GFP_NOWARN); - } -/* Switch to emergency path. */ -if (unlikely (!bnode)) +if (!bnode)return false;/* Initialize the new block. */
@@ -3452,12 +3487,10 @@ void kvfree_call_rcu(struct rcu_head *head, rcu_callback_t func) goto unlock_return; } -/* -* Under high memory pressure GFP_NOWAIT can fail, -* in that case the emergency path is maintained. -*/ success =kvfree_call_rcu_add_ptr_to_bulk (krcp, ptr);if (!success) { +run_page_cache_worker (krcp); +if (head ==NULL )// Inline if kvfree_rcu(one_arg) call. goto unlock_return;
@@ -4449,24 +4482,14 @@ static void __init kfree_rcu_batch_init(void)for_each_possible_cpu (cpu) {struct kfree_rcu_cpu *krcp =per_cpu_ptr (&krc, cpu); -struct kvfree_rcu_bulk_data *bnode;for (i =0 ; i <KFREE_N_BATCHES ; i++) {INIT_RCU_WORK (&krcp->krw_arr[i].rcu_work, kfree_rcu_work); krcp->krw_arr[i].krcp = krcp; } -for (i =0 ; i < rcu_min_cached_objs; i++) { - bnode = (struct kvfree_rcu_bulk_data *) -__get_free_page (GFP_NOWAIT | __GFP_NOWARN); - -if (bnode) -put_cached_bnode (krcp, bnode); -else -pr_err ("Failed to preallocate for %d CPU!\n" , cpu); - } -INIT_DELAYED_WORK (&krcp->monitor_work, kfree_rcu_monitor); +INIT_WORK (&krcp->page_cache_work, fill_page_cache_func); krcp->initialized = true; }if (register_shrinker (&kfree_rcu_shrinker))
Signature · 1 line
2.20.1
Uladzislau Rezki (Sony)
maintainerPATCH 2/16 lib/debug: Remove pointless ARCH_NO_PREEMPT dependencies
From: Thomas Gleixner <[email protected]>
ARCH_NO_PREEMPT disables the selection of CONFIG_PREEMPT_VOLUNTARY and CONFIG_PREEMPT, but architectures which set this config option still support preempt count for hard and softirq accounting.
There is absolutely no reason to prevent lockdep from using the preempt counter nor is there a reason to prevent the enablement of CONFIG_DEBUG_ATOMIC_SLEEP on such architectures.
Remove the dependencies.
--- lib/Kconfig.debug | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-)
lib/Kconfig.debug +1 -2
@@ -1159,7 +1159,7 @@ config PROVE_LOCKING select DEBUG_RWSEMS select DEBUG_WW_MUTEX_SLOWPATH select DEBUG_LOCK_ALLOC - select PREEMPT_COUNT if !ARCH_NO_PREEMPT + select PREEMPT_COUNT select TRACE_IRQFLAGS default n help
@@ -1321,7 +1321,6 @@ config DEBUG_ATOMIC_SLEEP bool "Sleep inside atomic section checking" select PREEMPT_COUNT depends on DEBUG_KERNEL - depends on !ARCH_NO_PREEMPT help If you say Y here, various routines which may sleep will become very noisy if they are called inside atomic sections: when a spinlock is
Signature · 1 line
2.20.1
Uladzislau Rezki (Sony)
maintainerPATCH 6/16 mm/pagemap: Cleanup PREEMPT_COUNT leftovers
In reply to Uladzislau Rezki (Sony)From: Thomas Gleixner <[email protected]>
CONFIG_PREEMPT_COUNT is now unconditionally enabled and will be removed. Cleanup the leftovers before doing so.
--- include/linux/pagemap.h | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-)
include/linux/pagemap.h +1 -3
@@ -204,9 +204,7 @@ void release_pages(struct page **pages, int nr);static inline int __page_cache_add_speculative (struct page *page,int count) {#ifdef CONFIG_TINY_RCU -# ifdef CONFIG_PREEMPT_COUNT -VM_BUG_ON (!in_atomic () && !irqs_disabled ()); -# endif +VM_BUG_ON (preemptible ())/* * Preempt must be disabled here - we rely on rcu_read_lock doing * this for us.
Signature · 1 line
2.20.1
Uladzislau Rezki (Sony)
maintainerPATCH 7/16 locking/bitspinlock: Cleanup PREEMPT_COUNT leftovers
In reply to Uladzislau Rezki (Sony)From: Thomas Gleixner <[email protected]>
CONFIG_PREEMPT_COUNT is now unconditionally enabled and will be removed. Cleanup the leftovers before doing so.
--- include/linux/bit_spinlock.h | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-)
include/linux/bit_spinlock.h +1 -3
@@ -90,10 +90,8 @@ static inline int bit_spin_is_locked(int bitnum, unsigned long *addr) {#if defined(CONFIG_SMP ) || defined(CONFIG_DEBUG_SPINLOCK )return test_bit (bitnum, addr); -#elif definedCONFIG_PREEMPT_COUNT -return preempt_count ();#else -return 1 ; +return preempt_count ();#endif }
Signature · 1 line
2.20.1
Uladzislau Rezki (Sony)
maintainerPATCH 9/16 sched: Cleanup PREEMPT_COUNT leftovers
In reply to Uladzislau Rezki (Sony)From: Thomas Gleixner <[email protected]>
CONFIG_PREEMPT_COUNT is now unconditionally enabled and will be removed. Cleanup the leftovers before doing so.
--- kernel/sched/core.c | 6 +----- lib/Kconfig.debug | 1 - 2 files changed, 1 insertion(+), 6 deletions(-)
kernel/sched/core.c +1 -5
@@ -3702,8 +3702,7 @@ asmlinkage __visible void schedule_tail(struct task_struct *prev)* finish_task_switch() for details. * * finish_task_switch() will drop rq->lock() and lower preempt_count -* and the preempt_enable() will end up enabling preemption (on -* PREEMPT_COUNT kernels). +* and the preempt_enable() will end up enabling preemption. */ rq =finish_task_switch (prev);
@@ -7307,9 +7306,6 @@ void __cant_sleep(const char *file, int line, int preempt_offset)if (irqs_disabled ())return ; -if (!IS_ENABLED (CONFIG_PREEMPT_COUNT )) -return ; -if (preempt_count () > preempt_offset)return ;
lib/Kconfig.debug +0 -1
@@ -1318,7 +1318,6 @@ config DEBUG_LOCKDEP config DEBUG_ATOMIC_SLEEP bool "Sleep inside atomic section checking" - select PREEMPT_COUNT depends on DEBUG_KERNEL help If you say Y here, various routines which may sleep will become very
Signature · 1 line
2.20.1
Uladzislau Rezki (Sony)
maintainerPATCH 10/16 ARM: Cleanup PREEMPT_COUNT leftovers
In reply to Uladzislau Rezki (Sony)From: Thomas Gleixner <[email protected]>
CONFIG_PREEMPT_COUNT is now unconditionally enabled and will be removed. Cleanup the leftovers before doing so.
--- arch/arm/include/asm/assembler.h | 11 ----------- arch/arm/kernel/iwmmxt.S | 2 -- arch/arm/mach-ep93xx/crunch-bits.S | 2 -- 3 files changed, 15 deletions(-)
arch/arm/include/asm/assembler.h +0 -11
@@ -212,7 +212,6 @@/* * Increment/decrement the preempt count. */ -#ifdef CONFIG_PREEMPT_COUNT .macro inc_preempt_count, ti, tmp ldr \tmp, [\ti, #TI_PREEMPT] @ get preempt count add \tmp, \tmp, #1 @ increment it
@@ -229,16 +228,6 @@ get_thread_info \ti dec_preempt_count \ti, \tmp .endm -#else - .macro inc_preempt_count, ti, tmp - .endm - - .macro dec_preempt_count, ti, tmp - .endm - - .macro dec_preempt_count_ti, ti, tmp - .endm -#endif #define USERL(l, x...) \ 9999: x; \
arch/arm/kernel/iwmmxt.S +0 -2
@@ -94,9 +94,7 @@ ENTRY(iwmmxt_task_enable) mov r2, r2 @ cpwait bl concan_save -#ifdef CONFIG_PREEMPT_COUNT get_thread_info r10 -#endif 4: dec_preempt_count r10, r3 ret r9 @ normal exit from exception
arch/arm/mach-ep93xx/crunch-bits.S +0 -2
@@ -191,9 +191,7 @@ crunch_load: cfldr64 mvdx15, [r0, #CRUNCH_MVDX15] 1: -#ifdef CONFIG_PREEMPT_COUNT get_thread_info r10 -#endif 2: dec_preempt_count r10, r3 ret lr
Signature · 1 line
2.20.1
Uladzislau Rezki (Sony)
maintainerPATCH 8/16 uaccess: Cleanup PREEMPT_COUNT leftovers
In reply to Uladzislau Rezki (Sony)From: Thomas Gleixner <[email protected]>
CONFIG_PREEMPT_COUNT is now unconditionally enabled and will be removed. Cleanup the leftovers before doing so.
--- include/linux/uaccess.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-)
include/linux/uaccess.h +3 -3
@@ -275,9 +275,9 @@ static inline bool pagefault_disabled(void)* * This function should only be used by the fault handlers. Other users should * stick to pagefault_disabled(). -* Please NEVER use preempt_disable() to disable the fault handler. With -* !CONFIG_PREEMPT_COUNT, this is like a NOP. So the handler won't be disabled. -* in_atomic() will report different values based on !CONFIG_PREEMPT_COUNT. +* +* Please NEVER use preempt_disable() or local_irq_disable() to disable the +* fault handler. */ #define faulthandler_disabled () (pagefault_disabled() || in_atomic())
Signature · 1 line
2.20.1
Uladzislau Rezki (Sony)
maintainerPATCH 12/16 drm/i915: Cleanup PREEMPT_COUNT leftovers
In reply to Uladzislau Rezki (Sony)From: Thomas Gleixner <[email protected]>
CONFIG_PREEMPT_COUNT is now unconditionally enabled and will be removed. Cleanup the leftovers before doing so.
--- drivers/gpu/drm/i915/Kconfig.debug | 1 - drivers/gpu/drm/i915/i915_utils.h | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-)
drivers/gpu/drm/i915/Kconfig.debug +0 -1
@@ -20,7 +20,6 @@ config DRM_I915_DEBUG bool "Enable additional driver debugging" depends on DRM_I915 select DEBUG_FS - select PREEMPT_COUNT select I2C_CHARDEV select STACKDEPOT select DRM_DP_AUX_CHARDEV
drivers/gpu/drm/i915/i915_utils.h +1 -2
@@ -337,8 +337,7 @@ wait_remaining_ms_from_jiffies(unsigned long timestamp_jiffies, int to_wait_ms) (Wmax))#define wait_for (COND ,MS ) _wait_for((COND), (MS) * 1000, 10, 1000) -/* If CONFIG_PREEMPT_COUNT is disabled, in_atomic() always reports false. */ -#if defined(CONFIG_DRM_I915_DEBUG ) && defined(CONFIG_PREEMPT_COUNT ) +#ifdef CONFIG_DRM_I915_DEBUG # define _WAIT_FOR_ATOMIC_CHECK (ATOMIC ) WARN_ON_ONCE((ATOMIC) && !in_atomic())#else # define _WAIT_FOR_ATOMIC_CHECK (ATOMIC ) do { } while (0)
Signature · 1 line
2.20.1
Uladzislau Rezki (Sony)
maintainerPATCH 16/16 rcu/tree: Use delayed work instead of hrtimer to refill the cache
In reply to Uladzislau Rezki (Sony)A CONFIG_PREEMPT_COUNT is unconditionally enabled, thus a page can be obtained directly from a kvfree_rcu() path. To distinguish that and take a decision the preemptable() macro is used when it is save to enter allocator.
It means that refilling a cache is not important from timing point of view. Switch to a delayed work, so the actual work is queued from the timer interrupt with 1 jiffy delay. An immediate placing a task on a current CPU can lead to rq->lock double lock. That is why a delayed method is in place.
--- kernel/rcu/tree.c | 26 +++++--------------------- 1 file changed, 5 insertions(+), 21 deletions(-)
kernel/rcu/tree.c +5 -21
@@ -3086,7 +3086,6 @@ struct kfree_rcu_cpu_work {* per-cpu lock. * @page_cache_work: A work to refill the cache when it is empty * @work_in_progress: Indicates that page_cache_work is running -* @hrtimer: A hrtimer for scheduling a page_cache_work * @nr_bkv_objs: number of allocated objects at @bkvcache. * * This is a per-CPU structure. The reason that it is not included in
@@ -3104,9 +3103,8 @@ struct kfree_rcu_cpu {bool initialized;int count; -struct work_struct page_cache_work; +struct delayed_work page_cache_work;atomic_t work_in_progress; -struct hrtimer hrtimer;struct llist_head bkvcache;int nr_bkv_objs;
@@ -3355,22 +3353,12 @@ static void kfree_rcu_monitor(struct work_struct *work)raw_spin_unlock_irqrestore (&krcp->lock, flags); } -static enum hrtimer_restart -schedule_page_work_fn (struct hrtimer *t) -{ -struct kfree_rcu_cpu *krcp = - container_of(t, struct kfree_rcu_cpu, hrtimer); - -queue_work (system_highpri_wq, &krcp->page_cache_work); -return HRTIMER_NORESTART ; -} -static void fill_page_cache_func (struct work_struct *work) {struct kvfree_rcu_bulk_data *bnode;struct kfree_rcu_cpu *krcp = container_of(work, struct kfree_rcu_cpu, - page_cache_work); + page_cache_work.work);unsigned long flags;bool pushed;int i;
@@ -3398,12 +3386,8 @@ static void run_page_cache_worker(struct kfree_rcu_cpu *krcp) {if (rcu_scheduler_active ==RCU_SCHEDULER_RUNNING && - !atomic_xchg (&krcp->work_in_progress,1 )) { -hrtimer_init (&krcp->hrtimer,CLOCK_MONOTONIC , -HRTIMER_MODE_REL ); - krcp->hrtimer.function = schedule_page_work_fn; -hrtimer_start (&krcp->hrtimer,0 ,HRTIMER_MODE_REL ); - } + !atomic_xchg (&krcp->work_in_progress,1 )) +schedule_delayed_work (&krcp->page_cache_work,1 ); }// Record ptr in a page managed by krcp, with the pre-krc_this_cpu_lock()
@@ -4503,7 +4487,7 @@ static void __init kfree_rcu_batch_init(void) }INIT_DELAYED_WORK (&krcp->monitor_work, kfree_rcu_monitor); -INIT_WORK (&krcp->page_cache_work, fill_page_cache_func); +INIT_DELAYED_WORK (&krcp->page_cache_work, fill_page_cache_func); krcp->initialized = true; }if (register_shrinker (&kfree_rcu_shrinker))
Signature · 1 line
2.20.1
Uladzislau Rezki (Sony)
maintainerPATCH 15/16 rcu/tree: Allocate a page when caller is preemptible
In reply to Uladzislau Rezki (Sony)Given that CONFIG_PREEMPT_COUNT is unconditionally enabled by the earlier commits in this series, the preemptible() macro now properly detects preempt-disable code regions even in kernels built with CONFIG_PREEMPT_NONE.
This commit therefore uses preemptible() to determine whether allocation is possible at all for double-argument kvfree_rcu(). If !preemptible(), then allocation is not possible, and kvfree_rcu() falls back to using the less cache-friendly rcu_head approach. Even when preemptible(), the caller might be involved in reclaim, so the GFP_ flags used by double-argument kvfree_rcu() must avoid invoking reclaim processing.
Note that single-argument kvfree_rcu() must be invoked in sleepable contexts, and that its fallback is the relatively high latency synchronize_rcu(). Single-argument kvfree_rcu() therefore uses GFP_KERNEL|__GFP_RETRY_MAYFAIL to allow limited sleeping within the memory allocator.
--- kernel/rcu/tree.c | 48 ++++++++++++++++++++++++++++++----------------- 1 file changed, 31 insertions(+), 17 deletions(-)
kernel/rcu/tree.c +31 -17
@@ -3406,37 +3406,55 @@ run_page_cache_worker(struct kfree_rcu_cpu *krcp) } } +// Record ptr in a page managed by krcp, with the pre-krc_this_cpu_lock() +// state specified by flags. If can_sleep is true, the caller must +// be schedulable and not be holding any locks or mutexes that might be +// acquired by the memory allocator or anything that it might invoke. +// If !can_sleep, then if !preemptible() no allocation will be undertaken, +// otherwise the allocation will use GFP_ATOMIC to avoid the remainder of +// the aforementioned deadlock possibilities. Returns true if ptr was +// successfully recorded, else the caller must use a fallback. static inline bool -kvfree_call_rcu_add_ptr_to_bulk (struct kfree_rcu_cpu *krcp,void *ptr) +add_ptr_to_bulk_krc_lock (struct kfree_rcu_cpu **krcp, +unsigned long *flags,void *ptr,bool can_sleep) {struct kvfree_rcu_bulk_data *bnode; +bool can_alloc_page =preemptible (); +gfp_t gfp = (can_sleep ?GFP_KERNEL | __GFP_RETRY_MAYFAIL : +GFP_ATOMIC ) | __GFP_NOWARN;int idx; -if (unlikely (!krcp->initialized)) + *krcp =krc_this_cpu_lock (flags); +if (unlikely (!(*krcp)->initialized))return false; -lockdep_assert_held (&krcp->lock); idx = !!is_vmalloc_addr (ptr);/* Check if a new block is required. */ -if (!krcp->bkvhead[idx] || - krcp->bkvhead[idx]->nr_records ==KVFREE_BULK_MAX_ENTR ) { - bnode =get_cached_bnode (krcp); -/* Switch to emergency path. */ +if (!(*krcp)->bkvhead[idx] || + (*krcp)->bkvhead[idx]->nr_records ==KVFREE_BULK_MAX_ENTR ) { + bnode =get_cached_bnode (*krcp); +if (!bnode && can_alloc_page) { +krc_this_cpu_unlock (*krcp, *flags); + bnode = (struct kvfree_rcu_bulk_data *) +__get_free_page (gfp); + *krcp =krc_this_cpu_lock (flags); + } +if (!bnode)return false;/* Initialize the new block. */ bnode->nr_records =0 ; - bnode->next = krcp->bkvhead[idx]; + bnode->next = (*krcp)->bkvhead[idx];/* Attach it to the head. */ - krcp->bkvhead[idx] = bnode; + (*krcp)->bkvhead[idx] = bnode; }/* Finally insert. */ - krcp->bkvhead[idx]->records - [krcp->bkvhead[idx]->nr_records++] = ptr; + (*krcp)->bkvhead[idx]->records + [(*krcp)->bkvhead[idx]->nr_records++] = ptr;return true; }
@@ -3474,20 +3492,16 @@ void kvfree_call_rcu(struct rcu_head *head, rcu_callback_t func) ptr = (unsigned long *) func; } - krcp =krc_this_cpu_lock (&flags); -// Queue the object but don't yet schedule the batch. if (debug_rcu_head_queue (ptr)) {// Probable double kfree_rcu(), just leak. WARN_ONCE (1 ,"%s(): Double-freed call. rcu_head %p\n" , __func__, head); -// Mark as success and leave. - success = true; - goto unlock_return; +return ; } - success =kvfree_call_rcu_add_ptr_to_bulk (krcp, ptr); + success =add_ptr_to_bulk_krc_lock (&krcp, &flags, ptr, !head);if (!success) {run_page_cache_worker (krcp);
Signature · 1 line
2.20.1
Uladzislau Rezki (Sony)
maintainerPATCH 14/16 preempt: Remove PREEMPT_COUNT from Kconfig
In reply to Uladzislau Rezki (Sony)From: Thomas Gleixner <[email protected]>
All conditionals and irritations are gone.
--- kernel/Kconfig.preempt | 3 --- 1 file changed, 3 deletions(-)
kernel/Kconfig.preempt +0 -3
@@ -74,8 +74,5 @@ config PREEMPT_RT endchoice -config PREEMPT_COUNT - def_bool y - config PREEMPTION bool
Signature · 1 line
2.20.1
Uladzislau Rezki (Sony)
maintainerPATCH 13/16 rcutorture: Cleanup PREEMPT_COUNT leftovers
In reply to Uladzislau Rezki (Sony)From: Thomas Gleixner <[email protected]>
CONFIG_PREEMPT_COUNT is now unconditionally enabled and will be removed. Cleanup the leftovers before doing so.
--- tools/testing/selftests/rcutorture/configs/rcu/SRCU-t | 1 - tools/testing/selftests/rcutorture/configs/rcu/SRCU-u | 1 - tools/testing/selftests/rcutorture/configs/rcu/TINY01 | 1 - tools/testing/selftests/rcutorture/doc/TINY_RCU.txt | 5 ++--- tools/testing/selftests/rcutorture/doc/TREE_RCU-kconfig.txt | 1 - .../selftests/rcutorture/formal/srcu-cbmc/src/config.h | 1 - 6 files changed, 2 insertions(+), 8 deletions(-)
tools/testing/selftests/rcutorture/configs/rcu/SRCU-t +0 -1
@@ -7,4 +7,3 @@ CONFIG_RCU_TRACE=n CONFIG_DEBUG_LOCK_ALLOC=n CONFIG_DEBUG_OBJECTS_RCU_HEAD=n CONFIG_DEBUG_ATOMIC_SLEEP=y -#CHECK#CONFIG_PREEMPT_COUNT=y
tools/testing/selftests/rcutorture/configs/rcu/SRCU-u +0 -1
@@ -7,4 +7,3 @@ CONFIG_RCU_TRACE=n CONFIG_DEBUG_LOCK_ALLOC=y CONFIG_PROVE_LOCKING=y CONFIG_DEBUG_OBJECTS_RCU_HEAD=n -CONFIG_PREEMPT_COUNT=n
tools/testing/selftests/rcutorture/configs/rcu/TINY01 +0 -1
@@ -10,4 +10,3 @@ CONFIG_RCU_TRACE=n #CHECK#CONFIG_RCU_STALL_COMMON=n CONFIG_DEBUG_LOCK_ALLOC=n CONFIG_DEBUG_OBJECTS_RCU_HEAD=n -CONFIG_PREEMPT_COUNT=n
tools/testing/selftests/rcutorture/doc/TINY_RCU.txt +2 -3
@@ -3,11 +3,10 @@ This document gives a brief rationale for the TINY_RCU test cases. Kconfig Parameters: -CONFIG_DEBUG_LOCK_ALLOC -- Do all three and none of the three. -CONFIG_PREEMPT_COUNT +CONFIG_DEBUG_LOCK_ALLOC -- Do both and none of the two. CONFIG_RCU_TRACE -The theory here is that randconfig testing will hit the other six possible +The theory here is that randconfig testing will hit the other two possible combinations of these parameters.
tools/testing/selftests/rcutorture/doc/TREE_RCU-kconfig.txt +0 -1
@@ -43,7 +43,6 @@ CONFIG_64BIT Used only to check CONFIG_RCU_FANOUT value, inspection suffices. -CONFIG_PREEMPT_COUNT CONFIG_PREEMPT_RCU Redundant with CONFIG_PREEMPT, ignore.
tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/config.h +0 -1
@@ -8,7 +8,6 @@#undef CONFIG_HOTPLUG_CPU#undef CONFIG_MODULES#undef CONFIG_NO_HZ_FULL_SYSIDLE -#undef CONFIG_PREEMPT_COUNT#undef CONFIG_PREEMPT_RCU#undef CONFIG_PROVE_RCU#undef CONFIG_RCU_NOCB_CPU
Signature · 1 line
2.20.1
Uladzislau Rezki (Sony)
maintainerPATCH 11/16 xtensa: Cleanup PREEMPT_COUNT leftovers
In reply to Uladzislau Rezki (Sony)From: Thomas Gleixner <[email protected]>
CONFIG_PREEMPT_COUNT is now unconditionally enabled and will be removed. Cleanup the leftovers before doing so.
--- arch/xtensa/kernel/entry.S | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
arch/xtensa/kernel/entry.S +1 -1
@@ -819,7 +819,7 @@ ENTRY(debug_exception) * preemption if we have HW breakpoints to preserve DEBUGCAUSE.DBNUM * meaning. */ -#if defined(CONFIG_PREEMPT_COUNT) && defined(CONFIG_HAVE_HW_BREAKPOINT) +#ifdef CONFIG_HAVE_HW_BREAKPOINT GET_THREAD_INFO(a2, a1) l32i a3, a2, TI_PRE_COUNT addi a3, a3, 1
Signature · 1 line
2.20.1
Uladzislau Rezki (Sony)
maintainerPATCH 4/16 preempt: Cleanup PREEMPT_COUNT leftovers
In reply to Uladzislau Rezki (Sony)From: Thomas Gleixner <[email protected]>
CONFIG_PREEMPT_COUNT is now unconditionally enabled and will be removed. Cleanup the leftovers before doing so.
--- include/linux/preempt.h | 37 ++++--------------------------------- 1 file changed, 4 insertions(+), 33 deletions(-)
include/linux/preempt.h +4 -33
@@ -56,8 +56,7 @@#define PREEMPT_DISABLED (PREEMPT_DISABLE_OFFSET + PREEMPT_ENABLED)/* -* Disable preemption until the scheduler is running -- use an unconditional -* value so that it also works on !PREEMPT_COUNT kernels. +* Disable preemption until the scheduler is running. * * Reset by start_kernel()->sched_init()->init_idle()->init_idle_preempt_count(). */
@@ -69,7 +68,6 @@* * preempt_count() == 2*PREEMPT_DISABLE_OFFSET * -* Note: PREEMPT_DISABLE_OFFSET is 0 for !PREEMPT_COUNT kernels. * Note: See finish_task_switch(). */ #define FORK_PREEMPT_COUNT (2*PREEMPT_DISABLE_OFFSET + PREEMPT_ENABLED)
@@ -106,11 +104,7 @@/* * The preempt_count offset after preempt_disable(); */ -#if defined(CONFIG_PREEMPT_COUNT ) -# define PREEMPT_DISABLE_OFFSET PREEMPT_OFFSET -#else -# define PREEMPT_DISABLE_OFFSET 0 -#endif +#define PREEMPT_DISABLE_OFFSET PREEMPT_OFFSET/* * The preempt_count offset after spin_lock()
@@ -122,8 +116,8 @@* * spin_lock_bh() * -* Which need to disable both preemption (CONFIG_PREEMPT_COUNT) and -* softirqs, such that unlock sequences of: +* Which need to disable both preemption and softirqs, such that unlock +* sequences of: * * spin_unlock(); * local_bh_enable();
@@ -164,8 +158,6 @@ extern void preempt_count_sub(int val);#define preempt_count_inc () preempt_count_add(1)#define preempt_count_dec () preempt_count_sub(1) -#ifdef CONFIG_PREEMPT_COUNT -#define preempt_disable () \ do { \ preempt_count_inc(); \
@@ -231,27 +223,6 @@ do { \__preempt_count_dec (); \ }while (0 ) -#else /* !CONFIG_PREEMPT_COUNT */ - -/* -* Even if we don't have any preemption, we need preempt disable/enable -* to be barriers, so that we don't have things like get_user/put_user -* that can cause faults and scheduling migrate into our preempt-protected -* region. -*/ -#define preempt_disable () barrier() -#define sched_preempt_enable_no_resched () barrier() -#define preempt_enable_no_resched () barrier() -#define preempt_enable () barrier() -#define preempt_check_resched () do { } while (0) - -#define preempt_disable_notrace () barrier() -#define preempt_enable_no_resched_notrace () barrier() -#define preempt_enable_notrace () barrier() -#define preemptible () 0 - -#endif /* CONFIG_PREEMPT_COUNT */ - #ifdef MODULE/* * Modules have no business playing preemption tricks.
Signature · 1 line
2.20.1
Uladzislau Rezki (Sony)
maintainerPATCH 5/16 lockdep: Cleanup PREEMPT_COUNT leftovers
In reply to Uladzislau Rezki (Sony)From: Thomas Gleixner <[email protected]>
CONFIG_PREEMPT_COUNT is now unconditionally enabled and will be removed. Cleanup the leftovers before doing so.
--- include/linux/lockdep.h | 6 ++---- lib/Kconfig.debug | 1 - 2 files changed, 2 insertions(+), 5 deletions(-)
include/linux/lockdep.h +2 -4
@@ -580,16 +580,14 @@ do { \#define lockdep_assert_preemption_enabled () \ do { \ - WARN_ON_ONCE(IS_ENABLED(CONFIG_PREEMPT_COUNT) && \ - __lockdep_enabled && \ + WARN_ON_ONCE(__lockdep_enabled && \ (preempt_count() != 0 || \ !this_cpu_read(hardirqs_enabled))); \ } while (0)#define lockdep_assert_preemption_disabled () \ do { \ - WARN_ON_ONCE(IS_ENABLED(CONFIG_PREEMPT_COUNT) && \ - __lockdep_enabled && \ + WARN_ON_ONCE(__lockdep_enabled && \ (preempt_count() == 0 && \ this_cpu_read(hardirqs_enabled))); \ } while (0)
lib/Kconfig.debug +0 -1
@@ -1159,7 +1159,6 @@ config PROVE_LOCKING select DEBUG_RWSEMS select DEBUG_WW_MUTEX_SLOWPATH select DEBUG_LOCK_ALLOC - select PREEMPT_COUNT select TRACE_IRQFLAGS default n help
Signature · 1 line
2.20.1
Uladzislau Rezki (Sony)
maintainerPATCH 3/16 preempt: Make preempt count unconditional
In reply to Uladzislau Rezki (Sony)From: Thomas Gleixner <[email protected]>
The handling of preempt_count() is inconsistent accross kernel configurations. On kernels which have PREEMPT_COUNT=n preempt_disable/enable() and the lock/unlock functions are not affecting the preempt count, only local_bh_disable/enable() and _bh variants of locking, soft interrupt delivery, hard interrupt and NMI context affect it.
It's therefore impossible to have a consistent set of checks which provide information about the context in which a function is called. In many cases it makes sense to have seperate functions for seperate contexts, but there are valid reasons to avoid that and handle different calling contexts conditionally.
The lack of such indicators which work on all kernel configuratios is a constant source of trouble because developers either do not understand the implications or try to work around this inconsistency in weird ways. Neither seem these issues be catched by reviewers and testing.
Recently merged code does:
gfp = preemptible() ? GFP_KERNEL : GFP_ATOMIC;
Looks obviously correct, except for the fact that preemptible() is unconditionally false for CONFIF_PREEMPT_COUNT=n, i.e. all allocations in that code use GFP_ATOMIC on such kernels.
Attempts to make preempt count unconditional and consistent have been rejected in the past with handwaving performance arguments.
Freshly conducted benchmarks did not reveal any measurable impact from enabling preempt count unconditionally. On kernels with CONFIG_PREEMPT_NONE or CONFIG_PREEMPT_VOLUNTARY the preempt count is only incremented and decremented but the result of the decrement is not tested. Contrary to that enabling CONFIG_PREEMPT which tests the result has a small but measurable impact due to the conditional branch/call.
It's about time to make essential functionality of the kernel consistent accross the various preemption models.
Enable CONFIG_PREEMPT_COUNT unconditionally. Follow up changes will remove the #ifdeffery and remove the config option at the end.
--- kernel/Kconfig.preempt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-)
kernel/Kconfig.preempt +1 -2
@@ -75,8 +75,7 @@ config PREEMPT_RT endchoice config PREEMPT_COUNT - bool + def_bool y config PREEMPTION bool - select PREEMPT_COUNT
Signature · 1 line
2.20.1
Paul E. McKenney
maintainerRe: [PATCH 16/16] rcu/tree: Use delayed work instead of hrtimer to refill the cache
In reply to Uladzislau Rezki (Sony)Quoted 5 lines · Uladzislau Rezki (Sony)
A
CONFIG_PREEMPT_COUNTis unconditionally enabled, thus a page can be obtained directly from akvfree_rcu()path. To distinguish that and take a decision thepreemptable()macro is used when it is save to enter allocator.
Quoted 7 lines
It means that refilling a cache is not important from timing point of view. Switch to a delayed work, so the actual work is queued from the timer interrupt with 1 jiffy delay. An immediate placing a task on a current CPU can lead to
rq->lockdouble lock. That is why a delayed method is in place.Signed-off-by Uladzislau Rezki (Sony) <[email protected]>
Thank you, Uladzislau!
I applied this on top of v5.10-rc1 and got the following from the single-CPU builds:
SYNC include/config/auto.conf.cmd
DESCEND objtool
CC kernel/bounds.s
CALL scripts/atomic/check-atomics.sh
UPD include/generated/bounds.h
CC arch/x86/kernel/asm-offsets.s
In file included from ./include/asm-generic/atomic-instrumented.h:20:0,
from ./include/linux/atomic.h:82,
from ./include/linux/crypto.h:15,
from arch/x86/kernel/asm-offsets.c:9:
./include/linux/pagemap.h: In function ‘__page_cache_add_speculative’:
./include/linux/build_bug.h:30:34: error: called object is not a function or function pointer
#define BUILD_BUG_ON_INVALID(e) ((void)(sizeof((__force long)(e))))
~^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
./include/linux/mmdebug.h:45:25: note: in expansion of macro ‘BUILD_BUG_ON_INVALID’
#define VM_BUG_ON(cond) BUILD_BUG_ON_INVALID(cond)
^~~~~~~~~~~~~~~~~~~~./include/linux/pagemap.h:207:2: note: in expansion of macro ‘VM_BUG_ON’
VM_BUG_ON(preemptible())
^~~~~~~~~scripts/Makefile.build:117: recipe for target 'arch/x86/kernel/asm-offsets.s' failed make[1]: *** [arch/x86/kernel/asm-offsets.s] Error 1
Makefile:1199: recipe for target 'prepare0' failed
make: *** [prepare0] Error 2
I vaguely recall something like this showing up in the previous series and that we did something or another to address it. Could you please check against the old series at -rcu branch dev.2020.10.22a? (I verified that the old series does run correctly in the single-CPU scenarios.)
Thanx, Paul
Quoted 78 lines
--- kernel/rcu/tree.c | 26 +++++--------------------- 1 file changed, 5 insertions(+), 21 deletions(-)kernel/rcu/tree.c +5
-21@@ -3086,7 +3086,6 @@ struct kfree_rcu_cpu_work {* per-cpu lock. * @page_cache_work: A work to refill the cache when it is empty * @work_in_progress: Indicates that page_cache_work is running -* @hrtimer: A hrtimer for scheduling a page_cache_work * @nr_bkv_objs: number of allocated objects at @bkvcache. * * This is a per-CPU structure. The reason that it is not included in @@ -3104,9 +3103,8 @@ struct kfree_rcu_cpu {bool initialized;int count; -struct work_struct page_cache_work; +struct delayed_work page_cache_work;atomic_t work_in_progress; -struct hrtimer hrtimer;struct llist_head bkvcache;int nr_bkv_objs;@@ -3355,22 +3353,12 @@ static void kfree_rcu_monitor(struct work_struct *work)raw_spin_unlock_irqrestore (&krcp->lock, flags); } -static enum hrtimer_restart -schedule_page_work_fn (struct hrtimer *t) -{ -struct kfree_rcu_cpu *krcp = - container_of(t, struct kfree_rcu_cpu, hrtimer); - -queue_work (system_highpri_wq, &krcp->page_cache_work); -return HRTIMER_NORESTART ; -} -static void fill_page_cache_func (struct work_struct *work) {struct kvfree_rcu_bulk_data *bnode;struct kfree_rcu_cpu *krcp = container_of(work, struct kfree_rcu_cpu, - page_cache_work); + page_cache_work.work);unsigned long flags;bool pushed;int i;@@ -3398,12 +3386,8 @@ static void run_page_cache_worker(struct kfree_rcu_cpu *krcp) {if (rcu_scheduler_active ==RCU_SCHEDULER_RUNNING && - !atomic_xchg (&krcp->work_in_progress,1 )) { -hrtimer_init (&krcp->hrtimer,CLOCK_MONOTONIC , -HRTIMER_MODE_REL ); - krcp->hrtimer.function = schedule_page_work_fn; -hrtimer_start (&krcp->hrtimer,0 ,HRTIMER_MODE_REL ); - } + !atomic_xchg (&krcp->work_in_progress,1 )) +schedule_delayed_work (&krcp->page_cache_work,1 ); }// Record ptr in a page managed by krcp, with the pre-krc_this_cpu_lock() @@ -4503,7 +4487,7 @@ static void __init kfree_rcu_batch_init(void) }INIT_DELAYED_WORK (&krcp->monitor_work, kfree_rcu_monitor); -INIT_WORK (&krcp->page_cache_work, fill_page_cache_func); +INIT_DELAYED_WORK (&krcp->page_cache_work, fill_page_cache_func); krcp->initialized = true; }if (register_shrinker (&kfree_rcu_shrinker))Signature · 1 line
2.20.1
Uladzislau Rezki
maintainerRe: [PATCH 16/16] rcu/tree: Use delayed work instead of hrtimer to refill the cache
Quoted 44 lines · Paul E. McKenney
Quoted 5 lines · Uladzislau Rezki (Sony)
A
CONFIG_PREEMPT_COUNTis unconditionally enabled, thus a page can be obtained directly from akvfree_rcu()path. To distinguish that and take a decision thepreemptable()macro is used when it is save to enter allocator.Quoted 7 lines
It means that refilling a cache is not important from timing point of view. Switch to a delayed work, so the actual work is queued from the timer interrupt with 1 jiffy delay. An immediate placing a task on a current CPU can lead to
rq->lockdouble lock. That is why a delayed method is in place.Signed-off-by Uladzislau Rezki (Sony) <[email protected]>Thank you, Uladzislau!
I applied this on top of v5.10-rc1 and got the following from the single-CPU builds:
SYNC
include/config/auto.conf.cmd
DESCEND objtoolCC kernel/bounds.s CALL scripts/atomic/check-atomics.sh UPD include/generated/bounds.h CC arch/x86/kernel/asm-offsets.s In file included from ./include/asm-generic/atomic-instrumented.h:20:0, from ./include/linux/atomic.h:82, from ./include/linux/crypto.h:15, from arch/x86/kernel/asm-offsets.c:9: ./include/linux/pagemap.h: In function ‘__page_cache_add_speculative’: ./include/linux/build_bug.h:30:34: error: called object is not a function or function pointer #define BUILD_BUG_ON_INVALID(e) ((void)(sizeof((__force long)(e)))) ~^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ./include/linux/mmdebug.h:45:25: note: in expansion of macro ‘BUILD_BUG_ON_INVALID’ #define VM_BUG_ON(cond) BUILD_BUG_ON_INVALID(cond) ^~~~~~~~~~~~~~~~~~~~
./include/linux/pagemap.h:207:2: note: in expansion of macro ‘VM_BUG_ON’
VM_BUG_ON(preemptible())
^~~~~~~~~scripts/Makefile.build:117: recipe for target 'arch/x86/kernel/asm-offsets.s' failed make[1]: *** [arch/x86/kernel/asm-offsets.s] Error 1
Makefile:1199: recipe for target 'prepare0' failed
make: *** [prepare0] Error 2
Quoted 5 lines
I vaguely recall something like this showing up in the previous series and that we did something or another to address it. Could you please check against the old series at -rcu branch dev.2020.10.22a? (I verified that the old series does run correctly in the single-CPU scenarios.)
I see the same build error. Will double check if we have similar in the previous series also. It looks like the error is caused by the Thomas series.
Will check!
--
Vlad Rezki
Uladzislau Rezki
maintainerRe: [PATCH 16/16] rcu/tree: Use delayed work instead of hrtimer to refill the cache
Quoted 50 lines · Uladzislau Rezki
Quoted 44 lines · Paul E. McKenney
Quoted 5 lines · Uladzislau Rezki (Sony)
A
CONFIG_PREEMPT_COUNTis unconditionally enabled, thus a page can be obtained directly from akvfree_rcu()path. To distinguish that and take a decision thepreemptable()macro is used when it is save to enter allocator.Quoted 7 lines
It means that refilling a cache is not important from timing point of view. Switch to a delayed work, so the actual work is queued from the timer interrupt with 1 jiffy delay. An immediate placing a task on a current CPU can lead to
rq->lockdouble lock. That is why a delayed method is in place.Signed-off-by Uladzislau Rezki (Sony) <[email protected]>Thank you, Uladzislau!
I applied this on top of v5.10-rc1 and got the following from the single-CPU builds:
SYNC
include/config/auto.conf.cmd
DESCEND objtoolCC kernel/bounds.s CALL scripts/atomic/check-atomics.sh UPD include/generated/bounds.h CC arch/x86/kernel/asm-offsets.s In file included from ./include/asm-generic/atomic-instrumented.h:20:0, from ./include/linux/atomic.h:82, from ./include/linux/crypto.h:15, from arch/x86/kernel/asm-offsets.c:9: ./include/linux/pagemap.h: In function ‘__page_cache_add_speculative’: ./include/linux/build_bug.h:30:34: error: called object is not a function or function pointer #define BUILD_BUG_ON_INVALID(e) ((void)(sizeof((__force long)(e)))) ~^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ./include/linux/mmdebug.h:45:25: note: in expansion of macro ‘BUILD_BUG_ON_INVALID’ #define VM_BUG_ON(cond) BUILD_BUG_ON_INVALID(cond) ^~~~~~~~~~~~~~~~~~~~
./include/linux/pagemap.h:207:2: note: in expansion of macro ‘VM_BUG_ON’
VM_BUG_ON(preemptible())
^~~~~~~~~scripts/Makefile.build:117: recipe for target 'arch/x86/kernel/asm-offsets.s' failed make[1]: *** [arch/x86/kernel/asm-offsets.s] Error 1
Makefile:1199: recipe for target 'prepare0' failed
make: *** [prepare0] Error 2Quoted 5 lines
I vaguely recall something like this showing up in the previous series and that we did something or another to address it. Could you please check against the old series at -rcu branch dev.2020.10.22a? (I verified that the old series does run correctly in the single-CPU scenarios.)
Quoted 5 lines
I see the same build error. Will double check if we have similar in the previous series also. It looks like the error is caused by the Thomas series.
Will check!
OK. Found it:
urezki@pc638:~/data/coding/linux.git$ git diff
include/linux/pagemap.h +1 -1
@@ -204,7 +204,7 @@ void release_pages(struct page **pages, int nr);static inline int __page_cache_add_speculative (struct page *page,int count) {#ifdef CONFIG_TINY_RCU -VM_BUG_ON (preemptible ()) +VM_BUG_ON (preemptible ());/* * Preempt must be disabled here - we rely on rcu_read_lock doing * this for us.
urezki@pc638:~/data/coding/linux.git$
I guess we had some extra patch that fixes a kernel compilation for !SMP case. Will check dev.2020.10.22a.
--
Vlad Rezki
Paul E. McKenney
maintainerRe: [PATCH 16/16] rcu/tree: Use delayed work instead of hrtimer to refill the cache
Quoted 74 lines · Uladzislau Rezki
Quoted 50 lines · Uladzislau Rezki
Quoted 44 lines · Paul E. McKenney
Quoted 5 lines · Uladzislau Rezki (Sony)
A
CONFIG_PREEMPT_COUNTis unconditionally enabled, thus a page can be obtained directly from akvfree_rcu()path. To distinguish that and take a decision thepreemptable()macro is used when it is save to enter allocator.Quoted 7 lines
It means that refilling a cache is not important from timing point of view. Switch to a delayed work, so the actual work is queued from the timer interrupt with 1 jiffy delay. An immediate placing a task on a current CPU can lead to
rq->lockdouble lock. That is why a delayed method is in place.Signed-off-by Uladzislau Rezki (Sony) <[email protected]>Thank you, Uladzislau!
I applied this on top of v5.10-rc1 and got the following from the single-CPU builds:
SYNC
include/config/auto.conf.cmd
DESCEND objtoolCC kernel/bounds.s CALL scripts/atomic/check-atomics.sh UPD include/generated/bounds.h CC arch/x86/kernel/asm-offsets.s In file included from ./include/asm-generic/atomic-instrumented.h:20:0, from ./include/linux/atomic.h:82, from ./include/linux/crypto.h:15, from arch/x86/kernel/asm-offsets.c:9: ./include/linux/pagemap.h: In function ‘__page_cache_add_speculative’: ./include/linux/build_bug.h:30:34: error: called object is not a function or function pointer #define BUILD_BUG_ON_INVALID(e) ((void)(sizeof((__force long)(e)))) ~^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ./include/linux/mmdebug.h:45:25: note: in expansion of macro ‘BUILD_BUG_ON_INVALID’ #define VM_BUG_ON(cond) BUILD_BUG_ON_INVALID(cond) ^~~~~~~~~~~~~~~~~~~~
./include/linux/pagemap.h:207:2: note: in expansion of macro ‘VM_BUG_ON’
VM_BUG_ON(preemptible())
^~~~~~~~~scripts/Makefile.build:117: recipe for target 'arch/x86/kernel/asm-offsets.s' failed make[1]: *** [arch/x86/kernel/asm-offsets.s] Error 1
Makefile:1199: recipe for target 'prepare0' failed
make: *** [prepare0] Error 2Quoted 5 lines
I vaguely recall something like this showing up in the previous series and that we did something or another to address it. Could you please check against the old series at -rcu branch dev.2020.10.22a? (I verified that the old series does run correctly in the single-CPU scenarios.)
Quoted 5 lines
I see the same build error. Will double check if we have similar in the previous series also. It looks like the error is caused by the Thomas series.
Will check!
OK. Found it:
urezki@pc638:~/data/coding/linux.git$ git diffinclude/linux/pagemap.h +1
-1@@ -204,7 +204,7 @@ void release_pages(struct page **pages, int nr);static inline int __page_cache_add_speculative (struct page *page,int count) {#ifdef CONFIG_TINY_RCU -VM_BUG_ON (preemptible ()) +VM_BUG_ON (preemptible ());/* * Preempt must be disabled here - we rely on rcu_read_lock doing * this for us. urezki@pc638:~/data/coding/linux.git$
Quoted 2 lines
I guess we had some extra patch that fixes a kernel compilation for !SMP case. Will check dev.2020.10.22a.
I guess I am blind today!
And yes, that semicolon is in the corresponding commit in -tip.
So, an important safety tip: When forward porting, start from the commits that have been tested rather than from the original series of patches. ;-)
Thanx, Paul
Uladzislau Rezki
maintainerRe: [PATCH 06/16] mm/pagemap: Cleanup PREEMPT_COUNT leftovers
In reply to Uladzislau Rezki (Sony)Quoted 31 lines · Uladzislau Rezki (Sony)
From: Thomas Gleixner <[email protected]>
CONFIG_PREEMPT_COUNTis now unconditionally enabled and will be removed. Cleanup the leftovers before doing so.Signed-off-by Thomas Gleixner <[email protected]>Cc Andrew Morton <[email protected]>Signed-off-by Uladzislau Rezki (Sony) <[email protected]>--- include/linux/pagemap.h | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-)include/linux/pagemap.h +1
-3@@ -204,9 +204,7 @@ void release_pages(struct page **pages, int nr);static inline int __page_cache_add_speculative (struct page *page,int count) {#ifdef CONFIG_TINY_RCU -# ifdef CONFIG_PREEMPT_COUNT -VM_BUG_ON (!in_atomic () && !irqs_disabled ()); -# endif +VM_BUG_ON (preemptible ())/* * Preempt must be disabled here - we rely on rcu_read_lock doing * this for us. Signature · 1 line
2.20.1
Hello, Paul.
Sorry for a small mistake, it was fixed by you before, whereas i took an old version of the patch that is question. Please use below one instead of posted one:
Author: Thomas Gleixner <[email protected]> Date: Mon Sep 14 19:25:00 2020 +0200
mm/pagemap: Cleanup PREEMPT_COUNT leftovers
CONFIG_PREEMPT_COUNT is now unconditionally enabled and will be removed. Cleanup the leftovers before doing so.
include/linux/pagemap.h +1 -3
@@ -168,9 +168,7 @@ void release_pages(struct page **pages, int nr);static inline int __page_cache_add_speculative (struct page *page,int count) {#ifdef CONFIG_TINY_RCU -# ifdef CONFIG_PREEMPT_COUNT -VM_BUG_ON (!in_atomic () && !irqs_disabled ()); -# endif +VM_BUG_ON (preemptible ());/* * Preempt must be disabled here - we rely on rcu_read_lock doing * this for us.
--
Vlad Rezki
Uladzislau Rezki
maintainerRe: [PATCH 16/16] rcu/tree: Use delayed work instead of hrtimer to refill the cache
In reply to Paul E. McKenneyQuoted 80 lines · Paul E. McKenney
Quoted 74 lines · Uladzislau Rezki
Quoted 50 lines · Uladzislau Rezki
Quoted 44 lines · Paul E. McKenney
Quoted 5 lines · Uladzislau Rezki (Sony)
A
CONFIG_PREEMPT_COUNTis unconditionally enabled, thus a page can be obtained directly from akvfree_rcu()path. To distinguish that and take a decision thepreemptable()macro is used when it is save to enter allocator.Quoted 7 lines
It means that refilling a cache is not important from timing point of view. Switch to a delayed work, so the actual work is queued from the timer interrupt with 1 jiffy delay. An immediate placing a task on a current CPU can lead to
rq->lockdouble lock. That is why a delayed method is in place.Signed-off-by Uladzislau Rezki (Sony) <[email protected]>Thank you, Uladzislau!
I applied this on top of v5.10-rc1 and got the following from the single-CPU builds:
SYNC
include/config/auto.conf.cmd
DESCEND objtoolCC kernel/bounds.s CALL scripts/atomic/check-atomics.sh UPD include/generated/bounds.h CC arch/x86/kernel/asm-offsets.s In file included from ./include/asm-generic/atomic-instrumented.h:20:0, from ./include/linux/atomic.h:82, from ./include/linux/crypto.h:15, from arch/x86/kernel/asm-offsets.c:9: ./include/linux/pagemap.h: In function ‘__page_cache_add_speculative’: ./include/linux/build_bug.h:30:34: error: called object is not a function or function pointer #define BUILD_BUG_ON_INVALID(e) ((void)(sizeof((__force long)(e)))) ~^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ./include/linux/mmdebug.h:45:25: note: in expansion of macro ‘BUILD_BUG_ON_INVALID’ #define VM_BUG_ON(cond) BUILD_BUG_ON_INVALID(cond) ^~~~~~~~~~~~~~~~~~~~
./include/linux/pagemap.h:207:2: note: in expansion of macro ‘VM_BUG_ON’
VM_BUG_ON(preemptible())
^~~~~~~~~scripts/Makefile.build:117: recipe for target 'arch/x86/kernel/asm-offsets.s' failed make[1]: *** [arch/x86/kernel/asm-offsets.s] Error 1
Makefile:1199: recipe for target 'prepare0' failed
make: *** [prepare0] Error 2Quoted 5 lines
I vaguely recall something like this showing up in the previous series and that we did something or another to address it. Could you please check against the old series at -rcu branch dev.2020.10.22a? (I verified that the old series does run correctly in the single-CPU scenarios.)
Quoted 5 lines
I see the same build error. Will double check if we have similar in the previous series also. It looks like the error is caused by the Thomas series.
Will check!
OK. Found it:
urezki@pc638:~/data/coding/linux.git$ git diffinclude/linux/pagemap.h +1
-1@@ -204,7 +204,7 @@ void release_pages(struct page **pages, int nr);static inline int __page_cache_add_speculative (struct page *page,int count) {#ifdef CONFIG_TINY_RCU -VM_BUG_ON (preemptible ()) +VM_BUG_ON (preemptible ());/* * Preempt must be disabled here - we rely on rcu_read_lock doing * this for us. urezki@pc638:~/data/coding/linux.git$
Quoted 2 lines
I guess we had some extra patch that fixes a kernel compilation for !SMP case. Will check dev.2020.10.22a.
I guess I am blind today!
Quoted 7 lines
And yes, that semicolon is in the corresponding commit in -tip.
So, an important safety tip: When forward porting, start from the commits that have been tested rather than from the original series of patches. ;-)
Thanx, Paul
Right. Sorry for that. I remember you mentioned about a build error some time ago. My bad :)
I have updated the: [PATCH 06/16] mm/pagemap: Cleanup PREEMPT_COUNT leftovers in this series.
--
Vlad Rezki
kernel test robot
build botRe: [PATCH 06/16] mm/pagemap: Cleanup PREEMPT_COUNT leftovers
In reply to Uladzislau Rezki (Sony)I love your patch! Yet something to improve:
Hi "Uladzislau,
I love your patch! Yet something to improve:
[auto build test ERROR on arm/for-next]
[also build test ERROR on tip/sched/core linus/master v5.10-rc1] [cannot apply to rcu/dev drm-intel/for-linux-next next-20201029] [If your patch is applied to the wrong git tree, kindly drop us a note. And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch]
url: https://github.com/0day-ci/linux/commits/Uladzislau-Rezki-Sony/rcu-tree-Add-a-work-to-allocate-pages-from-regular-context/20201030-005247 base: git://git.armlinux.org.uk/~rmk/linux-arm.git for-next config: i386-randconfig-a016-20201029 (attached as .config) compiler: gcc-9 (Debian 9.3.0-15) 9.3.0 reproduce (this is a W=1 build): # https://github.com/0day-ci/linux/commit/1e0b66ce2b779463194f8b398b8e48caa95aabf4 git remote add linux-review https://github.com/0day-ci/linux git fetch --no-tags linux-review Uladzislau-Rezki-Sony/rcu-tree-Add-a-work-to-allocate-pages-from-regular-context/20201030-005247 git checkout 1e0b66ce2b779463194f8b398b8e48caa95aabf4 # save the attached .config to linux build tree make W=1 ARCH=i386
If you fix the issue, kindly add following tag as appropriate
All errors (new ones prefixed by >>):
In file included from include/asm-generic/atomic-instrumented.h:20,
from include/linux/atomic.h:82,
from include/linux/crypto.h:15,
from arch/x86/kernel/asm-offsets.c:9:
include/linux/pagemap.h: In function '__page_cache_add_speculative':
>> include/linux/build_bug.h:30:34: error: called object is not a function or function pointer
30 | #define BUILD_BUG_ON_INVALID(e) ((void)(sizeof((__force long)(e))))
| ~^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
include/linux/mmdebug.h:45:25: note: in expansion of macro 'BUILD_BUG_ON_INVALID'
45 | #define VM_BUG_ON(cond) BUILD_BUG_ON_INVALID(cond)
| ^~~~~~~~~~~~~~~~~~~~
include/linux/pagemap.h:207:2: note: in expansion of macro 'VM_BUG_ON'
207 | VM_BUG_ON(preemptible())
| ^~~~~~~~~ make[2]: *** [scripts/Makefile.build:117: arch/x86/kernel/asm-offsets.s] Error 1 make[2]: Target '__build' not remade because of errors.
make[1]: *** [Makefile:1200: prepare0] Error 2
make[1]: Target 'prepare' not remade because of errors.
make: *** [Makefile:185: __sub-make] Error 2
make: Target 'prepare' not remade because of errors.
vim +30 include/linux/build_bug.h
527edbc18a70e74 Masahiro Yamada 2019-01-03 18 527edbc18a70e74 Masahiro Yamada 2019-01-03 19 /* Force a compilation error if a constant expression is not a power of 2 */ 527edbc18a70e74 Masahiro Yamada 2019-01-03 20 #define __BUILD_BUG_ON_NOT_POWER_OF_2(n) \ 527edbc18a70e74 Masahiro Yamada 2019-01-03 21 BUILD_BUG_ON(((n) & ((n) - 1)) != 0) 527edbc18a70e74 Masahiro Yamada 2019-01-03 22 #define BUILD_BUG_ON_NOT_POWER_OF_2(n) \ 527edbc18a70e74 Masahiro Yamada 2019-01-03 23 BUILD_BUG_ON((n) == 0 || (((n) & ((n) - 1)) != 0)) bc6245e5efd70c4 Ian Abbott 2017-07-10 24 bc6245e5efd70c4 Ian Abbott 2017-07-10 25 /* bc6245e5efd70c4 Ian Abbott 2017-07-10 26 * BUILD_BUG_ON_INVALID() permits the compiler to check the validity of the bc6245e5efd70c4 Ian Abbott 2017-07-10 27 * expression but avoids the generation of any code, even if that expression bc6245e5efd70c4 Ian Abbott 2017-07-10 28 * has side-effects. bc6245e5efd70c4 Ian Abbott 2017-07-10 29 */
bc6245e5efd70c4 Ian Abbott 2017-07-10 @30 #define BUILD_BUG_ON_INVALID(e) ((void)(sizeof((__force long)(e))))bc6245e5efd70c4 Ian Abbott 2017-07-10 31
---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/kbuild-all(a)lists.01.org
kernel test robot
build botRe: [PATCH 06/16] mm/pagemap: Cleanup PREEMPT_COUNT leftovers
In reply to Uladzislau Rezki (Sony)I love your patch! Perhaps something to improve:
Hi "Uladzislau,
I love your patch! Perhaps something to improve:
[auto build test WARNING on arm/for-next]
[also build test WARNING on tip/sched/core linus/master v5.10-rc1] [cannot apply to rcu/dev drm-intel/for-linux-next next-20201029] [If your patch is applied to the wrong git tree, kindly drop us a note. And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch]
url: https://github.com/0day-ci/linux/commits/Uladzislau-Rezki-Sony/rcu-tree-Add-a-work-to-allocate-pages-from-regular-context/20201030-005247 base: git://git.armlinux.org.uk/~rmk/linux-arm.git for-next config: arm-randconfig-r004-20201029 (attached as .config) compiler: clang version 12.0.0 (https://github.com/llvm/llvm-project 772aaa602383cf82795572ebcd86b0e660f3579f)
reproduce (this is a W=1 build):
wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
chmod +x ~/bin/make.cross
# install arm cross compiling tool for clang build
# apt-get install binutils-arm-linux-gnueabi
# https://github.com/0day-ci/linux/commit/1e0b66ce2b779463194f8b398b8e48caa95aabf4
git remote add linux-review https://github.com/0day-ci/linux
git fetch --no-tags linux-review Uladzislau-Rezki-Sony/rcu-tree-Add-a-work-to-allocate-pages-from-regular-context/20201030-005247 git checkout 1e0b66ce2b779463194f8b398b8e48caa95aabf4
# save the attached .config to linux build tree
COMPILER_INSTALL_PATH=$HOME/0day COMPILER=clang make.cross ARCH=arm
If you fix the issue, kindly add following tag as appropriate
All warnings (new ones prefixed by >>):
In file included from drivers/char/random.c:348:
In file included from include/trace/events/random.h:8:
In file included from include/linux/writeback.h:14:
In file included from include/linux/blk-cgroup.h:23:
In file included from include/linux/blkdev.h:14:
include/linux/pagemap.h:207:26: error: expected ';' after do/while statement
VM_BUG_ON(preemptible())
^
;
>> drivers/char/random.c:1269:41: warning: shift count >= width of type [-Wshift-count-overflow]
c_high = (sizeof(cycles) > 4) ? cycles >> 32 : 0;
^ ~~
drivers/char/random.c:1270:35: warning: shift count >= width of type [-Wshift-count-overflow]
j_high = (sizeof(now) > 4) ? now >> 32 : 0;
^ ~~
drivers/char/random.c:2296:6: warning: no previous prototype for function 'add_hwgenerator_randomness' [-Wmissing-prototypes]
void add_hwgenerator_randomness(const char *buffer, size_t count,
^ drivers/char/random.c:2296:1: note: declare 'static' if the function is not intended to be used outside of this translation unit void add_hwgenerator_randomness(const char *buffer, size_t count,
^
static
3 warnings and 1 error generated.
--
In file included from drivers/usb/host/ehci-platform.c:37:
In file included from include/linux/usb/of.h:12:
In file included from include/linux/usb/otg.h:13:
In file included from include/linux/phy/phy.h:17:
In file included from include/linux/regulator/consumer.h:35:
In file included from include/linux/suspend.h:5:
In file included from include/linux/swap.h:9:
In file included from include/linux/memcontrol.h:22:
In file included from include/linux/writeback.h:14:
In file included from include/linux/blk-cgroup.h:23:
In file included from include/linux/blkdev.h:14:
include/linux/pagemap.h:207:26: error: expected ';' after do/while statement
VM_BUG_ON(preemptible())
^
;
>> drivers/usb/host/ehci-platform.c:259:24: warning: shift count >= width of type [-Wshift-count-overflow]
pdata->dma_mask_64 ? DMA_BIT_MASK(64) : DMA_BIT_MASK(32));
^~~~~~~~~~~~~~~~
include/linux/dma-mapping.h:76:54: note: expanded from macro 'DMA_BIT_MASK'
#define DMA_BIT_MASK(n) (((n) == 64) ? ~0ULL : ((1ULL<<(n))-1))
^ ~~~ 1 warning and 1 error generated.
--
In file included from drivers/usb/host/xhci-plat.c:21:
In file included from include/linux/usb/of.h:12:
In file included from include/linux/usb/otg.h:13:
In file included from include/linux/phy/phy.h:17:
In file included from include/linux/regulator/consumer.h:35:
In file included from include/linux/suspend.h:5:
In file included from include/linux/swap.h:9:
In file included from include/linux/memcontrol.h:22:
In file included from include/linux/writeback.h:14:
In file included from include/linux/blk-cgroup.h:23:
In file included from include/linux/blkdev.h:14:
include/linux/pagemap.h:207:26: error: expected ';' after do/while statement
VM_BUG_ON(preemptible())
^
;
>> drivers/usb/host/xhci-plat.c:222:10: warning: shift count >= width of type [-Wshift-count-overflow]
DMA_BIT_MASK(64));
^~~~~~~~~~~~~~~~
include/linux/dma-mapping.h:76:54: note: expanded from macro 'DMA_BIT_MASK'
#define DMA_BIT_MASK(n) (((n) == 64) ? ~0ULL : ((1ULL<<(n))-1))
^ ~~~
drivers/usb/host/xhci-plat.c:224:43: warning: shift count >= width of type [-Wshift-count-overflow]
ret = dma_set_mask_and_coherent(sysdev, DMA_BIT_MASK(64));
^~~~~~~~~~~~~~~~
include/linux/dma-mapping.h:76:54: note: expanded from macro 'DMA_BIT_MASK'
#define DMA_BIT_MASK(n) (((n) == 64) ? ~0ULL : ((1ULL<<(n))-1))
^ ~~~ 2 warnings and 1 error generated.
--
In file included from drivers/mmc/host/sdhci.c:24:
In file included from include/linux/regulator/consumer.h:35:
In file included from include/linux/suspend.h:5:
In file included from include/linux/swap.h:9:
In file included from include/linux/memcontrol.h:22:
In file included from include/linux/writeback.h:14:
In file included from include/linux/blk-cgroup.h:23:
In file included from include/linux/blkdev.h:14:
include/linux/pagemap.h:207:26: error: expected ';' after do/while statement
VM_BUG_ON(preemptible())
^
;
>> drivers/mmc/host/sdhci.c:3953:40: warning: shift count >= width of type [-Wshift-count-overflow]
ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
^~~~~~~~~~~~~~~~
include/linux/dma-mapping.h:76:54: note: expanded from macro 'DMA_BIT_MASK'
#define DMA_BIT_MASK(n) (((n) == 64) ? ~0ULL : ((1ULL<<(n))-1))
^ ~~~
drivers/mmc/host/sdhci.c:4254:20: warning: shift count >= width of type [-Wshift-count-overflow]
host->dma_mask = DMA_BIT_MASK(64);
^~~~~~~~~~~~~~~~
include/linux/dma-mapping.h:76:54: note: expanded from macro 'DMA_BIT_MASK'
#define DMA_BIT_MASK(n) (((n) == 64) ? ~0ULL : ((1ULL<<(n))-1))
^ ~~~2 warnings and 1 error generated.
vim +1269 drivers/char/random.c
ee3e00e9e7101c8 Theodore Ts'o 2014-06-15 1254
775f4b297b78060 Theodore Ts'o 2012-07-02 1255 void add_interrupt_randomness(int irq, int irq_flags)
^1da177e4c3f415 Linus Torvalds 2005-04-16 1256 {
775f4b297b78060 Theodore Ts'o 2012-07-02 1257 struct entropy_store *r;
1b2a1a7e8ad1144 Christoph Lameter 2014-08-17 1258 struct fast_pool *fast_pool = this_cpu_ptr(&irq_randomness);
775f4b297b78060 Theodore Ts'o 2012-07-02 1259 struct pt_regs *regs = get_irq_regs();
775f4b297b78060 Theodore Ts'o 2012-07-02 1260 unsigned long now = jiffies;
655b226470b2295 Theodore Ts'o 2013-09-22 1261 cycles_t cycles = random_get_entropy();
43759d4f429c8d5 Theodore Ts'o 2014-06-14 1262 __u32 c_high, j_high;
655b226470b2295 Theodore Ts'o 2013-09-22 1263 __u64 ip;
83664a6928a420b H. Peter Anvin 2014-03-17 1264 unsigned long seed;
91fcb532efe366d Theodore Ts'o 2014-06-10 1265 int credit = 0;
655b226470b2295 Theodore Ts'o 2013-09-22 1266
ee3e00e9e7101c8 Theodore Ts'o 2014-06-15 1267 if (cycles == 0)
ee3e00e9e7101c8 Theodore Ts'o 2014-06-15 1268 cycles = get_reg(fast_pool, regs);
655b226470b2295 Theodore Ts'o 2013-09-22 @1269 c_high = (sizeof(cycles) > 4) ? cycles >> 32 : 0;
655b226470b2295 Theodore Ts'o 2013-09-22 1270 j_high = (sizeof(now) > 4) ? now >> 32 : 0;
43759d4f429c8d5 Theodore Ts'o 2014-06-14 1271 fast_pool->pool[0] ^= cycles ^ j_high ^ irq;
43759d4f429c8d5 Theodore Ts'o 2014-06-14 1272 fast_pool->pool[1] ^= now ^ c_high;
655b226470b2295 Theodore Ts'o 2013-09-22 1273 ip = regs ? instruction_pointer(regs) : _RET_IP_;
43759d4f429c8d5 Theodore Ts'o 2014-06-14 1274 fast_pool->pool[2] ^= ip;
ee3e00e9e7101c8 Theodore Ts'o 2014-06-15 1275 fast_pool->pool[3] ^= (sizeof(ip) > 4) ? ip >> 32 :
ee3e00e9e7101c8 Theodore Ts'o 2014-06-15 1276 get_reg(fast_pool, regs);
3060d6fe2857064 Yinghai Lu 2008-08-19 1277
43759d4f429c8d5 Theodore Ts'o 2014-06-14 1278 fast_mix(fast_pool);
43759d4f429c8d5 Theodore Ts'o 2014-06-14 1279 add_interrupt_bench(cycles);
3060d6fe2857064 Yinghai Lu 2008-08-19 1280
43838a23a05fbd1 Theodore Ts'o 2018-04-11 1281 if (unlikely(crng_init == 0)) {
e192be9d9a30555 Theodore Ts'o 2016-06-12 1282 if ((fast_pool->count >= 64) &&
e192be9d9a30555 Theodore Ts'o 2016-06-12 1283 crng_fast_load((char *) fast_pool->pool,
e192be9d9a30555 Theodore Ts'o 2016-06-12 1284 sizeof(fast_pool->pool))) {
e192be9d9a30555 Theodore Ts'o 2016-06-12 1285 fast_pool->count = 0;
e192be9d9a30555 Theodore Ts'o 2016-06-12 1286 fast_pool->last = now;
e192be9d9a30555 Theodore Ts'o 2016-06-12 1287 }
e192be9d9a30555 Theodore Ts'o 2016-06-12 1288 return;
e192be9d9a30555 Theodore Ts'o 2016-06-12 1289 }
e192be9d9a30555 Theodore Ts'o 2016-06-12 1290
840f95077ffd640 Theodore Ts'o 2014-06-14 1291 if ((fast_pool->count < 64) &&
840f95077ffd640 Theodore Ts'o 2014-06-14 1292 !time_after(now, fast_pool->last + HZ))
^1da177e4c3f415 Linus Torvalds 2005-04-16 1293 return;
^1da177e4c3f415 Linus Torvalds 2005-04-16 1294
e192be9d9a30555 Theodore Ts'o 2016-06-12 1295 r = &input_pool;
840f95077ffd640 Theodore Ts'o 2014-06-14 1296 if (!spin_trylock(&r->lock))
91fcb532efe366d Theodore Ts'o 2014-06-10 1297 return;
83664a6928a420b H. Peter Anvin 2014-03-17 1298
91fcb532efe366d Theodore Ts'o 2014-06-10 1299 fast_pool->last = now;
85608f8e16c28f8 Theodore Ts'o 2014-06-10 1300 __mix_pool_bytes(r, &fast_pool->pool, sizeof(fast_pool->pool));
83664a6928a420b H. Peter Anvin 2014-03-17 1301
83664a6928a420b H. Peter Anvin 2014-03-17 1302 /*
83664a6928a420b H. Peter Anvin 2014-03-17 1303 * If we have architectural seed generator, produce a seed and
48d6be955a7167b Theodore Ts'o 2014-07-17 1304 * add it to the pool. For the sake of paranoia don't let the
48d6be955a7167b Theodore Ts'o 2014-07-17 1305 * architectural seed generator dominate the input from the
48d6be955a7167b Theodore Ts'o 2014-07-17 1306 * interrupt noise.
83664a6928a420b H. Peter Anvin 2014-03-17 1307 */
83664a6928a420b H. Peter Anvin 2014-03-17 1308 if (arch_get_random_seed_long(&seed)) {
85608f8e16c28f8 Theodore Ts'o 2014-06-10 1309 __mix_pool_bytes(r, &seed, sizeof(seed));
48d6be955a7167b Theodore Ts'o 2014-07-17 1310 credit = 1;
83664a6928a420b H. Peter Anvin 2014-03-17 1311 }
91fcb532efe366d Theodore Ts'o 2014-06-10 1312 spin_unlock(&r->lock);
83664a6928a420b H. Peter Anvin 2014-03-17 1313
ee3e00e9e7101c8 Theodore Ts'o 2014-06-15 1314 fast_pool->count = 0;
83664a6928a420b H. Peter Anvin 2014-03-17 1315
ee3e00e9e7101c8 Theodore Ts'o 2014-06-15 1316 /* award one bit for the contents of the fast pool */
ee3e00e9e7101c8 Theodore Ts'o 2014-06-15 1317 credit_entropy_bits(r, credit + 1);
^1da177e4c3f415 Linus Torvalds 2005-04-16 1318 }
4b44f2d18a33056 Stephan Mueller 2016-05-02 1319 EXPORT_SYMBOL_GPL(add_interrupt_randomness);
^1da177e4c3f415 Linus Torvalds 2005-04-16 1320 ---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/kbuild-all(a)lists.01.org
Paul E. McKenney
maintainerRe: [PATCH 06/16] mm/pagemap: Cleanup PREEMPT_COUNT leftovers
In reply to Uladzislau RezkiQuoted 34 lines · Uladzislau Rezki
Quoted 31 lines · Uladzislau Rezki (Sony)
From: Thomas Gleixner <[email protected]>
CONFIG_PREEMPT_COUNTis now unconditionally enabled and will be removed. Cleanup the leftovers before doing so.Signed-off-by Thomas Gleixner <[email protected]>Cc Andrew Morton <[email protected]>Signed-off-by Uladzislau Rezki (Sony) <[email protected]>--- include/linux/pagemap.h | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-)include/linux/pagemap.h +1
-3@@ -204,9 +204,7 @@ void release_pages(struct page **pages, int nr);static inline int __page_cache_add_speculative (struct page *page,int count) {#ifdef CONFIG_TINY_RCU -# ifdef CONFIG_PREEMPT_COUNT -VM_BUG_ON (!in_atomic () && !irqs_disabled ()); -# endif +VM_BUG_ON (preemptible ())/* * Preempt must be disabled here - we rely on rcu_read_lock doing * this for us. Signature · 1 line
2.20.1Hello, Paul.
Quoted 3 lines
Sorry for a small mistake, it was fixed by you before, whereas i took an old version of the patch that is question. Please use below one instead of posted one:
We have all been there and done that! ;-)
I will give this update a spin and see what happens.
Thanx, Paul
Quoted 32 lines
Author: Thomas Gleixner <[email protected]> Date: Mon Sep 14 19:25:00 2020 +0200mm/pagemap: Cleanup
PREEMPT_COUNTleftovers
CONFIG_PREEMPT_COUNTis now unconditionally enabled and will be removed. Cleanup the leftovers before doing so.Signed-off-by Thomas Gleixner <[email protected]>Cc Andrew Morton <[email protected]>[ paulmck: Fix !SMP build error per kernel test robot feedback. ]Signed-off-by Paul E. McKenney <[email protected]>include/linux/pagemap.h +1
-4@@ -168,9 +168,7 @@ void release_pages(struct page **pages, int nr);static inline int __page_cache_add_speculative (struct page *page,int count) {#ifdef CONFIG_TINY_RCU -# ifdef CONFIG_PREEMPT_COUNT -VM_BUG_ON (!in_atomic () && !irqs_disabled ()); -# endif +VM_BUG_ON (preemptible ());/* * Preempt must be disabled here - we rely on rcu_read_lock doing * this for us. -- Vlad Rezki
kernel test robot
build botRe: [PATCH 06/16] mm/pagemap: Cleanup PREEMPT_COUNT leftovers
In reply to Uladzislau Rezki (Sony)I love your patch! Yet something to improve:
Hi "Uladzislau,
I love your patch! Yet something to improve:
[auto build test ERROR on arm/for-next]
[also build test ERROR on tip/sched/core linus/master v5.10-rc1] [cannot apply to rcu/dev drm-intel/for-linux-next next-20201029] [If your patch is applied to the wrong git tree, kindly drop us a note. And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch]
url: https://github.com/0day-ci/linux/commits/Uladzislau-Rezki-Sony/rcu-tree-Add-a-work-to-allocate-pages-from-regular-context/20201030-005247 base: git://git.armlinux.org.uk/~rmk/linux-arm.git for-next config: xtensa-allyesconfig (attached as .config) compiler: xtensa-linux-gcc (GCC) 9.3.0
reproduce (this is a W=1 build):
wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
chmod +x ~/bin/make.cross
# https://github.com/0day-ci/linux/commit/1e0b66ce2b779463194f8b398b8e48caa95aabf4
git remote add linux-review https://github.com/0day-ci/linux
git fetch --no-tags linux-review Uladzislau-Rezki-Sony/rcu-tree-Add-a-work-to-allocate-pages-from-regular-context/20201030-005247 git checkout 1e0b66ce2b779463194f8b398b8e48caa95aabf4
# save the attached .config to linux build tree
COMPILER_INSTALL_PATH=$HOME/0day COMPILER=gcc-9.3.0 make.cross ARCH=xtensa
If you fix the issue, kindly add following tag as appropriate
All errors (new ones prefixed by >>):
In file included from include/linux/gfp.h:5,
from include/linux/umh.h:4,
from include/linux/kmod.h:9,
from include/linux/module.h:16,
from drivers/staging/wlan-ng/p80211req.c:54:
include/linux/pagemap.h: In function '__page_cache_add_speculative':
>> include/linux/mmdebug.h:20:2: error: expected ';' before 'do'
20 | do { \
| ^~
include/linux/pagemap.h:217:2: note: in expansion of macro 'VM_BUG_ON_PAGE'
217 | VM_BUG_ON_PAGE(page_count(page) == 0, page);
| ^~~~~~~~~~~~~~
--
In file included from include/linux/percpu.h:5,
from include/linux/radix-tree.h:14,
from include/linux/fs.h:15,
from include/linux/dax.h:5,
from mm/filemap.c:15:
include/linux/pagemap.h: In function '__page_cache_add_speculative':
>> include/linux/mmdebug.h:20:2: error: expected ';' before 'do'
20 | do { \
| ^~
include/linux/pagemap.h:217:2: note: in expansion of macro 'VM_BUG_ON_PAGE'
217 | VM_BUG_ON_PAGE(page_count(page) == 0, page);
| ^~~~~~~~~~~~~~ mm/filemap.c: At top level:
mm/filemap.c:830:14: warning: no previous prototype for '__add_to_page_cache_locked' [-Wmissing-prototypes]
830 | noinline int __add_to_page_cache_locked(struct page *page,
| ^~~~~~~~~~~~~~~~~~~~~~~~~~
--
In file included from include/linux/percpu.h:5,
from include/linux/radix-tree.h:14,
from include/linux/fs.h:15,
from include/linux/backing-dev.h:13,
from mm/truncate.c:12:
include/linux/pagemap.h: In function '__page_cache_add_speculative':
>> include/linux/mmdebug.h:20:2: error: expected ';' before 'do'
20 | do { \
| ^~
include/linux/pagemap.h:217:2: note: in expansion of macro 'VM_BUG_ON_PAGE'
217 | VM_BUG_ON_PAGE(page_count(page) == 0, page);
| ^~~~~~~~~~~~~~ mm/truncate.c: At top level:
mm/truncate.c:531:15: warning: no previous prototype for '__invalidate_mapping_pages' [-Wmissing-prototypes]
531 | unsigned long __invalidate_mapping_pages(struct address_space *mapping,
| ^~~~~~~~~~~~~~~~~~~~~~~~~~
--
In file included from include/linux/percpu.h:5,
from include/linux/hrtimer.h:19,
from include/linux/sched.h:20,
from include/linux/cgroup.h:12,
from include/linux/memcontrol.h:13,
from mm/workingset.c:8:
include/linux/pagemap.h: In function '__page_cache_add_speculative':
>> include/linux/mmdebug.h:20:2: error: expected ';' before 'do'
20 | do { \
| ^~
include/linux/pagemap.h:217:2: note: in expansion of macro 'VM_BUG_ON_PAGE'
217 | VM_BUG_ON_PAGE(page_count(page) == 0, page);
| ^~~~~~~~~~~~~~
mm/workingset.c: In function 'unpack_shadow':
mm/workingset.c:201:15: warning: variable 'nid' set but not used [-Wunused-but-set-variable]
201 | int memcgid, nid;
| ^~~
--
In file included from include/linux/mm.h:9,
from mm/vmalloc.c:12:
include/linux/pagemap.h: In function '__page_cache_add_speculative':
>> include/linux/mmdebug.h:20:2: error: expected ';' before 'do'
20 | do { \
| ^~
include/linux/pagemap.h:217:2: note: in expansion of macro 'VM_BUG_ON_PAGE'
217 | VM_BUG_ON_PAGE(page_count(page) == 0, page);
| ^~~~~~~~~~~~~~ mm/vmalloc.c: At top level:
mm/vmalloc.c:1318:6: warning: no previous prototype for 'set_iounmap_nonlazy' [-Wmissing-prototypes]
1318 | void set_iounmap_nonlazy(void)
| ^~~~~~~~~~~~~~~~~~~
--
In file included from include/linux/mm.h:9,
from mm/page_alloc.c:19:
include/linux/pagemap.h: In function '__page_cache_add_speculative':
>> include/linux/mmdebug.h:20:2: error: expected ';' before 'do'
20 | do { \
| ^~
include/linux/pagemap.h:217:2: note: in expansion of macro 'VM_BUG_ON_PAGE'
217 | VM_BUG_ON_PAGE(page_count(page) == 0, page);
| ^~~~~~~~~~~~~~ mm/page_alloc.c: At top level:
mm/page_alloc.c:3558:15: warning: no previous prototype for 'should_fail_alloc_page' [-Wmissing-prototypes]
3558 | noinline bool should_fail_alloc_page(gfp_t gfp_mask, unsigned int order)
| ^~~~~~~~~~~~~~~~~~~~~~
mm/page_alloc.c:6183:23: warning: no previous prototype for 'memmap_init' [-Wmissing-prototypes]
6183 | void __meminit __weak memmap_init(unsigned long size, int nid,
| ^~~~~~~~~~~
--
In file included from include/linux/percpu.h:5,
from include/linux/hrtimer.h:19,
from include/linux/sched.h:20,
from include/linux/cgroup.h:12,
from include/linux/memcontrol.h:13,
from mm/memcontrol.c:26:
include/linux/pagemap.h: In function '__page_cache_add_speculative':
>> include/linux/mmdebug.h:20:2: error: expected ';' before 'do'
20 | do { \
| ^~
include/linux/pagemap.h:217:2: note: in expansion of macro 'VM_BUG_ON_PAGE'
217 | VM_BUG_ON_PAGE(page_count(page) == 0, page);
| ^~~~~~~~~~~~~~
In file included from arch/xtensa/include/asm/atomic.h:19,
from include/linux/atomic.h:7,
from include/linux/page_counter.h:5,
from mm/memcontrol.c:25:
mm/memcontrol.c: In function 'memcg_reparent_objcgs':
arch/xtensa/include/asm/cmpxchg.h:173:3: warning: value computed is not used [-Wunused-value]
173 | ((__typeof__(*(ptr)))__xchg((unsigned long)(x),(ptr),sizeof(*(ptr))))
| ~^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
mm/memcontrol.c:328:2: note: in expansion of macro 'xchg'
328 | xchg(&objcg->memcg, parent);
| ^~~~
arch/xtensa/include/asm/cmpxchg.h:173:3: warning: value computed is not used [-Wunused-value]
173 | ((__typeof__(*(ptr)))__xchg((unsigned long)(x),(ptr),sizeof(*(ptr))))
| ~^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
mm/memcontrol.c:335:3: note: in expansion of macro 'xchg'
335 | xchg(&iter->memcg, parent);
| ^~~~
--
In file included from include/linux/percpu.h:5,
from include/linux/hrtimer.h:19,
from include/linux/sched.h:20,
from include/linux/ratelimit.h:6,
from include/linux/dev_printk.h:16,
from include/linux/device.h:15,
from include/sound/core.h:10,
from drivers/gpu/drm/bridge/adv7511/adv7511_audio.c:9:
include/linux/pagemap.h: In function '__page_cache_add_speculative':
>> include/linux/mmdebug.h:20:2: error: expected ';' before 'do'
20 | do { \
| ^~
include/linux/pagemap.h:217:2: note: in expansion of macro 'VM_BUG_ON_PAGE'
217 | VM_BUG_ON_PAGE(page_count(page) == 0, page);
| ^~~~~~~~~~~~~~ drivers/gpu/drm/bridge/adv7511/adv7511_audio.c: At top level:
drivers/gpu/drm/bridge/adv7511/adv7511_audio.c:58:5: warning: no previous prototype for 'adv7511_hdmi_hw_params' [-Wmissing-prototypes]
58 | int adv7511_hdmi_hw_params(struct device *dev, void *data,
| ^~~~~~~~~~~~~~~~~~~~~~
--
In file included from include/linux/percpu.h:5,
from include/linux/bpf.h:11,
from security/security.c:13:
include/linux/pagemap.h: In function '__page_cache_add_speculative':
>> include/linux/mmdebug.h:20:2: error: expected ';' before 'do'
20 | do { \
| ^~
include/linux/pagemap.h:217:2: note: in expansion of macro 'VM_BUG_ON_PAGE'
217 | VM_BUG_ON_PAGE(page_count(page) == 0, page);
| ^~~~~~~~~~~~~~
At top level:
include/linux/lsm_hook_defs.h:394:18: warning: 'perf_event_write_default' defined but not used [-Wunused-const-variable=]
394 | LSM_HOOK(int, 0, perf_event_write, struct perf_event *event)
| ^~~~~~~~~~~~~~~~
security/security.c:681:32: note: in definition of macro 'LSM_RET_DEFAULT'
681 | #define LSM_RET_DEFAULT(NAME) (NAME##_default)
| ^~~~
security/security.c:686:2: note: in expansion of macro 'DECLARE_LSM_RET_DEFAULT_int'
686 | DECLARE_LSM_RET_DEFAULT_##RET(DEFAULT, NAME)
| ^~~~~~~~~~~~~~~~~~~~~~~~
include/linux/lsm_hook_defs.h:394:1: note: in expansion of macro 'LSM_HOOK'
394 | LSM_HOOK(int, 0, perf_event_write, struct perf_event *event)
| ^~~~~~~~
include/linux/lsm_hook_defs.h:393:18: warning: 'perf_event_read_default' defined but not used [-Wunused-const-variable=]
393 | LSM_HOOK(int, 0, perf_event_read, struct perf_event *event)
| ^~~~~~~~~~~~~~~
security/security.c:681:32: note: in definition of macro 'LSM_RET_DEFAULT'
681 | #define LSM_RET_DEFAULT(NAME) (NAME##_default)
| ^~~~
security/security.c:686:2: note: in expansion of macro 'DECLARE_LSM_RET_DEFAULT_int'
686 | DECLARE_LSM_RET_DEFAULT_##RET(DEFAULT, NAME)
| ^~~~~~~~~~~~~~~~~~~~~~~~
include/linux/lsm_hook_defs.h:393:1: note: in expansion of macro 'LSM_HOOK'
393 | LSM_HOOK(int, 0, perf_event_read, struct perf_event *event)
| ^~~~~~~~
include/linux/lsm_hook_defs.h:391:18: warning: 'perf_event_alloc_default' defined but not used [-Wunused-const-variable=]
391 | LSM_HOOK(int, 0, perf_event_alloc, struct perf_event *event)
| ^~~~~~~~~~~~~~~~
security/security.c:681:32: note: in definition of macro 'LSM_RET_DEFAULT'
681 | #define LSM_RET_DEFAULT(NAME) (NAME##_default)
| ^~~~
security/security.c:686:2: note: in expansion of macro 'DECLARE_LSM_RET_DEFAULT_int'
686 | DECLARE_LSM_RET_DEFAULT_##RET(DEFAULT, NAME)
| ^~~~~~~~~~~~~~~~~~~~~~~~
include/linux/lsm_hook_defs.h:391:1: note: in expansion of macro 'LSM_HOOK'
391 | LSM_HOOK(int, 0, perf_event_alloc, struct perf_event *event)
| ^~~~~~~~
include/linux/lsm_hook_defs.h:390:18: warning: 'perf_event_open_default' defined but not used [-Wunused-const-variable=]
390 | LSM_HOOK(int, 0, perf_event_open, struct perf_event_attr *attr, int type)
| ^~~~~~~~~~~~~~~
security/security.c:681:32: note: in definition of macro 'LSM_RET_DEFAULT'
681 | #define LSM_RET_DEFAULT(NAME) (NAME##_default)
| ^~~~
security/security.c:686:2: note: in expansion of macro 'DECLARE_LSM_RET_DEFAULT_int'
686 | DECLARE_LSM_RET_DEFAULT_##RET(DEFAULT, NAME)
| ^~~~~~~~~~~~~~~~~~~~~~~~
include/linux/lsm_hook_defs.h:390:1: note: in expansion of macro 'LSM_HOOK'
390 | LSM_HOOK(int, 0, perf_event_open, struct perf_event_attr *attr, int type)
| ^~~~~~~~
include/linux/lsm_hook_defs.h:387:18: warning: 'locked_down_default' defined but not used [-Wunused-const-variable=]
387 | LSM_HOOK(int, 0, locked_down, enum lockdown_reason what)
| ^~~~~~~~~~~
security/security.c:681:32: note: in definition of macro 'LSM_RET_DEFAULT'
681 | #define LSM_RET_DEFAULT(NAME) (NAME##_default)
| ^~~~
security/security.c:686:2: note: in expansion of macro 'DECLARE_LSM_RET_DEFAULT_int'
686 | DECLARE_LSM_RET_DEFAULT_##RET(DEFAULT, NAME)
| ^~~~~~~~~~~~~~~~~~~~~~~~
include/linux/lsm_hook_defs.h:387:1: note: in expansion of macro 'LSM_HOOK'
387 | LSM_HOOK(int, 0, locked_down, enum lockdown_reason what)
| ^~~~~~~~
include/linux/lsm_hook_defs.h:383:18: warning: 'bpf_prog_alloc_security_default' defined but not used [-Wunused-const-variable=]
383 | LSM_HOOK(int, 0, bpf_prog_alloc_security, struct bpf_prog_aux *aux)
| ^~~~~~~~~~~~~~~~~~~~~~~
security/security.c:681:32: note: in definition of macro 'LSM_RET_DEFAULT'
681 | #define LSM_RET_DEFAULT(NAME) (NAME##_default)
| ^~~~
security/security.c:686:2: note: in expansion of macro 'DECLARE_LSM_RET_DEFAULT_int'
686 | DECLARE_LSM_RET_DEFAULT_##RET(DEFAULT, NAME)
| ^~~~~~~~~~~~~~~~~~~~~~~~
include/linux/lsm_hook_defs.h:383:1: note: in expansion of macro 'LSM_HOOK'
383 | LSM_HOOK(int, 0, bpf_prog_alloc_security, struct bpf_prog_aux *aux)
| ^~~~~~~~
include/linux/lsm_hook_defs.h:381:18: warning: 'bpf_map_alloc_security_default' defined but not used [-Wunused-const-variable=]
381 | LSM_HOOK(int, 0, bpf_map_alloc_security, struct bpf_map *map)
| ^~~~~~~~~~~~~~~~~~~~~~
security/security.c:681:32: note: in definition of macro 'LSM_RET_DEFAULT'
681 | #define LSM_RET_DEFAULT(NAME) (NAME##_default)
| ^~~~
security/security.c:686:2: note: in expansion of macro 'DECLARE_LSM_RET_DEFAULT_int'
686 | DECLARE_LSM_RET_DEFAULT_##RET(DEFAULT, NAME)
| ^~~~~~~~~~~~~~~~~~~~~~~~
include/linux/lsm_hook_defs.h:381:1: note: in expansion of macro 'LSM_HOOK'
381 | LSM_HOOK(int, 0, bpf_map_alloc_security, struct bpf_map *map)
| ^~~~~~~~
include/linux/lsm_hook_defs.h:380:18: warning: 'bpf_prog_default' defined but not used [-Wunused-const-variable=]
380 | LSM_HOOK(int, 0, bpf_prog, struct bpf_prog *prog)
| ^~~~~~~~
security/security.c:681:32: note: in definition of macro 'LSM_RET_DEFAULT'
681 | #define LSM_RET_DEFAULT(NAME) (NAME##_default)
| ^~~~
security/security.c:686:2: note: in expansion of macro 'DECLARE_LSM_RET_DEFAULT_int'
686 | DECLARE_LSM_RET_DEFAULT_##RET(DEFAULT, NAME)
| ^~~~~~~~~~~~~~~~~~~~~~~~
include/linux/lsm_hook_defs.h:380:1: note: in expansion of macro 'LSM_HOOK'
--
In file included from include/linux/gfp.h:5,
from include/linux/umh.h:4,
from include/linux/kmod.h:9,
from include/linux/module.h:16,
from drivers/staging/vt6655/device.h:19,
from drivers/staging/vt6655/device_main.c:37:
include/linux/pagemap.h: In function '__page_cache_add_speculative':
>> include/linux/mmdebug.h:20:2: error: expected ';' before 'do'
20 | do { \
| ^~
include/linux/pagemap.h:217:2: note: in expansion of macro 'VM_BUG_ON_PAGE'
217 | VM_BUG_ON_PAGE(page_count(page) == 0, page);
| ^~~~~~~~~~~~~~
In file included from drivers/staging/vt6655/device_main.c:43:
drivers/staging/vt6655/rxtx.h: At top level:
drivers/staging/vt6655/rxtx.h:114:1: warning: alignment 1 of 'struct vnt_rts_g' is less than 2 [-Wpacked-not-aligned]
114 | } __packed;
| ^
drivers/staging/vt6655/rxtx.h:128:1: warning: alignment 1 of 'struct vnt_rts_g_fb' is less than 2 [-Wpacked-not-aligned]
128 | } __packed;
| ^
drivers/staging/vt6655/rxtx.h:135:1: warning: alignment 1 of 'struct vnt_rts_ab' is less than 2 [-Wpacked-not-aligned]
135 | } __packed;
| ^
drivers/staging/vt6655/rxtx.h:144:1: warning: alignment 1 of 'struct vnt_rts_a_fb' is less than 2 [-Wpacked-not-aligned]
144 | } __packed;
| ^
drivers/staging/vt6655/rxtx.h:153:1: warning: alignment 1 of 'struct vnt_cts' is less than 2 [-Wpacked-not-aligned]
153 | } __packed;
| ^
drivers/staging/vt6655/rxtx.h:163:1: warning: alignment 1 of 'struct vnt_cts_fb' is less than 2 [-Wpacked-not-aligned]
163 | } __packed;
| ^
--
In file included from include/linux/gfp.h:5,
from include/linux/umh.h:4,
from include/linux/kmod.h:9,
from include/linux/module.h:16,
from drivers/staging/vt6655/device.h:19,
from drivers/staging/vt6655/upc.h:19,
from drivers/staging/vt6655/srom.c:32:
include/linux/pagemap.h: In function '__page_cache_add_speculative':
>> include/linux/mmdebug.h:20:2: error: expected ';' before 'do'
20 | do { \
| ^~
include/linux/pagemap.h:217:2: note: in expansion of macro 'VM_BUG_ON_PAGE'
217 | VM_BUG_ON_PAGE(page_count(page) == 0, page);
| ^~~~~~~~~~~~~~
In file included from drivers/staging/vt6655/srom.c:32:
drivers/staging/vt6655/srom.c: In function 'SROMbyReadEmbedded':
drivers/staging/vt6655/upc.h:45:16: warning: variable 'byData' set but not used [-Wunused-but-set-variable]
45 | unsigned char byData; \
| ^~~~~~
drivers/staging/vt6655/srom.c:85:4: note: in expansion of macro 'PCAvDelayByIO'
85 | PCAvDelayByIO(CB_DELAY_LOOP_WAIT);
| ^~~~~~~~~~~~~..
vim +20 include/linux/mmdebug.h
309381feaee5642 Sasha Levin 2014-01-23 16
59ea746337c69f6 Jiri Slaby 2008-06-12 17 #ifdef CONFIG_DEBUG_VM
59ea746337c69f6 Jiri Slaby 2008-06-12 18 #define VM_BUG_ON(cond) BUG_ON(cond)
309381feaee5642 Sasha Levin 2014-01-23 19 #define VM_BUG_ON_PAGE(cond, page) \
e4f674229ce63da Dave Hansen 2014-06-04 @20 do { \
e4f674229ce63da Dave Hansen 2014-06-04 21 if (unlikely(cond)) { \
e4f674229ce63da Dave Hansen 2014-06-04 22 dump_page(page, "VM_BUG_ON_PAGE(" __stringify(cond)")");\
e4f674229ce63da Dave Hansen 2014-06-04 23 BUG(); \
e4f674229ce63da Dave Hansen 2014-06-04 24 } \
e4f674229ce63da Dave Hansen 2014-06-04 25 } while (0)
fa3759ccd5651c4 Sasha Levin 2014-10-09 26 #define VM_BUG_ON_VMA(cond, vma) \
fa3759ccd5651c4 Sasha Levin 2014-10-09 27 do { \
fa3759ccd5651c4 Sasha Levin 2014-10-09 28 if (unlikely(cond)) { \
fa3759ccd5651c4 Sasha Levin 2014-10-09 29 dump_vma(vma); \
fa3759ccd5651c4 Sasha Levin 2014-10-09 30 BUG(); \
fa3759ccd5651c4 Sasha Levin 2014-10-09 31 } \
fa3759ccd5651c4 Sasha Levin 2014-10-09 32 } while (0)
31c9afa6db122a5 Sasha Levin 2014-10-09 33 #define VM_BUG_ON_MM(cond, mm) \
31c9afa6db122a5 Sasha Levin 2014-10-09 34 do { \
31c9afa6db122a5 Sasha Levin 2014-10-09 35 if (unlikely(cond)) { \
31c9afa6db122a5 Sasha Levin 2014-10-09 36 dump_mm(mm); \
31c9afa6db122a5 Sasha Levin 2014-10-09 37 BUG(); \
31c9afa6db122a5 Sasha Levin 2014-10-09 38 } \
31c9afa6db122a5 Sasha Levin 2014-10-09 39 } while (0)
91241681c62a5a6 Michal Hocko 2018-04-05 40 #define VM_WARN_ON(cond) (void)WARN_ON(cond)
91241681c62a5a6 Michal Hocko 2018-04-05 41 #define VM_WARN_ON_ONCE(cond) (void)WARN_ON_ONCE(cond)
91241681c62a5a6 Michal Hocko 2018-04-05 42 #define VM_WARN_ONCE(cond, format...) (void)WARN_ONCE(cond, format)
91241681c62a5a6 Michal Hocko 2018-04-05 43 #define VM_WARN(cond, format...) (void)WARN(cond, format)
59ea746337c69f6 Jiri Slaby 2008-06-12 44 #else
02602a18c32d76f Konstantin Khlebnikov 2012-05-29 45 #define VM_BUG_ON(cond) BUILD_BUG_ON_INVALID(cond)
309381feaee5642 Sasha Levin 2014-01-23 46 #define VM_BUG_ON_PAGE(cond, page) VM_BUG_ON(cond)
fa3759ccd5651c4 Sasha Levin 2014-10-09 47 #define VM_BUG_ON_VMA(cond, vma) VM_BUG_ON(cond)
31c9afa6db122a5 Sasha Levin 2014-10-09 48 #define VM_BUG_ON_MM(cond, mm) VM_BUG_ON(cond)
02a8efeda894d35 Andrew Morton 2014-06-04 49 #define VM_WARN_ON(cond) BUILD_BUG_ON_INVALID(cond)
02a8efeda894d35 Andrew Morton 2014-06-04 50 #define VM_WARN_ON_ONCE(cond) BUILD_BUG_ON_INVALID(cond)
ef6b571fb8920d5 Andrew Morton 2014-08-06 51 #define VM_WARN_ONCE(cond, format...) BUILD_BUG_ON_INVALID(cond)
a54f9aebaa9f0ea Aneesh Kumar K.V 2016-07-26 52 #define VM_WARN(cond, format...) BUILD_BUG_ON_INVALID(cond)
59ea746337c69f6 Jiri Slaby 2008-06-12 53 #endif
59ea746337c69f6 Jiri Slaby 2008-06-12 54 ---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/kbuild-all(a)lists.01.org
Quoted 22 lines · Uladzislau Rezki (Sony)
The current memmory-allocation interface presents to following difficulties that this patch is designed to overcome:
a) If built with
CONFIG_PROVE_RAW_LOCK_NESTING, the lockdep will complain aboutviolation("BUG: Invalid wait context")of the nesting rules. It does theraw_spinlockvs. spinlock nesting checks, i.e. it is not legal to acquire aspinlock_twhile holding araw_spinlock_t.Internally the kfree_rcu() uses raw_spinlock_t whereas the "page allocator" internally deals with spinlock_t to access to its zones. The code also can be broken from higher level of view: <snip> raw_spin_lock(&some_lock); kfree_rcu(some_pointer, some_field_offset); <snip>b) If built with
CONFIG_PREEMPT_RT. Please note, in that casespinlock_tis converted into sleepable variant. Invoking the page allocator from atomic contexts leads to "BUG: scheduling while atomic".
Quoted 8 lines
c)
call_rcu()is invoked from raw atomic context andkfree_rcu()andkvfree_rcu()are expected to be called from atomic raw context as well.Move out a page allocation from contexts which trigger
kvfree_rcu()function to the separate worker. When a k[v]free_rcu() per-cpu page cache is empty a fallback mechanism is used and a special job is scheduled to refill the per-cpu cache.
Looks good, still reviewing here. BTW just for my education, I was wondering about Thomas's email:
https://lkml.org/lkml/2020/8/11/939
If slab allocations in pure raw-atomic context on RT is not allowed or recommended, should kfree_rcu() be allowed?
slab can have same issue right? If per-cpu cache is drained, it has to allocate page from buddy allocator and there's no GFP flag to tell it about context where alloc is happening from.
Or are we saying that we want to support kfree on RT from raw atomic atomic context, even though kmalloc is not supported? I hate to bring up this elephant in the room, but since I am a part of the people maintaining this code, I believe I would rather set some rules than supporting unsupported usages. :-\ (Once I know what is supported and what isn't that is). If indeed raw atomic kfree_rcu() is a bogus use case because of -RT, then we ought to put a giant warning than supporting it :-(.
But I don't stand in the way of this patch if all agree on it. thanks,
- Joel
Quoted 192 lines
Fixes 3042f83f19be ("rcu: Support reclaim for head-less object")Reported-by Sebastian Andrzej Siewior <[email protected]>Signed-off-by Uladzislau Rezki (Sony) <[email protected]>--- kernel/rcu/tree.c | 109 ++++++++++++++++++++++++++++------------------ 1 file changed, 66 insertions(+), 43 deletions(-)kernel/rcu/tree.c +66
-43@@ -177,7 +177,7 @@ module_param(rcu_unlock_delay, int, 0444);* per-CPU. Object size is equal to one page. This value * can be changed at boot time. */ -static int rcu_min_cached_objs =2 ; +static int rcu_min_cached_objs =5 ;module_param (rcu_min_cached_objs, int,0444 );/* Retrieve RCU kthreads priority for rcutorture */ @@ -3084,6 +3084,9 @@ struct kfree_rcu_cpu_work {* In order to save some per-cpu space the list is singular. * Even though it is lockless an access has to be protected by the * per-cpu lock. +* @page_cache_work: A work to refill the cache when it is empty +* @work_in_progress: Indicates that page_cache_work is running +* @hrtimer: A hrtimer for scheduling a page_cache_work * @nr_bkv_objs: number of allocated objects at @bkvcache. * * This is a per-CPU structure. The reason that it is not included in @@ -3100,6 +3103,11 @@ struct kfree_rcu_cpu {bool monitor_todo;bool initialized;int count; + +struct work_struct page_cache_work; +atomic_t work_in_progress; +struct hrtimer hrtimer; +struct llist_head bkvcache;int nr_bkv_objs; };@@ -3217,10 +3225,10 @@ static void kfree_rcu_work(struct work_struct *work) }rcu_lock_release (&rcu_callback_map); - krcp =krc_this_cpu_lock (&flags); +raw_spin_lock_irqsave (&krcp->lock, flags);if (put_cached_bnode (krcp, bkvhead[i])) bkvhead[i] =NULL ; -krc_this_cpu_unlock (krcp, flags); +raw_spin_unlock_irqrestore (&krcp->lock, flags);if (bkvhead[i])free_page ((unsigned long ) bkvhead[i]);@@ -3347,6 +3355,57 @@ static void kfree_rcu_monitor(struct work_struct *work)raw_spin_unlock_irqrestore (&krcp->lock, flags); } +static enum hrtimer_restart +schedule_page_work_fn (struct hrtimer *t) +{ +struct kfree_rcu_cpu *krcp = + container_of(t, struct kfree_rcu_cpu, hrtimer); + +queue_work (system_highpri_wq, &krcp->page_cache_work); +return HRTIMER_NORESTART ; +} + +static void fill_page_cache_func (struct work_struct *work) +{ +struct kvfree_rcu_bulk_data *bnode; +struct kfree_rcu_cpu *krcp = + container_of(work, struct kfree_rcu_cpu, + page_cache_work); +unsigned long flags; +bool pushed; +int i; + +for (i =0 ; i < rcu_min_cached_objs; i++) { + bnode = (struct kvfree_rcu_bulk_data *) +__get_free_page (GFP_KERNEL | __GFP_NOWARN); + +if (bnode) { +raw_spin_lock_irqsave (&krcp->lock, flags); + pushed =put_cached_bnode (krcp, bnode); +raw_spin_unlock_irqrestore (&krcp->lock, flags); + +if (!pushed) { +free_page ((unsigned long ) bnode); +break ; + } + } + } + +atomic_set (&krcp->work_in_progress,0 ); +} + +static void +run_page_cache_worker (struct kfree_rcu_cpu *krcp) +{ +if (rcu_scheduler_active ==RCU_SCHEDULER_RUNNING && + !atomic_xchg (&krcp->work_in_progress,1 )) { +hrtimer_init (&krcp->hrtimer,CLOCK_MONOTONIC , +HRTIMER_MODE_REL ); + krcp->hrtimer.function = schedule_page_work_fn; +hrtimer_start (&krcp->hrtimer,0 ,HRTIMER_MODE_REL ); + } +} +static inline bool kvfree_call_rcu_add_ptr_to_bulk (struct kfree_rcu_cpu *krcp,void *ptr) {@@ -3363,32 +3422,8 @@ kvfree_call_rcu_add_ptr_to_bulk(struct kfree_rcu_cpu *krcp, void *ptr)if (!krcp->bkvhead[idx] || krcp->bkvhead[idx]->nr_records ==KVFREE_BULK_MAX_ENTR ) { bnode =get_cached_bnode (krcp); -if (!bnode) { -/* -* To keep this path working on raw non-preemptible -* sections, prevent the optional entry into the -* allocator as it uses sleeping locks. In fact, even -* if the caller of kfree_rcu() is preemptible, this -* path still is not, as krcp->lock is a raw spinlock. -* With additional page pre-allocation in the works, -* hitting this return is going to be much less likely. -*/ -if (IS_ENABLED (CONFIG_PREEMPT_RT )) -return false; - -/* -* NOTE: For one argument of kvfree_rcu() we can -* drop the lock and get the page in sleepable -* context. That would allow to maintain an array -* for the CONFIG_PREEMPT_RT as well if no cached -* pages are available. -*/ - bnode = (struct kvfree_rcu_bulk_data *) -__get_free_page (GFP_NOWAIT | __GFP_NOWARN); - } -/* Switch to emergency path. */ -if (unlikely (!bnode)) +if (!bnode)return false;/* Initialize the new block. */ @@ -3452,12 +3487,10 @@ void kvfree_call_rcu(struct rcu_head *head, rcu_callback_t func) goto unlock_return; } -/* -* Under high memory pressure GFP_NOWAIT can fail, -* in that case the emergency path is maintained. -*/ success =kvfree_call_rcu_add_ptr_to_bulk (krcp, ptr);if (!success) { +run_page_cache_worker (krcp); +if (head ==NULL )// Inline if kvfree_rcu(one_arg) call. goto unlock_return;@@ -4449,24 +4482,14 @@ static void __init kfree_rcu_batch_init(void)for_each_possible_cpu (cpu) {struct kfree_rcu_cpu *krcp =per_cpu_ptr (&krc, cpu); -struct kvfree_rcu_bulk_data *bnode;for (i =0 ; i <KFREE_N_BATCHES ; i++) {INIT_RCU_WORK (&krcp->krw_arr[i].rcu_work, kfree_rcu_work); krcp->krw_arr[i].krcp = krcp; } -for (i =0 ; i < rcu_min_cached_objs; i++) { - bnode = (struct kvfree_rcu_bulk_data *) -__get_free_page (GFP_NOWAIT | __GFP_NOWARN); - -if (bnode) -put_cached_bnode (krcp, bnode); -else -pr_err ("Failed to preallocate for %d CPU!\n" , cpu); - } -INIT_DELAYED_WORK (&krcp->monitor_work, kfree_rcu_monitor); +INIT_WORK (&krcp->page_cache_work, fill_page_cache_func); krcp->initialized = true; }if (register_shrinker (&kfree_rcu_shrinker))Signature · 1 line
2.20.1
Uladzislau Rezki
maintainerQuoted 32 lines · Joel Fernandes
Quoted 22 lines · Uladzislau Rezki (Sony)
The current memmory-allocation interface presents to following difficulties that this patch is designed to overcome:
a) If built with
CONFIG_PROVE_RAW_LOCK_NESTING, the lockdep will complain aboutviolation("BUG: Invalid wait context")of the nesting rules. It does theraw_spinlockvs. spinlock nesting checks, i.e. it is not legal to acquire aspinlock_twhile holding araw_spinlock_t.Internally the kfree_rcu() uses raw_spinlock_t whereas the "page allocator" internally deals with spinlock_t to access to its zones. The code also can be broken from higher level of view: <snip> raw_spin_lock(&some_lock); kfree_rcu(some_pointer, some_field_offset); <snip>b) If built with
CONFIG_PREEMPT_RT. Please note, in that casespinlock_tis converted into sleepable variant. Invoking the page allocator from atomic contexts leads to "BUG: scheduling while atomic".Quoted 8 lines
c)
call_rcu()is invoked from raw atomic context andkfree_rcu()andkvfree_rcu()are expected to be called from atomic raw context as well.Move out a page allocation from contexts which trigger
kvfree_rcu()function to the separate worker. When a k[v]free_rcu() per-cpu page cache is empty a fallback mechanism is used and a special job is scheduled to refill the per-cpu cache.
Quoted 7 lines
Looks good, still reviewing here. BTW just for my education, I was wondering about Thomas's email:
https://lkml.org/lkml/2020/8/11/939If slab allocations in pure raw-atomic context on RT is not allowed or recommended, should
kfree_rcu()be allowed?
Thanks for reviewing, Joel :)
The decision was made that we need to support kfree_rcu() from "real atomic contexts", to align with how it used to be before. We can go and just convert our local locks to the spinlock_t variant but that was not Paul goal, it can be that some users need kfree_rcu() for raw atomics.
Quoted 5 lines
slab can have same issue right? If per-cpu cache is drained, it has to allocate page from buddy allocator and there's no GFP flag to tell it about context where alloc is happening from.
Sounds like that. Apart of that, it might turn out soon that we or somebody else will rise a question one more time about something GFP_RAW or GFP_NOLOCKS. So who knows..
Quoted 9 lines
Or are we saying that we want to support kfree on RT from raw atomic atomic context, even though kmalloc is not supported? I hate to bring up this elephant in the room, but since I am a part of the people maintaining this code, I believe I would rather set some rules than supporting unsupported usages. :-\ (Once I know what is supported and what isn't that is). If indeed raw atomic
kfree_rcu()is a bogus use case because of -RT, then we ought to put a giant warning than supporting it :-(.
We discussed it several times, the conclusion was that we need to support kfree_rcu() from raw contexts. At least that was a clear signal from Paul to me. I think, if we obtain the preemtable(), so it becomes versatile, we can drop the patch that is in question later on in the future.
--
Vlad Rezki
Quoted 2 lines · Uladzislau Rezki (Sony)
The current memmory-allocation interface presents to following difficulties that this patch is designed to overcome
[...]
Quoted 28 lines
--- kernel/rcu/tree.c | 109 ++++++++++++++++++++++++++++------------------ 1 file changed, 66 insertions(+), 43 deletions(-)kernel/rcu/tree.c +4
-1@@ -177,7 +177,7 @@ module_param(rcu_unlock_delay, int, 0444);* per-CPU. Object size is equal to one page. This value * can be changed at boot time. */ -static int rcu_min_cached_objs =2 ; +static int rcu_min_cached_objs =5 ;module_param (rcu_min_cached_objs, int,0444 );/* Retrieve RCU kthreads priority for rcutorture */ @@ -3084,6 +3084,9 @@ struct kfree_rcu_cpu_work {* In order to save some per-cpu space the list is singular. * Even though it is lockless an access has to be protected by the * per-cpu lock. +* @page_cache_work: A work to refill the cache when it is empty +* @work_in_progress: Indicates that page_cache_work is running +* @hrtimer: A hrtimer for scheduling a page_cache_work * @nr_bkv_objs: number of allocated objects at @bkvcache. * * This is a per-CPU structure. The reason that it is not included in
kernel/rcu/tree.c struct kfree_rcu_cpu { · line 3108
Quoted 6 lines
kernel/rcu/tree.c +3
-0@@ -3100,6 +3103,11 @@ struct kfree_rcu_cpu {bool monitor_todo;bool initialized;int count; + +struct work_struct page_cache_work; +atomic_t work_in_progress;
Does it need to be atomic? run_page_cache_work() is only called under a lock. You can use xchg() there. And when you do the atomic_set, you can use WRITE_ONCE as it is a data-race.
Quoted 14 lines
kernel/rcu/tree.c +0
-4@@ -4449,24 +4482,14 @@ static void __init kfree_rcu_batch_init(void)for_each_possible_cpu (cpu) {struct kfree_rcu_cpu *krcp =per_cpu_ptr (&krc, cpu); -struct kvfree_rcu_bulk_data *bnode;for (i =0 ; i <KFREE_N_BATCHES ; i++) {INIT_RCU_WORK (&krcp->krw_arr[i].rcu_work, kfree_rcu_work); krcp->krw_arr[i].krcp = krcp; } -for (i =0 ; i < rcu_min_cached_objs; i++) { - bnode = (struct kvfree_rcu_bulk_data *) -__get_free_page (GFP_NOWAIT | __GFP_NOWARN);
kernel/rcu/tree.c static void __init kfree_rcu_batch_init(void) · line 4493
Quoted 10 lines
kernel/rcu/tree.c +1
-7- -if (bnode) -put_cached_bnode (krcp, bnode); -else -pr_err ("Failed to preallocate for %d CPU!\n" , cpu); - } -INIT_DELAYED_WORK (&krcp->monitor_work, kfree_rcu_monitor); +INIT_WORK (&krcp->page_cache_work, fill_page_cache_func); krcp->initialized = true;
During initialization, is it not better to still pre-allocate? That way you don't have to wait to get into a situation where you need to initially allocate.
AFAICS after the above line deletions, the bulk pages are initially empty.
thanks,
- Joel
Quoted 5 lines
kernel/rcu/tree.c +0
-0}if (register_shrinker (&kfree_rcu_shrinker))Signature · 1 line
2.20.1
Re: [PATCH 15/16] rcu/tree: Allocate a page when caller is preemptible
In reply to Uladzislau Rezki (Sony)Hi Vlad,
Few minor nits:
Quoted 51 lines · Uladzislau Rezki (Sony)
Given that
CONFIG_PREEMPT_COUNTis unconditionally enabled by the earlier commits in this series, thepreemptible()macro now properly detects preempt-disable code regions even in kernels built withCONFIG_PREEMPT_NONE.This commit therefore uses
preemptible()to determine whether allocation is possible at all for double-argumentkvfree_rcu(). If !preemptible(), then allocation is not possible, andkvfree_rcu()falls back to using the less cache-friendlyrcu_headapproach. Even whenpreemptible(), the caller might be involved in reclaim, so theGFP_flags used by double-argumentkvfree_rcu()must avoid invoking reclaim processing.Note that single-argument
kvfree_rcu()must be invoked in sleepable contexts, and that its fallback is the relatively high latencysynchronize_rcu(). Single-argumentkvfree_rcu()therefore uses GFP_KERNEL|__GFP_RETRY_MAYFAIL to allow limited sleeping within the memory allocator.[ paulmck: Add add_ptr_to_bulk_krc_lock header comment per Michal Hocko. ]Signed-off-by Uladzislau Rezki (Sony) <[email protected]>Signed-off-by Paul E. McKenney <[email protected]>--- kernel/rcu/tree.c | 48 ++++++++++++++++++++++++++++++----------------- 1 file changed, 31 insertions(+), 17 deletions(-)kernel/rcu/tree.c +13
-1@@ -3406,37 +3406,55 @@ run_page_cache_worker(struct kfree_rcu_cpu *krcp) } } +// Record ptr in a page managed by krcp, with the pre-krc_this_cpu_lock() +// state specified by flags. If can_sleep is true, the caller must +// be schedulable and not be holding any locks or mutexes that might be +// acquired by the memory allocator or anything that it might invoke. +// If !can_sleep, then if !preemptible() no allocation will be undertaken, +// otherwise the allocation will use GFP_ATOMIC to avoid the remainder of +// the aforementioned deadlock possibilities. Returns true if ptr was +// successfully recorded, else the caller must use a fallback. static inline bool -kvfree_call_rcu_add_ptr_to_bulk (struct kfree_rcu_cpu *krcp,void *ptr) +add_ptr_to_bulk_krc_lock (struct kfree_rcu_cpu **krcp, +unsigned long *flags,void *ptr,bool can_sleep) {struct kvfree_rcu_bulk_data *bnode; +bool can_alloc_page =preemptible (); +gfp_t gfp = (can_sleep ?GFP_KERNEL | __GFP_RETRY_MAYFAIL : +GFP_ATOMIC ) | __GFP_NOWARN;int idx;
kernel/rcu/tree.c run_page_cache_worker(struct kfree_rcu_cpu *krcp) · line 3433
Quoted 10 lines
kernel/rcu/tree.c +2
-2-if (unlikely (!krcp->initialized)) + *krcp =krc_this_cpu_lock (flags); +if (unlikely (!(*krcp)->initialized))return false; -lockdep_assert_held (&krcp->lock); idx = !!is_vmalloc_addr (ptr);/* Check if a new block is required. */
Maybe convert this comment also to //... like the new ones you added (and the ones below).
kernel/rcu/tree.c run_page_cache_worker(struct kfree_rcu_cpu *krcp) · line 3437
Quoted 8 lines
kernel/rcu/tree.c +4
-4-if (!krcp->bkvhead[idx] || - krcp->bkvhead[idx]->nr_records ==KVFREE_BULK_MAX_ENTR ) { - bnode =get_cached_bnode (krcp); -/* Switch to emergency path. */ +if (!(*krcp)->bkvhead[idx] || + (*krcp)->bkvhead[idx]->nr_records ==KVFREE_BULK_MAX_ENTR ) { + bnode =get_cached_bnode (*krcp); +if (!bnode && can_alloc_page) {
I think you can directly put preemptible() here with a comment saying allocate only if preemptible and get rid of can_alloc_page.
kernel/rcu/tree.c run_page_cache_worker(struct kfree_rcu_cpu *krcp) · line 3443
Quoted 6 lines
kernel/rcu/tree.c +6
-0+krc_this_cpu_unlock (*krcp, *flags); + bnode = (struct kvfree_rcu_bulk_data *) +__get_free_page (gfp); + *krcp =krc_this_cpu_lock (flags); + } +
I think the "Switch to emergency path" comment should stay here before the if().
thanks,
- Joel
Quoted 47 lines
kernel/rcu/tree.c +6
-10if (!bnode)return false;/* Initialize the new block. */ bnode->nr_records =0 ; - bnode->next = krcp->bkvhead[idx]; + bnode->next = (*krcp)->bkvhead[idx];/* Attach it to the head. */ - krcp->bkvhead[idx] = bnode; + (*krcp)->bkvhead[idx] = bnode; }/* Finally insert. */ - krcp->bkvhead[idx]->records - [krcp->bkvhead[idx]->nr_records++] = ptr; + (*krcp)->bkvhead[idx]->records + [(*krcp)->bkvhead[idx]->nr_records++] = ptr;return true; }@@ -3474,20 +3492,16 @@ void kvfree_call_rcu(struct rcu_head *head, rcu_callback_t func) ptr = (unsigned long *) func; } - krcp =krc_this_cpu_lock (&flags); -// Queue the object but don't yet schedule the batch. if (debug_rcu_head_queue (ptr)) {// Probable double kfree_rcu(), just leak. WARN_ONCE (1 ,"%s(): Double-freed call. rcu_head %p\n" , __func__, head); -// Mark as success and leave. - success = true; - goto unlock_return; +return ; } - success =kvfree_call_rcu_add_ptr_to_bulk (krcp, ptr); + success =add_ptr_to_bulk_krc_lock (&krcp, &flags, ptr, !head);if (!success) {run_page_cache_worker (krcp);Signature · 1 line
2.20.1
Paul E. McKenney
maintainerQuoted 40 lines · Uladzislau Rezki
Quoted 32 lines · Joel Fernandes
Quoted 22 lines · Uladzislau Rezki (Sony)
The current memmory-allocation interface presents to following difficulties that this patch is designed to overcome:
a) If built with
CONFIG_PROVE_RAW_LOCK_NESTING, the lockdep will complain aboutviolation("BUG: Invalid wait context")of the nesting rules. It does theraw_spinlockvs. spinlock nesting checks, i.e. it is not legal to acquire aspinlock_twhile holding araw_spinlock_t.Internally the kfree_rcu() uses raw_spinlock_t whereas the "page allocator" internally deals with spinlock_t to access to its zones. The code also can be broken from higher level of view: <snip> raw_spin_lock(&some_lock); kfree_rcu(some_pointer, some_field_offset); <snip>b) If built with
CONFIG_PREEMPT_RT. Please note, in that casespinlock_tis converted into sleepable variant. Invoking the page allocator from atomic contexts leads to "BUG: scheduling while atomic".Quoted 8 lines
c)
call_rcu()is invoked from raw atomic context andkfree_rcu()andkvfree_rcu()are expected to be called from atomic raw context as well.Move out a page allocation from contexts which trigger
kvfree_rcu()function to the separate worker. When a k[v]free_rcu() per-cpu page cache is empty a fallback mechanism is used and a special job is scheduled to refill the per-cpu cache.Quoted 7 lines
Looks good, still reviewing here. BTW just for my education, I was wondering about Thomas's email:
https://lkml.org/lkml/2020/8/11/939If slab allocations in pure raw-atomic context on RT is not allowed or recommended, should
kfree_rcu()be allowed?
Quoted 6 lines
Thanks for reviewing, Joel :)
The decision was made that we need to support
kfree_rcu()from "real atomic contexts", to align with how it used to be before. We can go and just convert our local locks to thespinlock_tvariant but that was not Paul goal, it can be that some users needkfree_rcu()for raw atomics.
People invoke call_rcu() from raw atomics, and so we should provide the same for kfree_rcu(). Yes, people could work around a raw-atomic prohibition, but such prohibitions incur constant costs over time in terms of development effort, increased bug rate, and increased complexity. Yes, this does increase all of those for RCU, but the relative increase is negligible, RCU being what it is.
Quoted 7 lines
Quoted 4 lines
slab can have same issue right? If per-cpu cache is drained, it has to allocate page from buddy allocator and there's no GFP flag to tell it about context where alloc is happening from.
Sounds like that. Apart of that, it might turn out soon that we or somebody else will rise a question one more time about something
GFP_RAWorGFP_NOLOCKS. So who knows..
I would prefer that slab provide some way of dealing with raw atomic context, but the maintainers are thus far unconvinced.
Quoted 8 lines
Quoted 8 lines
Or are we saying that we want to support kfree on RT from raw atomic atomic context, even though kmalloc is not supported? I hate to bring up this elephant in the room, but since I am a part of the people maintaining this code, I believe I would rather set some rules than supporting unsupported usages. :-\ (Once I know what is supported and what isn't that is). If indeed raw atomic
kfree_rcu()is a bogus use case because of -RT, then we ought to put a giant warning than supporting it :-(.
Quoted 4 lines
We discussed it several times, the conclusion was that we need to support
kfree_rcu()from raw contexts. At least that was a clear signal from Paul to me. I think, if we obtain thepreemtable(), so it becomes versatile, we can drop the patch that is in question later on in the future.
Given a universally meaningful preemptible(), we could directly call the allocator in some cases. It might (or might not) still make sense to defer the allocation when preemptible() indicated that a direct call to the allocator was unsafe.
Thanx, Paul
Uladzislau Rezki
maintainerRe: [PATCH 15/16] rcu/tree: Allocate a page when caller is preemptible
In reply to Joel FernandesHello, Joel.
Quoted 59 lines · Uladzislau Rezki (Sony)
Quoted 57 lines
Given that
CONFIG_PREEMPT_COUNTis unconditionally enabled by the earlier commits in this series, thepreemptible()macro now properly detects preempt-disable code regions even in kernels built withCONFIG_PREEMPT_NONE.This commit therefore uses
preemptible()to determine whether allocation is possible at all for double-argumentkvfree_rcu(). If !preemptible(), then allocation is not possible, andkvfree_rcu()falls back to using the less cache-friendlyrcu_headapproach. Even whenpreemptible(), the caller might be involved in reclaim, so theGFP_flags used by double-argumentkvfree_rcu()must avoid invoking reclaim processing.Note that single-argument
kvfree_rcu()must be invoked in sleepable contexts, and that its fallback is the relatively high latencysynchronize_rcu(). Single-argumentkvfree_rcu()therefore uses GFP_KERNEL|__GFP_RETRY_MAYFAIL to allow limited sleeping within the memory allocator.[ paulmck: Add add_ptr_to_bulk_krc_lock header comment per Michal Hocko. ]Signed-off-by Uladzislau Rezki (Sony) <[email protected]>Signed-off-by Paul E. McKenney <[email protected]>--- kernel/rcu/tree.c | 48 ++++++++++++++++++++++++++++++----------------- 1 file changed, 31 insertions(+), 17 deletions(-)kernel/rcu/tree.c +15
-2@@ -3406,37 +3406,55 @@ run_page_cache_worker(struct kfree_rcu_cpu *krcp) } } +// Record ptr in a page managed by krcp, with the pre-krc_this_cpu_lock() +// state specified by flags. If can_sleep is true, the caller must +// be schedulable and not be holding any locks or mutexes that might be +// acquired by the memory allocator or anything that it might invoke. +// If !can_sleep, then if !preemptible() no allocation will be undertaken, +// otherwise the allocation will use GFP_ATOMIC to avoid the remainder of +// the aforementioned deadlock possibilities. Returns true if ptr was +// successfully recorded, else the caller must use a fallback. static inline bool -kvfree_call_rcu_add_ptr_to_bulk (struct kfree_rcu_cpu *krcp,void *ptr) +add_ptr_to_bulk_krc_lock (struct kfree_rcu_cpu **krcp, +unsigned long *flags,void *ptr,bool can_sleep) {struct kvfree_rcu_bulk_data *bnode; +bool can_alloc_page =preemptible (); +gfp_t gfp = (can_sleep ?GFP_KERNEL | __GFP_RETRY_MAYFAIL : +GFP_ATOMIC ) | __GFP_NOWARN;int idx; -if (unlikely (!krcp->initialized)) + *krcp =krc_this_cpu_lock (flags); +if (unlikely (!(*krcp)->initialized))return false;
Quoted 8 lines
kernel/rcu/tree.c run_page_cache_worker(struct kfree_rcu_cpu *krcp) · line 3433
Quoted 4 lines
kernel/rcu/tree.c +0
-1-lockdep_assert_held (&krcp->lock); idx = !!is_vmalloc_addr (ptr);/* Check if a new block is required. */ Maybe convert this comment also to //... like the new ones you added (and the ones below).
No, problem. I can convert it.
Quoted 12 lines
kernel/rcu/tree.c run_page_cache_worker(struct kfree_rcu_cpu *krcp) · line 3437
Quoted 8 lines
kernel/rcu/tree.c +4
-4-if (!krcp->bkvhead[idx] || - krcp->bkvhead[idx]->nr_records ==KVFREE_BULK_MAX_ENTR ) { - bnode =get_cached_bnode (krcp); -/* Switch to emergency path. */ +if (!(*krcp)->bkvhead[idx] || + (*krcp)->bkvhead[idx]->nr_records ==KVFREE_BULK_MAX_ENTR ) { + bnode =get_cached_bnode (*krcp); +if (!bnode && can_alloc_page) {I think you can directly put
preemptible()here with a comment saying allocate only if preemptible and get rid ofcan_alloc_page.
Not really. We check preemtable() before acquiring the internal lock, otherwise it will always return "false". Thus, it is checked on the entry in the beginning.
Quoted 10 lines
kernel/rcu/tree.c run_page_cache_worker(struct kfree_rcu_cpu *krcp) · line 3443
Quoted 6 lines
kernel/rcu/tree.c +6
-0+krc_this_cpu_unlock (*krcp, *flags); + bnode = (struct kvfree_rcu_bulk_data *) +__get_free_page (gfp); + *krcp =krc_this_cpu_lock (flags); + } +I think the "Switch to emergency path" comment should stay here before the
if().
Right. We need to keep it. Will take it back.
--
Vlad Rezki
Uladzislau Rezki
maintainerQuoted 34 lines · Joel Fernandes
Quoted 2 lines · Uladzislau Rezki (Sony)
The current memmory-allocation interface presents to following difficulties that this patch is designed to overcome
[...]
Quoted 30 lines
--- kernel/rcu/tree.c | 109 ++++++++++++++++++++++++++++------------------ 1 file changed, 66 insertions(+), 43 deletions(-)kernel/rcu/tree.c +4
-1@@ -177,7 +177,7 @@ module_param(rcu_unlock_delay, int, 0444);* per-CPU. Object size is equal to one page. This value * can be changed at boot time. */ -static int rcu_min_cached_objs =2 ; +static int rcu_min_cached_objs =5 ;module_param (rcu_min_cached_objs, int,0444 );/* Retrieve RCU kthreads priority for rcutorture */ @@ -3084,6 +3084,9 @@ struct kfree_rcu_cpu_work {* In order to save some per-cpu space the list is singular. * Even though it is lockless an access has to be protected by the * per-cpu lock. +* @page_cache_work: A work to refill the cache when it is empty +* @work_in_progress: Indicates that page_cache_work is running +* @hrtimer: A hrtimer for scheduling a page_cache_work * @nr_bkv_objs: number of allocated objects at @bkvcache. * * This is a per-CPU structure. The reason that it is not included in @@ -3100,6 +3103,11 @@ struct kfree_rcu_cpu {bool monitor_todo;bool initialized;
Quoted 9 lines
kernel/rcu/tree.c struct kfree_rcu_cpu { · line 3108
Quoted 4 lines
kernel/rcu/tree.c +3
-0int count; + +struct work_struct page_cache_work; +atomic_t work_in_progress;Does it need to be atomic?
run_page_cache_work()is only called under a lock. You can usexchg()there. And when you do theatomic_set, you can useWRITE_ONCEas it is a data-race.
We can use xchg together with *_ONCE() macro. Could you please clarify what is your concern about using atomic_t? Both xchg() and atomic_xchg() guarantee atamarity. Same as WRITE_ONCE() or atomic_set().
Quoted 20 lines
Quoted 20 lines
kernel/rcu/tree.c +0
-10@@ -4449,24 +4482,14 @@ static void __init kfree_rcu_batch_init(void)for_each_possible_cpu (cpu) {struct kfree_rcu_cpu *krcp =per_cpu_ptr (&krc, cpu); -struct kvfree_rcu_bulk_data *bnode;for (i =0 ; i <KFREE_N_BATCHES ; i++) {INIT_RCU_WORK (&krcp->krw_arr[i].rcu_work, kfree_rcu_work); krcp->krw_arr[i].krcp = krcp; } -for (i =0 ; i < rcu_min_cached_objs; i++) { - bnode = (struct kvfree_rcu_bulk_data *) -__get_free_page (GFP_NOWAIT | __GFP_NOWARN); - -if (bnode) -put_cached_bnode (krcp, bnode); -else -pr_err ("Failed to preallocate for %d CPU!\n" , cpu); - }
Quoted 9 lines
kernel/rcu/tree.c static void __init kfree_rcu_batch_init(void) · line 4493
Quoted 4 lines
kernel/rcu/tree.c +1
-1-INIT_DELAYED_WORK (&krcp->monitor_work, kfree_rcu_monitor); +INIT_WORK (&krcp->page_cache_work, fill_page_cache_func); krcp->initialized = true;During initialization, is it not better to still pre-allocate? That way you don't have to wait to get into a situation where you need to initially allocate.
Since we have a worker that does it when a cache is empty there is no a high need in doing it during initialization phase. If we can reduce an amount of code it is always good :)
Thanks, Joel.
--
Vlad Rezki
Uladzislau Rezki
maintainerQuoted 48 lines · Paul E. McKenney
Quoted 40 lines · Uladzislau Rezki
Quoted 32 lines · Joel Fernandes
Quoted 22 lines · Uladzislau Rezki (Sony)
The current memmory-allocation interface presents to following difficulties that this patch is designed to overcome:
a) If built with
CONFIG_PROVE_RAW_LOCK_NESTING, the lockdep will complain aboutviolation("BUG: Invalid wait context")of the nesting rules. It does theraw_spinlockvs. spinlock nesting checks, i.e. it is not legal to acquire aspinlock_twhile holding araw_spinlock_t.Internally the kfree_rcu() uses raw_spinlock_t whereas the "page allocator" internally deals with spinlock_t to access to its zones. The code also can be broken from higher level of view: <snip> raw_spin_lock(&some_lock); kfree_rcu(some_pointer, some_field_offset); <snip>b) If built with
CONFIG_PREEMPT_RT. Please note, in that casespinlock_tis converted into sleepable variant. Invoking the page allocator from atomic contexts leads to "BUG: scheduling while atomic".Quoted 8 lines
c)
call_rcu()is invoked from raw atomic context andkfree_rcu()andkvfree_rcu()are expected to be called from atomic raw context as well.Move out a page allocation from contexts which trigger
kvfree_rcu()function to the separate worker. When a k[v]free_rcu() per-cpu page cache is empty a fallback mechanism is used and a special job is scheduled to refill the per-cpu cache.Quoted 7 lines
Looks good, still reviewing here. BTW just for my education, I was wondering about Thomas's email:
https://lkml.org/lkml/2020/8/11/939If slab allocations in pure raw-atomic context on RT is not allowed or recommended, should
kfree_rcu()be allowed?Quoted 6 lines
Thanks for reviewing, Joel :)
The decision was made that we need to support
kfree_rcu()from "real atomic contexts", to align with how it used to be before. We can go and just convert our local locks to thespinlock_tvariant but that was not Paul goal, it can be that some users needkfree_rcu()for raw atomics.
Quoted 7 lines
People invoke
call_rcu()from raw atomics, and so we should provide the same forkfree_rcu(). Yes, people could work around a raw-atomic prohibition, but such prohibitions incur constant costs over time in terms of development effort, increased bug rate, and increased complexity. Yes, this does increase all of those for RCU, but the relative increase is negligible, RCU being what it is.
I see your point.
Quoted 11 lines
Quoted 7 lines
Quoted 4 lines
slab can have same issue right? If per-cpu cache is drained, it has to allocate page from buddy allocator and there's no GFP flag to tell it about context where alloc is happening from.
Sounds like that. Apart of that, it might turn out soon that we or somebody else will rise a question one more time about something
GFP_RAWorGFP_NOLOCKS. So who knows..I would prefer that slab provide some way of dealing with raw atomic context, but the maintainers are thus far unconvinced.
I think, when preempt_rt is fully integrated to the kernel, we might get new users with such demand. So, it is not a closed topic so far, IMHO.
Quoted 13 lines
Quoted 8 lines
Quoted 8 lines
Or are we saying that we want to support kfree on RT from raw atomic atomic context, even though kmalloc is not supported? I hate to bring up this elephant in the room, but since I am a part of the people maintaining this code, I believe I would rather set some rules than supporting unsupported usages. :-\ (Once I know what is supported and what isn't that is). If indeed raw atomic
kfree_rcu()is a bogus use case because of -RT, then we ought to put a giant warning than supporting it :-(.Quoted 4 lines
We discussed it several times, the conclusion was that we need to support
kfree_rcu()from raw contexts. At least that was a clear signal from Paul to me. I think, if we obtain thepreemtable(), so it becomes versatile, we can drop the patch that is in question later on in the future.
Quoted 5 lines
Given a universally meaningful
preemptible(), we could directly call the allocator in some cases. It might (or might not) still make sense to defer the allocation whenpreemptible()indicated that a direct call to the allocator was unsafe.
I do not have a strong opinion here. Giving the fact that maintaining of such "deferring" is not considered as a big effort, i think, we can live with it.
--
Vlad Rezki
Paul E. McKenney
maintainerQuoted 66 lines · Uladzislau Rezki
Quoted 48 lines · Paul E. McKenney
Quoted 40 lines · Uladzislau Rezki
Quoted 32 lines · Joel Fernandes
Quoted 22 lines · Uladzislau Rezki (Sony)
The current memmory-allocation interface presents to following difficulties that this patch is designed to overcome:
a) If built with
CONFIG_PROVE_RAW_LOCK_NESTING, the lockdep will complain aboutviolation("BUG: Invalid wait context")of the nesting rules. It does theraw_spinlockvs. spinlock nesting checks, i.e. it is not legal to acquire aspinlock_twhile holding araw_spinlock_t.Internally the kfree_rcu() uses raw_spinlock_t whereas the "page allocator" internally deals with spinlock_t to access to its zones. The code also can be broken from higher level of view: <snip> raw_spin_lock(&some_lock); kfree_rcu(some_pointer, some_field_offset); <snip>b) If built with
CONFIG_PREEMPT_RT. Please note, in that casespinlock_tis converted into sleepable variant. Invoking the page allocator from atomic contexts leads to "BUG: scheduling while atomic".Quoted 8 lines
c)
call_rcu()is invoked from raw atomic context andkfree_rcu()andkvfree_rcu()are expected to be called from atomic raw context as well.Move out a page allocation from contexts which trigger
kvfree_rcu()function to the separate worker. When a k[v]free_rcu() per-cpu page cache is empty a fallback mechanism is used and a special job is scheduled to refill the per-cpu cache.Quoted 7 lines
Looks good, still reviewing here. BTW just for my education, I was wondering about Thomas's email:
https://lkml.org/lkml/2020/8/11/939If slab allocations in pure raw-atomic context on RT is not allowed or recommended, should
kfree_rcu()be allowed?Quoted 6 lines
Thanks for reviewing, Joel :)
The decision was made that we need to support
kfree_rcu()from "real atomic contexts", to align with how it used to be before. We can go and just convert our local locks to thespinlock_tvariant but that was not Paul goal, it can be that some users needkfree_rcu()for raw atomics.Quoted 7 lines
People invoke
call_rcu()from raw atomics, and so we should provide the same forkfree_rcu(). Yes, people could work around a raw-atomic prohibition, but such prohibitions incur constant costs over time in terms of development effort, increased bug rate, and increased complexity. Yes, this does increase all of those for RCU, but the relative increase is negligible, RCU being what it is.I see your point.
Quoted 8 lines
Quoted 7 lines
Quoted 4 lines
slab can have same issue right? If per-cpu cache is drained, it has to allocate page from buddy allocator and there's no GFP flag to tell it about context where alloc is happening from.
Sounds like that. Apart of that, it might turn out soon that we or somebody else will rise a question one more time about something
GFP_RAWorGFP_NOLOCKS. So who knows..
Quoted 5 lines
Quoted 3 lines
I would prefer that slab provide some way of dealing with raw atomic context, but the maintainers are thus far unconvinced.
I think, when
preempt_rtis fully integrated to the kernel, we might get new users with such demand. So, it is not a closed topic so far, IMHO.
Agreed! ;-)
Quoted 13 lines
Quoted 13 lines
Quoted 12 lines
Quoted 8 lines
Or are we saying that we want to support kfree on RT from raw atomic atomic context, even though kmalloc is not supported? I hate to bring up this elephant in the room, but since I am a part of the people maintaining this code, I believe I would rather set some rules than supporting unsupported usages. :-\ (Once I know what is supported and what isn't that is). If indeed raw atomic
kfree_rcu()is a bogus use case because of -RT, then we ought to put a giant warning than supporting it :-(.We discussed it several times, the conclusion was that we need to support
kfree_rcu()from raw contexts. At least that was a clear signal from Paul to me. I think, if we obtain thepreemtable(), so it becomes versatile, we can drop the patch that is in question later on in the future.
Quoted 8 lines
Quoted 5 lines
Given a universally meaningful
preemptible(), we could directly call the allocator in some cases. It might (or might not) still make sense to defer the allocation whenpreemptible()indicated that a direct call to the allocator was unsafe.I do not have a strong opinion here. Giving the fact that maintaining of such "deferring" is not considered as a big effort, i think, we can live with it.
And agreed here as well. If this were instead a large body of complex code, I might feel otherwise. But as it is, why worry?
Thanx, Paul
Re: [PATCH 15/16] rcu/tree: Allocate a page when caller is preemptible
In reply to Uladzislau RezkiOn Wed, Nov 04, 2020 at 12:39:31PM +0100, Uladzislau Rezki wrote:
[..]
Quoted 12 lines
Quoted 12 lines
kernel/rcu/tree.c run_page_cache_worker(struct kfree_rcu_cpu *krcp) · line 3437
Quoted 8 lines
kernel/rcu/tree.c +4
-4-if (!krcp->bkvhead[idx] || - krcp->bkvhead[idx]->nr_records ==KVFREE_BULK_MAX_ENTR ) { - bnode =get_cached_bnode (krcp); -/* Switch to emergency path. */ +if (!(*krcp)->bkvhead[idx] || + (*krcp)->bkvhead[idx]->nr_records ==KVFREE_BULK_MAX_ENTR ) { + bnode =get_cached_bnode (*krcp); +if (!bnode && can_alloc_page) {I think you can directly put
preemptible()here with a comment saying allocate only if preemptible and get rid ofcan_alloc_page.
Quoted 3 lines
Not really. We check
preemtable()before acquiring the internal lock, otherwise it will always return "false". Thus, it is checked on the entry in the beginning.
You are right. Sorry. Sounds good to me.
- Joel
Uladzislau Rezki
maintainerQuoted 93 lines · Paul E. McKenney
Quoted 66 lines · Uladzislau Rezki
Quoted 48 lines · Paul E. McKenney
Quoted 40 lines · Uladzislau Rezki
Quoted 32 lines · Joel Fernandes
Quoted 22 lines · Uladzislau Rezki (Sony)
The current memmory-allocation interface presents to following difficulties that this patch is designed to overcome:
a) If built with
CONFIG_PROVE_RAW_LOCK_NESTING, the lockdep will complain aboutviolation("BUG: Invalid wait context")of the nesting rules. It does theraw_spinlockvs. spinlock nesting checks, i.e. it is not legal to acquire aspinlock_twhile holding araw_spinlock_t.Internally the kfree_rcu() uses raw_spinlock_t whereas the "page allocator" internally deals with spinlock_t to access to its zones. The code also can be broken from higher level of view: <snip> raw_spin_lock(&some_lock); kfree_rcu(some_pointer, some_field_offset); <snip>b) If built with
CONFIG_PREEMPT_RT. Please note, in that casespinlock_tis converted into sleepable variant. Invoking the page allocator from atomic contexts leads to "BUG: scheduling while atomic".Quoted 8 lines
c)
call_rcu()is invoked from raw atomic context andkfree_rcu()andkvfree_rcu()are expected to be called from atomic raw context as well.Move out a page allocation from contexts which trigger
kvfree_rcu()function to the separate worker. When a k[v]free_rcu() per-cpu page cache is empty a fallback mechanism is used and a special job is scheduled to refill the per-cpu cache.Quoted 7 lines
Looks good, still reviewing here. BTW just for my education, I was wondering about Thomas's email:
https://lkml.org/lkml/2020/8/11/939If slab allocations in pure raw-atomic context on RT is not allowed or recommended, should
kfree_rcu()be allowed?Quoted 6 lines
Thanks for reviewing, Joel :)
The decision was made that we need to support
kfree_rcu()from "real atomic contexts", to align with how it used to be before. We can go and just convert our local locks to thespinlock_tvariant but that was not Paul goal, it can be that some users needkfree_rcu()for raw atomics.Quoted 7 lines
People invoke
call_rcu()from raw atomics, and so we should provide the same forkfree_rcu(). Yes, people could work around a raw-atomic prohibition, but such prohibitions incur constant costs over time in terms of development effort, increased bug rate, and increased complexity. Yes, this does increase all of those for RCU, but the relative increase is negligible, RCU being what it is.I see your point.
Quoted 8 lines
Quoted 7 lines
Quoted 4 lines
slab can have same issue right? If per-cpu cache is drained, it has to allocate page from buddy allocator and there's no GFP flag to tell it about context where alloc is happening from.
Sounds like that. Apart of that, it might turn out soon that we or somebody else will rise a question one more time about something
GFP_RAWorGFP_NOLOCKS. So who knows..Quoted 5 lines
Quoted 3 lines
I would prefer that slab provide some way of dealing with raw atomic context, but the maintainers are thus far unconvinced.
I think, when
preempt_rtis fully integrated to the kernel, we might get new users with such demand. So, it is not a closed topic so far, IMHO.Agreed! ;-)
Quoted 18 lines
Quoted 18 lines
Quoted 8 lines
Quoted 8 lines
Or are we saying that we want to support kfree on RT from raw atomic atomic context, even though kmalloc is not supported? I hate to bring up this elephant in the room, but since I am a part of the people maintaining this code, I believe I would rather set some rules than supporting unsupported usages. :-\ (Once I know what is supported and what isn't that is). If indeed raw atomic
kfree_rcu()is a bogus use case because of -RT, then we ought to put a giant warning than supporting it :-(.Quoted 4 lines
We discussed it several times, the conclusion was that we need to support
kfree_rcu()from raw contexts. At least that was a clear signal from Paul to me. I think, if we obtain thepreemtable(), so it becomes versatile, we can drop the patch that is in question later on in the future.Given a universally meaningful
preemptible(), we could directly call the allocator in some cases. It might (or might not) still make sense to defer the allocation whenpreemptible()indicated that a direct call to the allocator was unsafe.
Quoted 7 lines
Quoted 3 lines
I do not have a strong opinion here. Giving the fact that maintaining of such "deferring" is not considered as a big effort, i think, we can live with it.
And agreed here as well. If this were instead a large body of complex code, I might feel otherwise. But as it is, why worry?
Agreed! I do not consider it as extra complexity.
--
Vlad Rezki
Quoted 44 lines · Uladzislau Rezki
Quoted 34 lines · Joel Fernandes
Quoted 2 lines · Uladzislau Rezki (Sony)
The current memmory-allocation interface presents to following difficulties that this patch is designed to overcome
[...]
Quoted 30 lines
--- kernel/rcu/tree.c | 109 ++++++++++++++++++++++++++++------------------ 1 file changed, 66 insertions(+), 43 deletions(-)kernel/rcu/tree.c +4
-1@@ -177,7 +177,7 @@ module_param(rcu_unlock_delay, int, 0444);* per-CPU. Object size is equal to one page. This value * can be changed at boot time. */ -static int rcu_min_cached_objs =2 ; +static int rcu_min_cached_objs =5 ;module_param (rcu_min_cached_objs, int,0444 );/* Retrieve RCU kthreads priority for rcutorture */ @@ -3084,6 +3084,9 @@ struct kfree_rcu_cpu_work {* In order to save some per-cpu space the list is singular. * Even though it is lockless an access has to be protected by the * per-cpu lock. +* @page_cache_work: A work to refill the cache when it is empty +* @work_in_progress: Indicates that page_cache_work is running +* @hrtimer: A hrtimer for scheduling a page_cache_work * @nr_bkv_objs: number of allocated objects at @bkvcache. * * This is a per-CPU structure. The reason that it is not included in @@ -3100,6 +3103,11 @@ struct kfree_rcu_cpu {bool monitor_todo;bool initialized;Quoted 9 lines
kernel/rcu/tree.c struct kfree_rcu_cpu { · line 3108
Quoted 4 lines
kernel/rcu/tree.c +3
-0int count; + +struct work_struct page_cache_work; +atomic_t work_in_progress;Does it need to be atomic?
run_page_cache_work()is only called under a lock. You can usexchg()there. And when you do theatomic_set, you can useWRITE_ONCEas it is a data-race.
Quoted 3 lines
We can use xchg together with
*_ONCE()macro. Could you please clarify what is your concern about usingatomic_t? Bothxchg()andatomic_xchg()guarantee atamarity. Same asWRITE_ONCE()oratomic_set().
Right, whether there's lock or not does not matter as xchg() is also atomic-swap.
atomic_t is a more complex type though, I would directly use int since atomic_t is not needed here and there's no lost-update issue here. It could be matter of style as well.
BTW I did think atomic_xchg() adds additional memory barriers
but I could not find that to be the case in the implementation. Is that not the case? Docs says "atomic_xchg must provide explicit memory barriers around the operation.".
Quoted 29 lines
Quoted 20 lines
Quoted 20 lines
kernel/rcu/tree.c +0
-10@@ -4449,24 +4482,14 @@ static void __init kfree_rcu_batch_init(void)for_each_possible_cpu (cpu) {struct kfree_rcu_cpu *krcp =per_cpu_ptr (&krc, cpu); -struct kvfree_rcu_bulk_data *bnode;for (i =0 ; i <KFREE_N_BATCHES ; i++) {INIT_RCU_WORK (&krcp->krw_arr[i].rcu_work, kfree_rcu_work); krcp->krw_arr[i].krcp = krcp; } -for (i =0 ; i < rcu_min_cached_objs; i++) { - bnode = (struct kvfree_rcu_bulk_data *) -__get_free_page (GFP_NOWAIT | __GFP_NOWARN); - -if (bnode) -put_cached_bnode (krcp, bnode); -else -pr_err ("Failed to preallocate for %d CPU!\n" , cpu); - }Quoted 9 lines
kernel/rcu/tree.c static void __init kfree_rcu_batch_init(void) · line 4493
Quoted 4 lines
kernel/rcu/tree.c +1
-1-INIT_DELAYED_WORK (&krcp->monitor_work, kfree_rcu_monitor); +INIT_WORK (&krcp->page_cache_work, fill_page_cache_func); krcp->initialized = true;During initialization, is it not better to still pre-allocate? That way you don't have to wait to get into a situation where you need to initially allocate.
Quoted 3 lines
Since we have a worker that does it when a cache is empty there is no a high need in doing it during initialization phase. If we can reduce an amount of code it is always good :)
I am all for not having more code than needed. But you would hit synchronize_rcu() slow path immediately on first headless kfree_rcu() right? That seems like a step back from the current code :)
thanks,
- Joel
Uladzislau Rezki
maintainerQuoted 20 lines
Quoted 13 lines
Quoted 4 lines
Quoted 4 lines
* This is a per-CPU structure. The reason that it is not included in
kernel/rcu/tree.c +0
-0@@ -3100,6 +3103,11 @@ struct kfree_rcu_cpu {bool monitor_todo;bool initialized;Quoted 9 lines
kernel/rcu/tree.c struct kfree_rcu_cpu { · line 3108
Quoted 4 lines
kernel/rcu/tree.c +3
-0int count; + +struct work_struct page_cache_work; +atomic_t work_in_progress;Does it need to be atomic?
run_page_cache_work()is only called under a lock. You can usexchg()there. And when you do theatomic_set, you can useWRITE_ONCEas it is a data-race.Quoted 3 lines
We can use xchg together with
*_ONCE()macro. Could you please clarify what is your concern about usingatomic_t? Bothxchg()andatomic_xchg()guarantee atamarity. Same asWRITE_ONCE()oratomic_set().Right, whether there's lock or not does not matter as
xchg()is also atomic-swap.
Quoted 9 lines
atomic_tis a more complex type though, I would directly use int sinceatomic_tis not needed here and there's no lost-update issue here. It could be matter of style as well.BTW I did think
atomic_xchg()adds additional memory barriers
but I could not find that to be the case in the implementation. Is that not the case? Docs says "atomic_xchgmust provide explicit memory barriers around the operation.".
In most of the systems atmoc_xchg() is same as xchg() and atomic_set() is same as WRITE_ONCE(). But there are exceptions, for example "parisc"
*** arch/parisc/include/asm/atomic.h:
<snip>
...
#define _atomic_spin_lock_irqsave(l,f) do { \
arch_spinlock_t *s = ATOMIC_HASH(l); \
local_irq_save(f); \
arch_spin_lock(s); \
} while(0)
...
static __inline__ void atomic_set(atomic_t *v, int i)
{
unsigned long flags;
_atomic_spin_lock_irqsave(v, flags);
v->counter = i;
_atomic_spin_unlock_irqrestore(v, flags);
}<snip>
I will switch to xchg() and WRITE_ONCE(), because of such specific ARCHs.
Quoted 29 lines
Quoted 29 lines
Quoted 29 lines
Quoted 14 lines
kernel/rcu/tree.c +0
-4@@ -4449,24 +4482,14 @@ static void __init kfree_rcu_batch_init(void)for_each_possible_cpu (cpu) {struct kfree_rcu_cpu *krcp =per_cpu_ptr (&krc, cpu); -struct kvfree_rcu_bulk_data *bnode;for (i =0 ; i <KFREE_N_BATCHES ; i++) {INIT_RCU_WORK (&krcp->krw_arr[i].rcu_work, kfree_rcu_work); krcp->krw_arr[i].krcp = krcp; } -for (i =0 ; i < rcu_min_cached_objs; i++) { - bnode = (struct kvfree_rcu_bulk_data *) -__get_free_page (GFP_NOWAIT | __GFP_NOWARN);kernel/rcu/tree.c static void __init kfree_rcu_batch_init(void) · line 4493
Quoted 10 lines
kernel/rcu/tree.c +1
-7- -if (bnode) -put_cached_bnode (krcp, bnode); -else -pr_err ("Failed to preallocate for %d CPU!\n" , cpu); - } -INIT_DELAYED_WORK (&krcp->monitor_work, kfree_rcu_monitor); +INIT_WORK (&krcp->page_cache_work, fill_page_cache_func); krcp->initialized = true;During initialization, is it not better to still pre-allocate? That way you don't have to wait to get into a situation where you need to initially allocate.
Quoted 8 lines
Quoted 3 lines
Since we have a worker that does it when a cache is empty there is no a high need in doing it during initialization phase. If we can reduce an amount of code it is always good :)
I am all for not having more code than needed. But you would hit
synchronize_rcu()slow path immediately on first headlesskfree_rcu()right? That seems like a step back from the current code :)
As for slow path and hitting the synchronize_rcu() immediately. Yes, a slow hit "counter" will be increased by 1, the difference between two variants will be N and N + 1 times. I do not consider N + 1 as a big difference and impact on performance.
Should we guarantee that a first user does not hit a fallback path that invokes synchronize_rcu()? If not, i would rather remove redundant code.
Any thoughts here?
Thanks!
--
Vlad Rezki