Skip to contentSkip to conversation

rcu/tree: Add a work to allocate pages from regular context

PATCH In review Uladzislau Rezki (Sony) 4 participants 39 messages Oct 29, 2020 – Nov 4, 2020

Subsystems (14)
lore
Share & tools
lore

Uladzislau Rezki (Sony)

author maintainer
#0 lore raw

PATCH 1/16 rcu/tree: Add a work to allocate pages from regular context

The current memmory-allocation interface presents to following difficulties that this patch is designed to overcome:

a) If built with CONFIG_PROVE_RAW_LOCK_NESTING, the lockdep will complain about violation("BUG: Invalid wait context") of the nesting rules. It does the raw_spinlock vs. spinlock nesting checks, i.e. it is not legal to acquire a spinlock_t while holding a raw_spinlock_t.

Internally the kfree_rcu() uses raw_spinlock_t whereas the "page allocator" internally deals with spinlock_t to access to its zones. The code also can be broken from higher level of view:
<snip>

       raw_spin_lock(&some_lock);
       kfree_rcu(some_pointer, some_field_offset);

<snip>

b) If built with CONFIG_PREEMPT_RT. Please note, in that case spinlock_t is converted into sleepable variant. Invoking the page allocator from atomic contexts leads to "BUG: scheduling while atomic".

c) call_rcu() is invoked from raw atomic context and kfree_rcu() and kvfree_rcu() are expected to be called from atomic raw context as well.

Move out a page allocation from contexts which trigger kvfree_rcu() function to the separate worker. When a k[v]free_rcu() per-cpu page cache is empty a fallback mechanism is used and a special job is scheduled to refill the per-cpu cache.

---
 kernel/rcu/tree.c | 109 ++++++++++++++++++++++++++++------------------
 1 file changed, 66 insertions(+), 43 deletions(-)
kernel/rcu/tree.c +66 -43
index 06895ef85d69..f2da2a1cc716 100644
@@ -177,7 +177,7 @@ module_param(rcu_unlock_delay, int, 0444);
  * per-CPU. Object size is equal to one page. This value
  * can be changed at boot time.
  */
-static int rcu_min_cached_objs = 2;
+static int rcu_min_cached_objs = 5;
 module_param(rcu_min_cached_objs, int, 0444);
 
 /* Retrieve RCU kthreads priority for rcutorture */
@@ -3084,6 +3084,9 @@ struct kfree_rcu_cpu_work {
  *	In order to save some per-cpu space the list is singular.
  *	Even though it is lockless an access has to be protected by the
  *	per-cpu lock.
+ * @page_cache_work: A work to refill the cache when it is empty
+ * @work_in_progress: Indicates that page_cache_work is running
+ * @hrtimer: A hrtimer for scheduling a page_cache_work
  * @nr_bkv_objs: number of allocated objects at @bkvcache.
  *
  * This is a per-CPU structure.  The reason that it is not included in
@@ -3100,6 +3103,11 @@ struct kfree_rcu_cpu {
 	bool monitor_todo;
 	bool initialized;
 	int count;
+
+	struct work_struct page_cache_work;
+	atomic_t work_in_progress;
+	struct hrtimer hrtimer;
+
 	struct llist_head bkvcache;
 	int nr_bkv_objs;
 };
@@ -3217,10 +3225,10 @@ static void kfree_rcu_work(struct work_struct *work)
 			}
 			rcu_lock_release(&rcu_callback_map);
 
-			krcp = krc_this_cpu_lock(&flags);
+			raw_spin_lock_irqsave(&krcp->lock, flags);
 			if (put_cached_bnode(krcp, bkvhead[i]))
 				bkvhead[i] = NULL;
-			krc_this_cpu_unlock(krcp, flags);
+			raw_spin_unlock_irqrestore(&krcp->lock, flags);
 
 			if (bkvhead[i])
 				free_page((unsigned long) bkvhead[i]);
@@ -3347,6 +3355,57 @@ static void kfree_rcu_monitor(struct work_struct *work)
 		raw_spin_unlock_irqrestore(&krcp->lock, flags);
 }
 
+static enum hrtimer_restart
+schedule_page_work_fn(struct hrtimer *t)
+{
+	struct kfree_rcu_cpu *krcp =
+		container_of(t, struct kfree_rcu_cpu, hrtimer);
+
+	queue_work(system_highpri_wq, &krcp->page_cache_work);
+	return HRTIMER_NORESTART;
+}
+
+static void fill_page_cache_func(struct work_struct *work)
+{
+	struct kvfree_rcu_bulk_data *bnode;
+	struct kfree_rcu_cpu *krcp =
+		container_of(work, struct kfree_rcu_cpu,
+			page_cache_work);
+	unsigned long flags;
+	bool pushed;
+	int i;
+
+	for (i = 0; i < rcu_min_cached_objs; i++) {
+		bnode = (struct kvfree_rcu_bulk_data *)
+			__get_free_page(GFP_KERNEL | __GFP_NOWARN);
+
+		if (bnode) {
+			raw_spin_lock_irqsave(&krcp->lock, flags);
+			pushed = put_cached_bnode(krcp, bnode);
+			raw_spin_unlock_irqrestore(&krcp->lock, flags);
+
+			if (!pushed) {
+				free_page((unsigned long) bnode);
+				break;
+			}
+		}
+	}
+
+	atomic_set(&krcp->work_in_progress, 0);
+}
+
+static void
+run_page_cache_worker(struct kfree_rcu_cpu *krcp)
+{
+	if (rcu_scheduler_active == RCU_SCHEDULER_RUNNING &&
+			!atomic_xchg(&krcp->work_in_progress, 1)) {
+		hrtimer_init(&krcp->hrtimer, CLOCK_MONOTONIC,
+			HRTIMER_MODE_REL);
+		krcp->hrtimer.function = schedule_page_work_fn;
+		hrtimer_start(&krcp->hrtimer, 0, HRTIMER_MODE_REL);
+	}
+}
+
 static inline bool
 kvfree_call_rcu_add_ptr_to_bulk(struct kfree_rcu_cpu *krcp, void *ptr)
 {
@@ -3363,32 +3422,8 @@ kvfree_call_rcu_add_ptr_to_bulk(struct kfree_rcu_cpu *krcp, void *ptr)
 	if (!krcp->bkvhead[idx] ||
 			krcp->bkvhead[idx]->nr_records == KVFREE_BULK_MAX_ENTR) {
 		bnode = get_cached_bnode(krcp);
-		if (!bnode) {
-			/*
-			 * To keep this path working on raw non-preemptible
-			 * sections, prevent the optional entry into the
-			 * allocator as it uses sleeping locks. In fact, even
-			 * if the caller of kfree_rcu() is preemptible, this
-			 * path still is not, as krcp->lock is a raw spinlock.
-			 * With additional page pre-allocation in the works,
-			 * hitting this return is going to be much less likely.
-			 */
-			if (IS_ENABLED(CONFIG_PREEMPT_RT))
-				return false;
-
-			/*
-			 * NOTE: For one argument of kvfree_rcu() we can
-			 * drop the lock and get the page in sleepable
-			 * context. That would allow to maintain an array
-			 * for the CONFIG_PREEMPT_RT as well if no cached
-			 * pages are available.
-			 */
-			bnode = (struct kvfree_rcu_bulk_data *)
-				__get_free_page(GFP_NOWAIT | __GFP_NOWARN);
-		}
-
 		/* Switch to emergency path. */
-		if (unlikely(!bnode))
+		if (!bnode)
 			return false;
 
 		/* Initialize the new block. */
@@ -3452,12 +3487,10 @@ void kvfree_call_rcu(struct rcu_head *head, rcu_callback_t func)
 		goto unlock_return;
 	}
 
-	/*
-	 * Under high memory pressure GFP_NOWAIT can fail,
-	 * in that case the emergency path is maintained.
-	 */
 	success = kvfree_call_rcu_add_ptr_to_bulk(krcp, ptr);
 	if (!success) {
+		run_page_cache_worker(krcp);
+
 		if (head == NULL)
 			// Inline if kvfree_rcu(one_arg) call.
 			goto unlock_return;
@@ -4449,24 +4482,14 @@ static void __init kfree_rcu_batch_init(void)
 
 	for_each_possible_cpu(cpu) {
 		struct kfree_rcu_cpu *krcp = per_cpu_ptr(&krc, cpu);
-		struct kvfree_rcu_bulk_data *bnode;
 
 		for (i = 0; i < KFREE_N_BATCHES; i++) {
 			INIT_RCU_WORK(&krcp->krw_arr[i].rcu_work, kfree_rcu_work);
 			krcp->krw_arr[i].krcp = krcp;
 		}
 
-		for (i = 0; i < rcu_min_cached_objs; i++) {
-			bnode = (struct kvfree_rcu_bulk_data *)
-				__get_free_page(GFP_NOWAIT | __GFP_NOWARN);
-
-			if (bnode)
-				put_cached_bnode(krcp, bnode);
-			else
-				pr_err("Failed to preallocate for %d CPU!\n", cpu);
-		}
-
 		INIT_DELAYED_WORK(&krcp->monitor_work, kfree_rcu_monitor);
+		INIT_WORK(&krcp->page_cache_work, fill_page_cache_func);
 		krcp->initialized = true;
 	}
 	if (register_shrinker(&kfree_rcu_shrinker))
Signature · 1 line
2.20.1
author maintainer
#1 lore raw

PATCH 2/16 lib/debug: Remove pointless ARCH_NO_PREEMPT dependencies

From: Thomas Gleixner <[email protected]>

ARCH_NO_PREEMPT disables the selection of CONFIG_PREEMPT_VOLUNTARY and CONFIG_PREEMPT, but architectures which set this config option still support preempt count for hard and softirq accounting.

There is absolutely no reason to prevent lockdep from using the preempt counter nor is there a reason to prevent the enablement of CONFIG_DEBUG_ATOMIC_SLEEP on such architectures.

Remove the dependencies.

---
 lib/Kconfig.debug | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)
lib/Kconfig.debug +1 -2
index d7a7bc3b6098..89c9a177fb9b 100644
@@ -1159,7 +1159,7 @@ config PROVE_LOCKING
 	select DEBUG_RWSEMS
 	select DEBUG_WW_MUTEX_SLOWPATH
 	select DEBUG_LOCK_ALLOC
-	select PREEMPT_COUNT if !ARCH_NO_PREEMPT
+	select PREEMPT_COUNT
 	select TRACE_IRQFLAGS
 	default n
 	help
@@ -1321,7 +1321,6 @@ config DEBUG_ATOMIC_SLEEP
 	bool "Sleep inside atomic section checking"
 	select PREEMPT_COUNT
 	depends on DEBUG_KERNEL
-	depends on !ARCH_NO_PREEMPT
 	help
 	  If you say Y here, various routines which may sleep will become very
 	  noisy if they are called inside atomic sections: when a spinlock is
Signature · 1 line
2.20.1
author maintainer
#5 lore raw

PATCH 6/16 mm/pagemap: Cleanup PREEMPT_COUNT leftovers

In reply to Uladzislau Rezki (Sony)

From: Thomas Gleixner <[email protected]>

CONFIG_PREEMPT_COUNT is now unconditionally enabled and will be removed. Cleanup the leftovers before doing so.

---
 include/linux/pagemap.h | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)
include/linux/pagemap.h +1 -3
index c77b7c31b2e4..cbfbe2bcca75 100644
@@ -204,9 +204,7 @@ void release_pages(struct page **pages, int nr);
 static inline int __page_cache_add_speculative(struct page *page, int count)
 {
 #ifdef CONFIG_TINY_RCU
-# ifdef CONFIG_PREEMPT_COUNT
-	VM_BUG_ON(!in_atomic() && !irqs_disabled());
-# endif
+	VM_BUG_ON(preemptible())
 	/*
 	 * Preempt must be disabled here - we rely on rcu_read_lock doing
 	 * this for us.
Signature · 1 line
2.20.1
author maintainer
#11 lore raw

PATCH 7/16 locking/bitspinlock: Cleanup PREEMPT_COUNT leftovers

In reply to Uladzislau Rezki (Sony)

From: Thomas Gleixner <[email protected]>

CONFIG_PREEMPT_COUNT is now unconditionally enabled and will be removed. Cleanup the leftovers before doing so.

---
 include/linux/bit_spinlock.h | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)
include/linux/bit_spinlock.h +1 -3
index bbc4730a6505..1e03d54b0b6f 100644
@@ -90,10 +90,8 @@ static inline int bit_spin_is_locked(int bitnum, unsigned long *addr)
 {
 #if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK)
 	return test_bit(bitnum, addr);
-#elif defined CONFIG_PREEMPT_COUNT
-	return preempt_count();
 #else
-	return 1;
+	return preempt_count();
 #endif
 }
 
Signature · 1 line
2.20.1
author maintainer
#13 lore raw

PATCH 9/16 sched: Cleanup PREEMPT_COUNT leftovers

In reply to Uladzislau Rezki (Sony)

From: Thomas Gleixner <[email protected]>

CONFIG_PREEMPT_COUNT is now unconditionally enabled and will be removed. Cleanup the leftovers before doing so.

Cc Ingo Molnar <[email protected]>
Cc Peter Zijlstra <[email protected]>
Cc Juri Lelli <[email protected]>
Cc Vincent Guittot <[email protected]>
Cc Dietmar Eggemann <[email protected]>
Cc Steven Rostedt <[email protected]>
Cc Ben Segall <[email protected]>
Cc Mel Gorman <[email protected]>
Cc Daniel Bristot de Oliveira <[email protected]>
---
 kernel/sched/core.c | 6 +-----
 lib/Kconfig.debug   | 1 -
 2 files changed, 1 insertion(+), 6 deletions(-)
kernel/sched/core.c +1 -5
index d2003a7d5ab5..e172f2ddfa16 100644
@@ -3702,8 +3702,7 @@ asmlinkage __visible void schedule_tail(struct task_struct *prev)
 	 * finish_task_switch() for details.
 	 *
 	 * finish_task_switch() will drop rq->lock() and lower preempt_count
-	 * and the preempt_enable() will end up enabling preemption (on
-	 * PREEMPT_COUNT kernels).
+	 * and the preempt_enable() will end up enabling preemption.
 	 */
 
 	rq = finish_task_switch(prev);
@@ -7307,9 +7306,6 @@ void __cant_sleep(const char *file, int line, int preempt_offset)
 	if (irqs_disabled())
 		return;
 
-	if (!IS_ENABLED(CONFIG_PREEMPT_COUNT))
-		return;
-
 	if (preempt_count() > preempt_offset)
 		return;
 
lib/Kconfig.debug +0 -1
index 03a85065805e..d62806c81f6d 100644
@@ -1318,7 +1318,6 @@ config DEBUG_LOCKDEP
 
 config DEBUG_ATOMIC_SLEEP
 	bool "Sleep inside atomic section checking"
-	select PREEMPT_COUNT
 	depends on DEBUG_KERNEL
 	help
 	  If you say Y here, various routines which may sleep will become very
Signature · 1 line
2.20.1
author maintainer
#14 lore raw

PATCH 10/16 ARM: Cleanup PREEMPT_COUNT leftovers

In reply to Uladzislau Rezki (Sony)

From: Thomas Gleixner <[email protected]>

CONFIG_PREEMPT_COUNT is now unconditionally enabled and will be removed. Cleanup the leftovers before doing so.

---
 arch/arm/include/asm/assembler.h   | 11 -----------
 arch/arm/kernel/iwmmxt.S           |  2 --
 arch/arm/mach-ep93xx/crunch-bits.S |  2 --
 3 files changed, 15 deletions(-)
arch/arm/include/asm/assembler.h +0 -11
index feac2c8b86f2..fce52eedc6e1 100644
@@ -212,7 +212,6 @@
 /*
  * Increment/decrement the preempt count.
  */
-#ifdef CONFIG_PREEMPT_COUNT
 	.macro	inc_preempt_count, ti, tmp
 	ldr	\tmp, [\ti, #TI_PREEMPT]	@ get preempt count
 	add	\tmp, \tmp, #1			@ increment it
@@ -229,16 +228,6 @@
 	get_thread_info \ti
 	dec_preempt_count \ti, \tmp
 	.endm
-#else
-	.macro	inc_preempt_count, ti, tmp
-	.endm
-
-	.macro	dec_preempt_count, ti, tmp
-	.endm
-
-	.macro	dec_preempt_count_ti, ti, tmp
-	.endm
-#endif
 
 #define USERL(l, x...)				\
 9999:	x;					\
arch/arm/kernel/iwmmxt.S +0 -2
index 0dcae787b004..1f845be78c43 100644
@@ -94,9 +94,7 @@ ENTRY(iwmmxt_task_enable)
 	mov	r2, r2				@ cpwait
 	bl	concan_save
 
-#ifdef CONFIG_PREEMPT_COUNT
 	get_thread_info r10
-#endif
 4:	dec_preempt_count r10, r3
 	ret	r9				@ normal exit from exception
 
arch/arm/mach-ep93xx/crunch-bits.S +0 -2
index fb2dbf76f09e..0aabcf4ebe8e 100644
@@ -191,9 +191,7 @@ crunch_load:
 	cfldr64		mvdx15, [r0, #CRUNCH_MVDX15]
 
 1:
-#ifdef CONFIG_PREEMPT_COUNT
 	get_thread_info r10
-#endif
 2:	dec_preempt_count r10, r3
 	ret	lr
 
Signature · 1 line
2.20.1
author maintainer
#12 lore raw

PATCH 8/16 uaccess: Cleanup PREEMPT_COUNT leftovers

In reply to Uladzislau Rezki (Sony)

From: Thomas Gleixner <[email protected]>

CONFIG_PREEMPT_COUNT is now unconditionally enabled and will be removed. Cleanup the leftovers before doing so.

---
 include/linux/uaccess.h | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)
include/linux/uaccess.h +3 -3
index c7c6e8b8344d..d6473a72a336 100644
@@ -275,9 +275,9 @@ static inline bool pagefault_disabled(void)
  *
  * This function should only be used by the fault handlers. Other users should
  * stick to pagefault_disabled().
- * Please NEVER use preempt_disable() to disable the fault handler. With
- * !CONFIG_PREEMPT_COUNT, this is like a NOP. So the handler won't be disabled.
- * in_atomic() will report different values based on !CONFIG_PREEMPT_COUNT.
+ *
+ * Please NEVER use preempt_disable() or local_irq_disable() to disable the
+ * fault handler.
  */
 #define faulthandler_disabled() (pagefault_disabled() || in_atomic())
 
Signature · 1 line
2.20.1
author maintainer
#16 lore raw

PATCH 12/16 drm/i915: Cleanup PREEMPT_COUNT leftovers

In reply to Uladzislau Rezki (Sony)

From: Thomas Gleixner <[email protected]>

CONFIG_PREEMPT_COUNT is now unconditionally enabled and will be removed. Cleanup the leftovers before doing so.

---
 drivers/gpu/drm/i915/Kconfig.debug | 1 -
 drivers/gpu/drm/i915/i915_utils.h  | 3 +--
 2 files changed, 1 insertion(+), 3 deletions(-)
drivers/gpu/drm/i915/Kconfig.debug +0 -1
index 25cd9788a4d5..b149f76d3ccd 100644
@@ -20,7 +20,6 @@ config DRM_I915_DEBUG
 	bool "Enable additional driver debugging"
 	depends on DRM_I915
 	select DEBUG_FS
-	select PREEMPT_COUNT
 	select I2C_CHARDEV
 	select STACKDEPOT
 	select DRM_DP_AUX_CHARDEV
drivers/gpu/drm/i915/i915_utils.h +1 -2
index 54773371e6bd..ecfed860b3f6 100644
@@ -337,8 +337,7 @@ wait_remaining_ms_from_jiffies(unsigned long timestamp_jiffies, int to_wait_ms)
 						   (Wmax))
 #define wait_for(COND, MS)		_wait_for((COND), (MS) * 1000, 10, 1000)
 
-/* If CONFIG_PREEMPT_COUNT is disabled, in_atomic() always reports false. */
-#if defined(CONFIG_DRM_I915_DEBUG) && defined(CONFIG_PREEMPT_COUNT)
+#ifdef CONFIG_DRM_I915_DEBUG
 # define _WAIT_FOR_ATOMIC_CHECK(ATOMIC) WARN_ON_ONCE((ATOMIC) && !in_atomic())
 #else
 # define _WAIT_FOR_ATOMIC_CHECK(ATOMIC) do { } while (0)
Signature · 1 line
2.20.1
author maintainer
#23 lore raw

PATCH 16/16 rcu/tree: Use delayed work instead of hrtimer to refill the cache

In reply to Uladzislau Rezki (Sony)

A CONFIG_PREEMPT_COUNT is unconditionally enabled, thus a page can be obtained directly from a kvfree_rcu() path. To distinguish that and take a decision the preemptable() macro is used when it is save to enter allocator.

It means that refilling a cache is not important from timing point of view. Switch to a delayed work, so the actual work is queued from the timer interrupt with 1 jiffy delay. An immediate placing a task on a current CPU can lead to rq->lock double lock. That is why a delayed method is in place.

---
 kernel/rcu/tree.c | 26 +++++---------------------
 1 file changed, 5 insertions(+), 21 deletions(-)
kernel/rcu/tree.c +5 -21
index 3f9b016a44dc..9ea55f800b4b 100644
@@ -3086,7 +3086,6 @@ struct kfree_rcu_cpu_work {
  *	per-cpu lock.
  * @page_cache_work: A work to refill the cache when it is empty
  * @work_in_progress: Indicates that page_cache_work is running
- * @hrtimer: A hrtimer for scheduling a page_cache_work
  * @nr_bkv_objs: number of allocated objects at @bkvcache.
  *
  * This is a per-CPU structure.  The reason that it is not included in
@@ -3104,9 +3103,8 @@ struct kfree_rcu_cpu {
 	bool initialized;
 	int count;
 
-	struct work_struct page_cache_work;
+	struct delayed_work page_cache_work;
 	atomic_t work_in_progress;
-	struct hrtimer hrtimer;
 
 	struct llist_head bkvcache;
 	int nr_bkv_objs;
@@ -3355,22 +3353,12 @@ static void kfree_rcu_monitor(struct work_struct *work)
 		raw_spin_unlock_irqrestore(&krcp->lock, flags);
 }
 
-static enum hrtimer_restart
-schedule_page_work_fn(struct hrtimer *t)
-{
-	struct kfree_rcu_cpu *krcp =
-		container_of(t, struct kfree_rcu_cpu, hrtimer);
-
-	queue_work(system_highpri_wq, &krcp->page_cache_work);
-	return HRTIMER_NORESTART;
-}
-
 static void fill_page_cache_func(struct work_struct *work)
 {
 	struct kvfree_rcu_bulk_data *bnode;
 	struct kfree_rcu_cpu *krcp =
 		container_of(work, struct kfree_rcu_cpu,
-			page_cache_work);
+			page_cache_work.work);
 	unsigned long flags;
 	bool pushed;
 	int i;
@@ -3398,12 +3386,8 @@ static void
 run_page_cache_worker(struct kfree_rcu_cpu *krcp)
 {
 	if (rcu_scheduler_active == RCU_SCHEDULER_RUNNING &&
-			!atomic_xchg(&krcp->work_in_progress, 1)) {
-		hrtimer_init(&krcp->hrtimer, CLOCK_MONOTONIC,
-			HRTIMER_MODE_REL);
-		krcp->hrtimer.function = schedule_page_work_fn;
-		hrtimer_start(&krcp->hrtimer, 0, HRTIMER_MODE_REL);
-	}
+			!atomic_xchg(&krcp->work_in_progress, 1))
+		schedule_delayed_work(&krcp->page_cache_work, 1);
 }
 
 // Record ptr in a page managed by krcp, with the pre-krc_this_cpu_lock()
@@ -4503,7 +4487,7 @@ static void __init kfree_rcu_batch_init(void)
 		}
 
 		INIT_DELAYED_WORK(&krcp->monitor_work, kfree_rcu_monitor);
-		INIT_WORK(&krcp->page_cache_work, fill_page_cache_func);
+		INIT_DELAYED_WORK(&krcp->page_cache_work, fill_page_cache_func);
 		krcp->initialized = true;
 	}
 	if (register_shrinker(&kfree_rcu_shrinker))
Signature · 1 line
2.20.1
author maintainer
#19 lore raw

PATCH 15/16 rcu/tree: Allocate a page when caller is preemptible

In reply to Uladzislau Rezki (Sony)

Given that CONFIG_PREEMPT_COUNT is unconditionally enabled by the earlier commits in this series, the preemptible() macro now properly detects preempt-disable code regions even in kernels built with CONFIG_PREEMPT_NONE.

This commit therefore uses preemptible() to determine whether allocation is possible at all for double-argument kvfree_rcu(). If !preemptible(), then allocation is not possible, and kvfree_rcu() falls back to using the less cache-friendly rcu_head approach. Even when preemptible(), the caller might be involved in reclaim, so the GFP_ flags used by double-argument kvfree_rcu() must avoid invoking reclaim processing.

Note that single-argument kvfree_rcu() must be invoked in sleepable contexts, and that its fallback is the relatively high latency synchronize_rcu(). Single-argument kvfree_rcu() therefore uses GFP_KERNEL|__GFP_RETRY_MAYFAIL to allow limited sleeping within the memory allocator.

[ paulmck: Add add_ptr_to_bulk_krc_lock header comment per Michal Hocko. ]
---
 kernel/rcu/tree.c | 48 ++++++++++++++++++++++++++++++-----------------
 1 file changed, 31 insertions(+), 17 deletions(-)
kernel/rcu/tree.c +31 -17
index f2da2a1cc716..3f9b016a44dc 100644
@@ -3406,37 +3406,55 @@ run_page_cache_worker(struct kfree_rcu_cpu *krcp)
 	}
 }
 
+// Record ptr in a page managed by krcp, with the pre-krc_this_cpu_lock()
+// state specified by flags.  If can_sleep is true, the caller must
+// be schedulable and not be holding any locks or mutexes that might be
+// acquired by the memory allocator or anything that it might invoke.
+// If !can_sleep, then if !preemptible() no allocation will be undertaken,
+// otherwise the allocation will use GFP_ATOMIC to avoid the remainder of
+// the aforementioned deadlock possibilities.  Returns true if ptr was
+// successfully recorded, else the caller must use a fallback.
 static inline bool
-kvfree_call_rcu_add_ptr_to_bulk(struct kfree_rcu_cpu *krcp, void *ptr)
+add_ptr_to_bulk_krc_lock(struct kfree_rcu_cpu **krcp,
+	unsigned long *flags, void *ptr, bool can_sleep)
 {
 	struct kvfree_rcu_bulk_data *bnode;
+	bool can_alloc_page = preemptible();
+	gfp_t gfp = (can_sleep ? GFP_KERNEL | __GFP_RETRY_MAYFAIL :
+		GFP_ATOMIC) | __GFP_NOWARN;
 	int idx;
 
-	if (unlikely(!krcp->initialized))
+	*krcp = krc_this_cpu_lock(flags);
+	if (unlikely(!(*krcp)->initialized))
 		return false;
 
-	lockdep_assert_held(&krcp->lock);
 	idx = !!is_vmalloc_addr(ptr);
 
 	/* Check if a new block is required. */
-	if (!krcp->bkvhead[idx] ||
-			krcp->bkvhead[idx]->nr_records == KVFREE_BULK_MAX_ENTR) {
-		bnode = get_cached_bnode(krcp);
-		/* Switch to emergency path. */
+	if (!(*krcp)->bkvhead[idx] ||
+			(*krcp)->bkvhead[idx]->nr_records == KVFREE_BULK_MAX_ENTR) {
+		bnode = get_cached_bnode(*krcp);
+		if (!bnode && can_alloc_page) {
+			krc_this_cpu_unlock(*krcp, *flags);
+			bnode = (struct kvfree_rcu_bulk_data *)
+				__get_free_page(gfp);
+			*krcp = krc_this_cpu_lock(flags);
+		}
+
 		if (!bnode)
 			return false;
 
 		/* Initialize the new block. */
 		bnode->nr_records = 0;
-		bnode->next = krcp->bkvhead[idx];
+		bnode->next = (*krcp)->bkvhead[idx];
 
 		/* Attach it to the head. */
-		krcp->bkvhead[idx] = bnode;
+		(*krcp)->bkvhead[idx] = bnode;
 	}
 
 	/* Finally insert. */
-	krcp->bkvhead[idx]->records
-		[krcp->bkvhead[idx]->nr_records++] = ptr;
+	(*krcp)->bkvhead[idx]->records
+		[(*krcp)->bkvhead[idx]->nr_records++] = ptr;
 
 	return true;
 }
@@ -3474,20 +3492,16 @@ void kvfree_call_rcu(struct rcu_head *head, rcu_callback_t func)
 		ptr = (unsigned long *) func;
 	}
 
-	krcp = krc_this_cpu_lock(&flags);
-
 	// Queue the object but don't yet schedule the batch.
 	if (debug_rcu_head_queue(ptr)) {
 		// Probable double kfree_rcu(), just leak.
 		WARN_ONCE(1, "%s(): Double-freed call. rcu_head %p\n",
 			  __func__, head);
 
-		// Mark as success and leave.
-		success = true;
-		goto unlock_return;
+		return;
 	}
 
-	success = kvfree_call_rcu_add_ptr_to_bulk(krcp, ptr);
+	success = add_ptr_to_bulk_krc_lock(&krcp, &flags, ptr, !head);
 	if (!success) {
 		run_page_cache_worker(krcp);
 
Signature · 1 line
2.20.1
author maintainer
#18 lore raw

PATCH 14/16 preempt: Remove PREEMPT_COUNT from Kconfig

In reply to Uladzislau Rezki (Sony)

From: Thomas Gleixner <[email protected]>

All conditionals and irritations are gone.

---
 kernel/Kconfig.preempt | 3 ---
 1 file changed, 3 deletions(-)
kernel/Kconfig.preempt +0 -3
index 3f4712ff073b..120b63f0c55a 100644
@@ -74,8 +74,5 @@ config PREEMPT_RT
 
 endchoice
 
-config PREEMPT_COUNT
-       def_bool y
-
 config PREEMPTION
        bool
Signature · 1 line
2.20.1
author maintainer
#17 lore raw

PATCH 13/16 rcutorture: Cleanup PREEMPT_COUNT leftovers

In reply to Uladzislau Rezki (Sony)

From: Thomas Gleixner <[email protected]>

CONFIG_PREEMPT_COUNT is now unconditionally enabled and will be removed. Cleanup the leftovers before doing so.

Cc "Paul E. McKenney" <[email protected]>
Cc Josh Triplett <[email protected]>
Cc Steven Rostedt <[email protected]>
Cc Mathieu Desnoyers <[email protected]>
Cc Lai Jiangshan <[email protected]>
Cc Shuah Khan <[email protected]>
---
 tools/testing/selftests/rcutorture/configs/rcu/SRCU-t        | 1 -
 tools/testing/selftests/rcutorture/configs/rcu/SRCU-u        | 1 -
 tools/testing/selftests/rcutorture/configs/rcu/TINY01        | 1 -
 tools/testing/selftests/rcutorture/doc/TINY_RCU.txt          | 5 ++---
 tools/testing/selftests/rcutorture/doc/TREE_RCU-kconfig.txt  | 1 -
 .../selftests/rcutorture/formal/srcu-cbmc/src/config.h       | 1 -
 6 files changed, 2 insertions(+), 8 deletions(-)
tools/testing/selftests/rcutorture/configs/rcu/SRCU-t +0 -1
index 6c78022c8cd8..553cf6534e67 100644
@@ -7,4 +7,3 @@ CONFIG_RCU_TRACE=n
 CONFIG_DEBUG_LOCK_ALLOC=n
 CONFIG_DEBUG_OBJECTS_RCU_HEAD=n
 CONFIG_DEBUG_ATOMIC_SLEEP=y
-#CHECK#CONFIG_PREEMPT_COUNT=y
tools/testing/selftests/rcutorture/configs/rcu/SRCU-u +0 -1
index c15ada821e45..99563da21732 100644
@@ -7,4 +7,3 @@ CONFIG_RCU_TRACE=n
 CONFIG_DEBUG_LOCK_ALLOC=y
 CONFIG_PROVE_LOCKING=y
 CONFIG_DEBUG_OBJECTS_RCU_HEAD=n
-CONFIG_PREEMPT_COUNT=n
tools/testing/selftests/rcutorture/configs/rcu/TINY01 +0 -1
index 6db705e55487..9b22b8e768d5 100644
@@ -10,4 +10,3 @@ CONFIG_RCU_TRACE=n
 #CHECK#CONFIG_RCU_STALL_COMMON=n
 CONFIG_DEBUG_LOCK_ALLOC=n
 CONFIG_DEBUG_OBJECTS_RCU_HEAD=n
-CONFIG_PREEMPT_COUNT=n
tools/testing/selftests/rcutorture/doc/TINY_RCU.txt +2 -3
index a75b16991a92..d30cedf07826 100644
@@ -3,11 +3,10 @@ This document gives a brief rationale for the TINY_RCU test cases.
 
 Kconfig Parameters:
 
-CONFIG_DEBUG_LOCK_ALLOC -- Do all three and none of the three.
-CONFIG_PREEMPT_COUNT
+CONFIG_DEBUG_LOCK_ALLOC -- Do both and none of the two.
 CONFIG_RCU_TRACE
 
-The theory here is that randconfig testing will hit the other six possible
+The theory here is that randconfig testing will hit the other two possible
 combinations of these parameters.
 
 
tools/testing/selftests/rcutorture/doc/TREE_RCU-kconfig.txt +0 -1
index 1b96d68473b8..cfdd48f689de 100644
@@ -43,7 +43,6 @@ CONFIG_64BIT
 
 	Used only to check CONFIG_RCU_FANOUT value, inspection suffices.
 
-CONFIG_PREEMPT_COUNT
 CONFIG_PREEMPT_RCU
 
 	Redundant with CONFIG_PREEMPT, ignore.
tools/testing/selftests/rcutorture/formal/srcu-cbmc/src/config.h +0 -1
index 283d7103334f..d0d485d48649 100644
@@ -8,7 +8,6 @@
 #undef CONFIG_HOTPLUG_CPU
 #undef CONFIG_MODULES
 #undef CONFIG_NO_HZ_FULL_SYSIDLE
-#undef CONFIG_PREEMPT_COUNT
 #undef CONFIG_PREEMPT_RCU
 #undef CONFIG_PROVE_RCU
 #undef CONFIG_RCU_NOCB_CPU
Signature · 1 line
2.20.1
author maintainer
#15 lore raw

PATCH 11/16 xtensa: Cleanup PREEMPT_COUNT leftovers

In reply to Uladzislau Rezki (Sony)

From: Thomas Gleixner <[email protected]>

CONFIG_PREEMPT_COUNT is now unconditionally enabled and will be removed. Cleanup the leftovers before doing so.

---
 arch/xtensa/kernel/entry.S | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
arch/xtensa/kernel/entry.S +1 -1
index 703cf6205efe..5a27dd34d3fc 100644
@@ -819,7 +819,7 @@ ENTRY(debug_exception)
 	 * preemption if we have HW breakpoints to preserve DEBUGCAUSE.DBNUM
 	 * meaning.
 	 */
-#if defined(CONFIG_PREEMPT_COUNT) && defined(CONFIG_HAVE_HW_BREAKPOINT)
+#ifdef CONFIG_HAVE_HW_BREAKPOINT
 	GET_THREAD_INFO(a2, a1)
 	l32i	a3, a2, TI_PRE_COUNT
 	addi	a3, a3, 1
Signature · 1 line
2.20.1
author maintainer
#3 lore raw

PATCH 4/16 preempt: Cleanup PREEMPT_COUNT leftovers

In reply to Uladzislau Rezki (Sony)

From: Thomas Gleixner <[email protected]>

CONFIG_PREEMPT_COUNT is now unconditionally enabled and will be removed. Cleanup the leftovers before doing so.

Cc Ingo Molnar <[email protected]>
Cc Peter Zijlstra <[email protected]>
Cc Juri Lelli <[email protected]>
Cc Vincent Guittot <[email protected]>
Cc Dietmar Eggemann <[email protected]>
Cc Steven Rostedt <[email protected]>
Cc Ben Segall <[email protected]>
Cc Mel Gorman <[email protected]>
Cc Daniel Bristot de Oliveira <[email protected]>
---
 include/linux/preempt.h | 37 ++++---------------------------------
 1 file changed, 4 insertions(+), 33 deletions(-)
include/linux/preempt.h +4 -33
index 7d9c1c0e149c..513769b1edf8 100644
@@ -56,8 +56,7 @@
 #define PREEMPT_DISABLED	(PREEMPT_DISABLE_OFFSET + PREEMPT_ENABLED)
 
 /*
- * Disable preemption until the scheduler is running -- use an unconditional
- * value so that it also works on !PREEMPT_COUNT kernels.
+ * Disable preemption until the scheduler is running.
  *
  * Reset by start_kernel()->sched_init()->init_idle()->init_idle_preempt_count().
  */
@@ -69,7 +68,6 @@
  *
  *    preempt_count() == 2*PREEMPT_DISABLE_OFFSET
  *
- * Note: PREEMPT_DISABLE_OFFSET is 0 for !PREEMPT_COUNT kernels.
  * Note: See finish_task_switch().
  */
 #define FORK_PREEMPT_COUNT	(2*PREEMPT_DISABLE_OFFSET + PREEMPT_ENABLED)
@@ -106,11 +104,7 @@
 /*
  * The preempt_count offset after preempt_disable();
  */
-#if defined(CONFIG_PREEMPT_COUNT)
-# define PREEMPT_DISABLE_OFFSET	PREEMPT_OFFSET
-#else
-# define PREEMPT_DISABLE_OFFSET	0
-#endif
+#define PREEMPT_DISABLE_OFFSET	PREEMPT_OFFSET
 
 /*
  * The preempt_count offset after spin_lock()
@@ -122,8 +116,8 @@
  *
  *  spin_lock_bh()
  *
- * Which need to disable both preemption (CONFIG_PREEMPT_COUNT) and
- * softirqs, such that unlock sequences of:
+ * Which need to disable both preemption and softirqs, such that unlock
+ * sequences of:
  *
  *  spin_unlock();
  *  local_bh_enable();
@@ -164,8 +158,6 @@ extern void preempt_count_sub(int val);
 #define preempt_count_inc() preempt_count_add(1)
 #define preempt_count_dec() preempt_count_sub(1)
 
-#ifdef CONFIG_PREEMPT_COUNT
-
 #define preempt_disable() \
 do { \
 	preempt_count_inc(); \
@@ -231,27 +223,6 @@ do { \
 	__preempt_count_dec(); \
 } while (0)
 
-#else /* !CONFIG_PREEMPT_COUNT */
-
-/*
- * Even if we don't have any preemption, we need preempt disable/enable
- * to be barriers, so that we don't have things like get_user/put_user
- * that can cause faults and scheduling migrate into our preempt-protected
- * region.
- */
-#define preempt_disable()			barrier()
-#define sched_preempt_enable_no_resched()	barrier()
-#define preempt_enable_no_resched()		barrier()
-#define preempt_enable()			barrier()
-#define preempt_check_resched()			do { } while (0)
-
-#define preempt_disable_notrace()		barrier()
-#define preempt_enable_no_resched_notrace()	barrier()
-#define preempt_enable_notrace()		barrier()
-#define preemptible()				0
-
-#endif /* CONFIG_PREEMPT_COUNT */
-
 #ifdef MODULE
 /*
  * Modules have no business playing preemption tricks.
Signature · 1 line
2.20.1
author maintainer
#4 lore raw

PATCH 5/16 lockdep: Cleanup PREEMPT_COUNT leftovers

In reply to Uladzislau Rezki (Sony)

From: Thomas Gleixner <[email protected]>

CONFIG_PREEMPT_COUNT is now unconditionally enabled and will be removed. Cleanup the leftovers before doing so.

Cc Peter Zijlstra <[email protected]>
Cc Ingo Molnar <[email protected]>
Cc Will Deacon <[email protected]>
[ Rezki: Adopted for 5.10.0-rc1 kernel. ]
---
 include/linux/lockdep.h | 6 ++----
 lib/Kconfig.debug       | 1 -
 2 files changed, 2 insertions(+), 5 deletions(-)
include/linux/lockdep.h +2 -4
index f5594879175a..d05db575f60f 100644
@@ -580,16 +580,14 @@ do {									\
 
 #define lockdep_assert_preemption_enabled()				\
 do {									\
-	WARN_ON_ONCE(IS_ENABLED(CONFIG_PREEMPT_COUNT)	&&		\
-		     __lockdep_enabled			&&		\
+	WARN_ON_ONCE(__lockdep_enabled			&&		\
 		     (preempt_count() != 0		||		\
 		      !this_cpu_read(hardirqs_enabled)));		\
 } while (0)
 
 #define lockdep_assert_preemption_disabled()				\
 do {									\
-	WARN_ON_ONCE(IS_ENABLED(CONFIG_PREEMPT_COUNT)	&&		\
-		     __lockdep_enabled			&&		\
+	WARN_ON_ONCE(__lockdep_enabled			&&		\
 		     (preempt_count() == 0		&&		\
 		      this_cpu_read(hardirqs_enabled)));		\
 } while (0)
lib/Kconfig.debug +0 -1
index 89c9a177fb9b..03a85065805e 100644
@@ -1159,7 +1159,6 @@ config PROVE_LOCKING
 	select DEBUG_RWSEMS
 	select DEBUG_WW_MUTEX_SLOWPATH
 	select DEBUG_LOCK_ALLOC
-	select PREEMPT_COUNT
 	select TRACE_IRQFLAGS
 	default n
 	help
Signature · 1 line
2.20.1
author maintainer
#2 lore raw

PATCH 3/16 preempt: Make preempt count unconditional

In reply to Uladzislau Rezki (Sony)

From: Thomas Gleixner <[email protected]>

The handling of preempt_count() is inconsistent accross kernel configurations. On kernels which have PREEMPT_COUNT=n preempt_disable/enable() and the lock/unlock functions are not affecting the preempt count, only local_bh_disable/enable() and _bh variants of locking, soft interrupt delivery, hard interrupt and NMI context affect it.

It's therefore impossible to have a consistent set of checks which provide information about the context in which a function is called. In many cases it makes sense to have seperate functions for seperate contexts, but there are valid reasons to avoid that and handle different calling contexts conditionally.

The lack of such indicators which work on all kernel configuratios is a constant source of trouble because developers either do not understand the implications or try to work around this inconsistency in weird ways. Neither seem these issues be catched by reviewers and testing.

Recently merged code does:

	 gfp = preemptible() ? GFP_KERNEL : GFP_ATOMIC;

Looks obviously correct, except for the fact that preemptible() is unconditionally false for CONFIF_PREEMPT_COUNT=n, i.e. all allocations in that code use GFP_ATOMIC on such kernels.

Attempts to make preempt count unconditional and consistent have been rejected in the past with handwaving performance arguments.

Freshly conducted benchmarks did not reveal any measurable impact from enabling preempt count unconditionally. On kernels with CONFIG_PREEMPT_NONE or CONFIG_PREEMPT_VOLUNTARY the preempt count is only incremented and decremented but the result of the decrement is not tested. Contrary to that enabling CONFIG_PREEMPT which tests the result has a small but measurable impact due to the conditional branch/call.

It's about time to make essential functionality of the kernel consistent accross the various preemption models.

Enable CONFIG_PREEMPT_COUNT unconditionally. Follow up changes will remove the #ifdeffery and remove the config option at the end.

---
 kernel/Kconfig.preempt | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)
kernel/Kconfig.preempt +1 -2
index bf82259cff96..3f4712ff073b 100644
@@ -75,8 +75,7 @@ config PREEMPT_RT
 endchoice
 
 config PREEMPT_COUNT
-       bool
+       def_bool y
 
 config PREEMPTION
        bool
-       select PREEMPT_COUNT
Signature · 1 line
2.20.1
maintainer
#24 lore raw

Re: [PATCH 16/16] rcu/tree: Use delayed work instead of hrtimer to refill the cache

In reply to Uladzislau Rezki (Sony)
Quoted 5 lines · Uladzislau Rezki (Sony)

A CONFIG_PREEMPT_COUNT is unconditionally enabled, thus a page can be obtained directly from a kvfree_rcu() path. To distinguish that and take a decision the preemptable() macro is used when it is save to enter allocator.

Quoted 7 lines

It means that refilling a cache is not important from timing point of view. Switch to a delayed work, so the actual work is queued from the timer interrupt with 1 jiffy delay. An immediate placing a task on a current CPU can lead to rq->lock double lock. That is why a delayed method is in place.

Thank you, Uladzislau!

I applied this on top of v5.10-rc1 and got the following from the single-CPU builds:

SYNC include/config/auto.conf.cmd
DESCEND objtool

  CC      kernel/bounds.s
  CALL    scripts/atomic/check-atomics.sh
  UPD     include/generated/bounds.h
  CC      arch/x86/kernel/asm-offsets.s
In file included from ./include/asm-generic/atomic-instrumented.h:20:0,
                 from ./include/linux/atomic.h:82,
                 from ./include/linux/crypto.h:15,
                 from arch/x86/kernel/asm-offsets.c:9:
./include/linux/pagemap.h: In function ‘__page_cache_add_speculative’:
./include/linux/build_bug.h:30:34: error: called object is not a function or function pointer
 #define BUILD_BUG_ON_INVALID(e) ((void)(sizeof((__force long)(e))))
                                 ~^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
./include/linux/mmdebug.h:45:25: note: in expansion of macro ‘BUILD_BUG_ON_INVALID’
 #define VM_BUG_ON(cond) BUILD_BUG_ON_INVALID(cond)
                         ^~~~~~~~~~~~~~~~~~~~

./include/linux/pagemap.h:207:2: note: in expansion of macro ‘VM_BUG_ON’
VM_BUG_ON(preemptible())
^~~~~~~~~
scripts/Makefile.build:117: recipe for target 'arch/x86/kernel/asm-offsets.s' failed make[1]: *** [arch/x86/kernel/asm-offsets.s] Error 1
Makefile:1199: recipe for target 'prepare0' failed
make: *** [prepare0] Error 2

I vaguely recall something like this showing up in the previous series and that we did something or another to address it. Could you please check against the old series at -rcu branch dev.2020.10.22a? (I verified that the old series does run correctly in the single-CPU scenarios.)

Thanx, Paul

Quoted 78 lines
---
 kernel/rcu/tree.c | 26 +++++---------------------
 1 file changed, 5 insertions(+), 21 deletions(-)
kernel/rcu/tree.c +5 -21
index 3f9b016a44dc..9ea55f800b4b 100644
@@ -3086,7 +3086,6 @@ struct kfree_rcu_cpu_work {
  *	per-cpu lock.
  * @page_cache_work: A work to refill the cache when it is empty
  * @work_in_progress: Indicates that page_cache_work is running
- * @hrtimer: A hrtimer for scheduling a page_cache_work
  * @nr_bkv_objs: number of allocated objects at @bkvcache.
  *
  * This is a per-CPU structure.  The reason that it is not included in
@@ -3104,9 +3103,8 @@ struct kfree_rcu_cpu {
 	bool initialized;
 	int count;
 
-	struct work_struct page_cache_work;
+	struct delayed_work page_cache_work;
 	atomic_t work_in_progress;
-	struct hrtimer hrtimer;
 
 	struct llist_head bkvcache;
 	int nr_bkv_objs;
@@ -3355,22 +3353,12 @@ static void kfree_rcu_monitor(struct work_struct *work)
 		raw_spin_unlock_irqrestore(&krcp->lock, flags);
 }
 
-static enum hrtimer_restart
-schedule_page_work_fn(struct hrtimer *t)
-{
-	struct kfree_rcu_cpu *krcp =
-		container_of(t, struct kfree_rcu_cpu, hrtimer);
-
-	queue_work(system_highpri_wq, &krcp->page_cache_work);
-	return HRTIMER_NORESTART;
-}
-
 static void fill_page_cache_func(struct work_struct *work)
 {
 	struct kvfree_rcu_bulk_data *bnode;
 	struct kfree_rcu_cpu *krcp =
 		container_of(work, struct kfree_rcu_cpu,
-			page_cache_work);
+			page_cache_work.work);
 	unsigned long flags;
 	bool pushed;
 	int i;
@@ -3398,12 +3386,8 @@ static void
 run_page_cache_worker(struct kfree_rcu_cpu *krcp)
 {
 	if (rcu_scheduler_active == RCU_SCHEDULER_RUNNING &&
-			!atomic_xchg(&krcp->work_in_progress, 1)) {
-		hrtimer_init(&krcp->hrtimer, CLOCK_MONOTONIC,
-			HRTIMER_MODE_REL);
-		krcp->hrtimer.function = schedule_page_work_fn;
-		hrtimer_start(&krcp->hrtimer, 0, HRTIMER_MODE_REL);
-	}
+			!atomic_xchg(&krcp->work_in_progress, 1))
+		schedule_delayed_work(&krcp->page_cache_work, 1);
 }
 
 // Record ptr in a page managed by krcp, with the pre-krc_this_cpu_lock()
@@ -4503,7 +4487,7 @@ static void __init kfree_rcu_batch_init(void)
 		}
 
 		INIT_DELAYED_WORK(&krcp->monitor_work, kfree_rcu_monitor);
-		INIT_WORK(&krcp->page_cache_work, fill_page_cache_func);
+		INIT_DELAYED_WORK(&krcp->page_cache_work, fill_page_cache_func);
 		krcp->initialized = true;
 	}
 	if (register_shrinker(&kfree_rcu_shrinker))
Signature · 1 line
2.20.1
author maintainer
#25 lore raw

Re: [PATCH 16/16] rcu/tree: Use delayed work instead of hrtimer to refill the cache

Quoted 44 lines · Paul E. McKenney
Quoted 5 lines · Uladzislau Rezki (Sony)

A CONFIG_PREEMPT_COUNT is unconditionally enabled, thus a page can be obtained directly from a kvfree_rcu() path. To distinguish that and take a decision the preemptable() macro is used when it is save to enter allocator.

Quoted 7 lines

It means that refilling a cache is not important from timing point of view. Switch to a delayed work, so the actual work is queued from the timer interrupt with 1 jiffy delay. An immediate placing a task on a current CPU can lead to rq->lock double lock. That is why a delayed method is in place.

Thank you, Uladzislau!

I applied this on top of v5.10-rc1 and got the following from the single-CPU builds:

SYNC include/config/auto.conf.cmd
DESCEND objtool

  CC      kernel/bounds.s
  CALL    scripts/atomic/check-atomics.sh
  UPD     include/generated/bounds.h
  CC      arch/x86/kernel/asm-offsets.s
In file included from ./include/asm-generic/atomic-instrumented.h:20:0,
                 from ./include/linux/atomic.h:82,
                 from ./include/linux/crypto.h:15,
                 from arch/x86/kernel/asm-offsets.c:9:
./include/linux/pagemap.h: In function ‘__page_cache_add_speculative’:
./include/linux/build_bug.h:30:34: error: called object is not a function or function pointer
 #define BUILD_BUG_ON_INVALID(e) ((void)(sizeof((__force long)(e))))
                                 ~^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
./include/linux/mmdebug.h:45:25: note: in expansion of macro ‘BUILD_BUG_ON_INVALID’
 #define VM_BUG_ON(cond) BUILD_BUG_ON_INVALID(cond)
                         ^~~~~~~~~~~~~~~~~~~~

./include/linux/pagemap.h:207:2: note: in expansion of macro ‘VM_BUG_ON’
VM_BUG_ON(preemptible())
^~~~~~~~~
scripts/Makefile.build:117: recipe for target 'arch/x86/kernel/asm-offsets.s' failed make[1]: *** [arch/x86/kernel/asm-offsets.s] Error 1
Makefile:1199: recipe for target 'prepare0' failed
make: *** [prepare0] Error 2

Quoted 5 lines

I vaguely recall something like this showing up in the previous series and that we did something or another to address it. Could you please check against the old series at -rcu branch dev.2020.10.22a? (I verified that the old series does run correctly in the single-CPU scenarios.)

I see the same build error. Will double check if we have similar in the previous series also. It looks like the error is caused by the Thomas series.

Will check!

--
Vlad Rezki

author maintainer
#26 lore raw

Re: [PATCH 16/16] rcu/tree: Use delayed work instead of hrtimer to refill the cache

Quoted 50 lines · Uladzislau Rezki
Quoted 44 lines · Paul E. McKenney
Quoted 5 lines · Uladzislau Rezki (Sony)

A CONFIG_PREEMPT_COUNT is unconditionally enabled, thus a page can be obtained directly from a kvfree_rcu() path. To distinguish that and take a decision the preemptable() macro is used when it is save to enter allocator.

Quoted 7 lines

It means that refilling a cache is not important from timing point of view. Switch to a delayed work, so the actual work is queued from the timer interrupt with 1 jiffy delay. An immediate placing a task on a current CPU can lead to rq->lock double lock. That is why a delayed method is in place.

Thank you, Uladzislau!

I applied this on top of v5.10-rc1 and got the following from the single-CPU builds:

SYNC include/config/auto.conf.cmd
DESCEND objtool

  CC      kernel/bounds.s
  CALL    scripts/atomic/check-atomics.sh
  UPD     include/generated/bounds.h
  CC      arch/x86/kernel/asm-offsets.s
In file included from ./include/asm-generic/atomic-instrumented.h:20:0,
                 from ./include/linux/atomic.h:82,
                 from ./include/linux/crypto.h:15,
                 from arch/x86/kernel/asm-offsets.c:9:
./include/linux/pagemap.h: In function ‘__page_cache_add_speculative’:
./include/linux/build_bug.h:30:34: error: called object is not a function or function pointer
 #define BUILD_BUG_ON_INVALID(e) ((void)(sizeof((__force long)(e))))
                                 ~^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
./include/linux/mmdebug.h:45:25: note: in expansion of macro ‘BUILD_BUG_ON_INVALID’
 #define VM_BUG_ON(cond) BUILD_BUG_ON_INVALID(cond)
                         ^~~~~~~~~~~~~~~~~~~~

./include/linux/pagemap.h:207:2: note: in expansion of macro ‘VM_BUG_ON’
VM_BUG_ON(preemptible())
^~~~~~~~~
scripts/Makefile.build:117: recipe for target 'arch/x86/kernel/asm-offsets.s' failed make[1]: *** [arch/x86/kernel/asm-offsets.s] Error 1
Makefile:1199: recipe for target 'prepare0' failed
make: *** [prepare0] Error 2

Quoted 5 lines

I vaguely recall something like this showing up in the previous series and that we did something or another to address it. Could you please check against the old series at -rcu branch dev.2020.10.22a? (I verified that the old series does run correctly in the single-CPU scenarios.)

Quoted 5 lines

I see the same build error. Will double check if we have similar in the previous series also. It looks like the error is caused by the Thomas series.

Will check!

OK. Found it:

urezki@pc638:~/data/coding/linux.git$ git diff
include/linux/pagemap.h +1 -1
index cbfbe2bcca75..7dd3523093db 100644
@@ -204,7 +204,7 @@ void release_pages(struct page **pages, int nr);
 static inline int __page_cache_add_speculative(struct page *page, int count)
 {
 #ifdef CONFIG_TINY_RCU
-       VM_BUG_ON(preemptible())
+       VM_BUG_ON(preemptible());
        /*
         * Preempt must be disabled here - we rely on rcu_read_lock doing
         * this for us.

urezki@pc638:~/data/coding/linux.git$

I guess we had some extra patch that fixes a kernel compilation for !SMP case. Will check dev.2020.10.22a.

--
Vlad Rezki

maintainer
#27 lore raw

Re: [PATCH 16/16] rcu/tree: Use delayed work instead of hrtimer to refill the cache

Quoted 74 lines · Uladzislau Rezki
Quoted 50 lines · Uladzislau Rezki
Quoted 44 lines · Paul E. McKenney
Quoted 5 lines · Uladzislau Rezki (Sony)

A CONFIG_PREEMPT_COUNT is unconditionally enabled, thus a page can be obtained directly from a kvfree_rcu() path. To distinguish that and take a decision the preemptable() macro is used when it is save to enter allocator.

Quoted 7 lines

It means that refilling a cache is not important from timing point of view. Switch to a delayed work, so the actual work is queued from the timer interrupt with 1 jiffy delay. An immediate placing a task on a current CPU can lead to rq->lock double lock. That is why a delayed method is in place.

Thank you, Uladzislau!

I applied this on top of v5.10-rc1 and got the following from the single-CPU builds:

SYNC include/config/auto.conf.cmd
DESCEND objtool

  CC      kernel/bounds.s
  CALL    scripts/atomic/check-atomics.sh
  UPD     include/generated/bounds.h
  CC      arch/x86/kernel/asm-offsets.s
In file included from ./include/asm-generic/atomic-instrumented.h:20:0,
                 from ./include/linux/atomic.h:82,
                 from ./include/linux/crypto.h:15,
                 from arch/x86/kernel/asm-offsets.c:9:
./include/linux/pagemap.h: In function ‘__page_cache_add_speculative’:
./include/linux/build_bug.h:30:34: error: called object is not a function or function pointer
 #define BUILD_BUG_ON_INVALID(e) ((void)(sizeof((__force long)(e))))
                                 ~^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
./include/linux/mmdebug.h:45:25: note: in expansion of macro ‘BUILD_BUG_ON_INVALID’
 #define VM_BUG_ON(cond) BUILD_BUG_ON_INVALID(cond)
                         ^~~~~~~~~~~~~~~~~~~~

./include/linux/pagemap.h:207:2: note: in expansion of macro ‘VM_BUG_ON’
VM_BUG_ON(preemptible())
^~~~~~~~~
scripts/Makefile.build:117: recipe for target 'arch/x86/kernel/asm-offsets.s' failed make[1]: *** [arch/x86/kernel/asm-offsets.s] Error 1
Makefile:1199: recipe for target 'prepare0' failed
make: *** [prepare0] Error 2

Quoted 5 lines

I vaguely recall something like this showing up in the previous series and that we did something or another to address it. Could you please check against the old series at -rcu branch dev.2020.10.22a? (I verified that the old series does run correctly in the single-CPU scenarios.)

Quoted 5 lines

I see the same build error. Will double check if we have similar in the previous series also. It looks like the error is caused by the Thomas series.

Will check!

OK. Found it:

urezki@pc638:~/data/coding/linux.git$ git diff
include/linux/pagemap.h +1 -1
index cbfbe2bcca75..7dd3523093db 100644
@@ -204,7 +204,7 @@ void release_pages(struct page **pages, int nr);
 static inline int __page_cache_add_speculative(struct page *page, int count)
 {
 #ifdef CONFIG_TINY_RCU
-       VM_BUG_ON(preemptible())
+       VM_BUG_ON(preemptible());
        /*
         * Preempt must be disabled here - we rely on rcu_read_lock doing
         * this for us.

urezki@pc638:~/data/coding/linux.git$

Quoted 2 lines

I guess we had some extra patch that fixes a kernel compilation for !SMP case. Will check dev.2020.10.22a.

I guess I am blind today!

And yes, that semicolon is in the corresponding commit in -tip.

So, an important safety tip: When forward porting, start from the commits that have been tested rather than from the original series of patches. ;-)

Thanx, Paul

author maintainer
#6 lore raw

Re: [PATCH 06/16] mm/pagemap: Cleanup PREEMPT_COUNT leftovers

In reply to Uladzislau Rezki (Sony)
Quoted 31 lines · Uladzislau Rezki (Sony)

From: Thomas Gleixner <[email protected]>

CONFIG_PREEMPT_COUNT is now unconditionally enabled and will be removed. Cleanup the leftovers before doing so.

---
 include/linux/pagemap.h | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)
include/linux/pagemap.h +1 -3
index c77b7c31b2e4..cbfbe2bcca75 100644
@@ -204,9 +204,7 @@ void release_pages(struct page **pages, int nr);
 static inline int __page_cache_add_speculative(struct page *page, int count)
 {
 #ifdef CONFIG_TINY_RCU
-# ifdef CONFIG_PREEMPT_COUNT
-	VM_BUG_ON(!in_atomic() && !irqs_disabled());
-# endif
+	VM_BUG_ON(preemptible())
 	/*
 	 * Preempt must be disabled here - we rely on rcu_read_lock doing
 	 * this for us.
Signature · 1 line
2.20.1

Hello, Paul.

Sorry for a small mistake, it was fixed by you before, whereas i took an old version of the patch that is question. Please use below one instead of posted one:

Author: Thomas Gleixner <[email protected]>
Date:   Mon Sep 14 19:25:00 2020 +0200

mm/pagemap: Cleanup PREEMPT_COUNT leftovers

CONFIG_PREEMPT_COUNT is now unconditionally enabled and will be removed. Cleanup the leftovers before doing so.

Cc Andrew Morton <[email protected]>
[ paulmck: Fix !SMP build error per kernel test robot feedback. ]
include/linux/pagemap.h +1 -3
index 7de11dcd534d..b3d9d9217ea0 100644
@@ -168,9 +168,7 @@ void release_pages(struct page **pages, int nr);
 static inline int __page_cache_add_speculative(struct page *page, int count)
 {
 #ifdef CONFIG_TINY_RCU
-# ifdef CONFIG_PREEMPT_COUNT
-       VM_BUG_ON(!in_atomic() && !irqs_disabled());
-# endif
+       VM_BUG_ON(preemptible());
        /*
         * Preempt must be disabled here - we rely on rcu_read_lock doing
         * this for us.

--
Vlad Rezki

author maintainer
#28 lore raw

Re: [PATCH 16/16] rcu/tree: Use delayed work instead of hrtimer to refill the cache

In reply to Paul E. McKenney
Quoted 80 lines · Paul E. McKenney
Quoted 74 lines · Uladzislau Rezki
Quoted 50 lines · Uladzislau Rezki
Quoted 44 lines · Paul E. McKenney
Quoted 5 lines · Uladzislau Rezki (Sony)

A CONFIG_PREEMPT_COUNT is unconditionally enabled, thus a page can be obtained directly from a kvfree_rcu() path. To distinguish that and take a decision the preemptable() macro is used when it is save to enter allocator.

Quoted 7 lines

It means that refilling a cache is not important from timing point of view. Switch to a delayed work, so the actual work is queued from the timer interrupt with 1 jiffy delay. An immediate placing a task on a current CPU can lead to rq->lock double lock. That is why a delayed method is in place.

Thank you, Uladzislau!

I applied this on top of v5.10-rc1 and got the following from the single-CPU builds:

SYNC include/config/auto.conf.cmd
DESCEND objtool

  CC      kernel/bounds.s
  CALL    scripts/atomic/check-atomics.sh
  UPD     include/generated/bounds.h
  CC      arch/x86/kernel/asm-offsets.s
In file included from ./include/asm-generic/atomic-instrumented.h:20:0,
                 from ./include/linux/atomic.h:82,
                 from ./include/linux/crypto.h:15,
                 from arch/x86/kernel/asm-offsets.c:9:
./include/linux/pagemap.h: In function ‘__page_cache_add_speculative’:
./include/linux/build_bug.h:30:34: error: called object is not a function or function pointer
 #define BUILD_BUG_ON_INVALID(e) ((void)(sizeof((__force long)(e))))
                                 ~^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
./include/linux/mmdebug.h:45:25: note: in expansion of macro ‘BUILD_BUG_ON_INVALID’
 #define VM_BUG_ON(cond) BUILD_BUG_ON_INVALID(cond)
                         ^~~~~~~~~~~~~~~~~~~~

./include/linux/pagemap.h:207:2: note: in expansion of macro ‘VM_BUG_ON’
VM_BUG_ON(preemptible())
^~~~~~~~~
scripts/Makefile.build:117: recipe for target 'arch/x86/kernel/asm-offsets.s' failed make[1]: *** [arch/x86/kernel/asm-offsets.s] Error 1
Makefile:1199: recipe for target 'prepare0' failed
make: *** [prepare0] Error 2

Quoted 5 lines

I vaguely recall something like this showing up in the previous series and that we did something or another to address it. Could you please check against the old series at -rcu branch dev.2020.10.22a? (I verified that the old series does run correctly in the single-CPU scenarios.)

Quoted 5 lines

I see the same build error. Will double check if we have similar in the previous series also. It looks like the error is caused by the Thomas series.

Will check!

OK. Found it:

urezki@pc638:~/data/coding/linux.git$ git diff
include/linux/pagemap.h +1 -1
index cbfbe2bcca75..7dd3523093db 100644
@@ -204,7 +204,7 @@ void release_pages(struct page **pages, int nr);
 static inline int __page_cache_add_speculative(struct page *page, int count)
 {
 #ifdef CONFIG_TINY_RCU
-       VM_BUG_ON(preemptible())
+       VM_BUG_ON(preemptible());
        /*
         * Preempt must be disabled here - we rely on rcu_read_lock doing
         * this for us.

urezki@pc638:~/data/coding/linux.git$

Quoted 2 lines

I guess we had some extra patch that fixes a kernel compilation for !SMP case. Will check dev.2020.10.22a.

I guess I am blind today!

Quoted 7 lines

And yes, that semicolon is in the corresponding commit in -tip.

So, an important safety tip: When forward porting, start from the commits that have been tested rather than from the original series of patches. ;-)

Thanx, Paul

Right. Sorry for that. I remember you mentioned about a build error some time ago. My bad :)

I have updated the: [PATCH 06/16] mm/pagemap: Cleanup PREEMPT_COUNT leftovers in this series.

--
Vlad Rezki

#8 lore raw

Re: [PATCH 06/16] mm/pagemap: Cleanup PREEMPT_COUNT leftovers

In reply to Uladzislau Rezki (Sony)
I love your patch! Yet something to improve:

Hi "Uladzislau,

I love your patch! Yet something to improve:

[auto build test ERROR on arm/for-next]
[also build test ERROR on tip/sched/core linus/master v5.10-rc1] [cannot apply to rcu/dev drm-intel/for-linux-next next-20201029] [If your patch is applied to the wrong git tree, kindly drop us a note. And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch]

url:    https://github.com/0day-ci/linux/commits/Uladzislau-Rezki-Sony/rcu-tree-Add-a-work-to-allocate-pages-from-regular-context/20201030-005247
base:   git://git.armlinux.org.uk/~rmk/linux-arm.git for-next
config: i386-randconfig-a016-20201029 (attached as .config)
compiler: gcc-9 (Debian 9.3.0-15) 9.3.0
reproduce (this is a W=1 build):
        # https://github.com/0day-ci/linux/commit/1e0b66ce2b779463194f8b398b8e48caa95aabf4
        git remote add linux-review https://github.com/0day-ci/linux
        git fetch --no-tags linux-review Uladzislau-Rezki-Sony/rcu-tree-Add-a-work-to-allocate-pages-from-regular-context/20201030-005247
        git checkout 1e0b66ce2b779463194f8b398b8e48caa95aabf4
        # save the attached .config to linux build tree
        make W=1 ARCH=i386 

If you fix the issue, kindly add following tag as appropriate

All errors (new ones prefixed by >>):

In file included from include/asm-generic/atomic-instrumented.h:20,

                    from include/linux/atomic.h:82,
                    from include/linux/crypto.h:15,
                    from arch/x86/kernel/asm-offsets.c:9:
   include/linux/pagemap.h: In function '__page_cache_add_speculative':
>> include/linux/build_bug.h:30:34: error: called object is not a function or function pointer
      30 | #define BUILD_BUG_ON_INVALID(e) ((void)(sizeof((__force long)(e))))
         |                                 ~^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
   include/linux/mmdebug.h:45:25: note: in expansion of macro 'BUILD_BUG_ON_INVALID'
      45 | #define VM_BUG_ON(cond) BUILD_BUG_ON_INVALID(cond)
         |                         ^~~~~~~~~~~~~~~~~~~~
   include/linux/pagemap.h:207:2: note: in expansion of macro 'VM_BUG_ON'
     207 |  VM_BUG_ON(preemptible())
         |  ^~~~~~~~~

make[2]: *** [scripts/Makefile.build:117: arch/x86/kernel/asm-offsets.s] Error 1 make[2]: Target '__build' not remade because of errors.
make[1]: *** [Makefile:1200: prepare0] Error 2
make[1]: Target 'prepare' not remade because of errors.
make: *** [Makefile:185: __sub-make] Error 2
make: Target 'prepare' not remade because of errors.

vim +30 include/linux/build_bug.h

527edbc18a70e74 Masahiro Yamada 2019-01-03  18  
527edbc18a70e74 Masahiro Yamada 2019-01-03  19  /* Force a compilation error if a constant expression is not a power of 2 */
527edbc18a70e74 Masahiro Yamada 2019-01-03  20  #define __BUILD_BUG_ON_NOT_POWER_OF_2(n)	\
527edbc18a70e74 Masahiro Yamada 2019-01-03  21  	BUILD_BUG_ON(((n) & ((n) - 1)) != 0)
527edbc18a70e74 Masahiro Yamada 2019-01-03  22  #define BUILD_BUG_ON_NOT_POWER_OF_2(n)			\
527edbc18a70e74 Masahiro Yamada 2019-01-03  23  	BUILD_BUG_ON((n) == 0 || (((n) & ((n) - 1)) != 0))
bc6245e5efd70c4 Ian Abbott      2017-07-10  24  
bc6245e5efd70c4 Ian Abbott      2017-07-10  25  /*
bc6245e5efd70c4 Ian Abbott      2017-07-10  26   * BUILD_BUG_ON_INVALID() permits the compiler to check the validity of the
bc6245e5efd70c4 Ian Abbott      2017-07-10  27   * expression but avoids the generation of any code, even if that expression
bc6245e5efd70c4 Ian Abbott      2017-07-10  28   * has side-effects.
bc6245e5efd70c4 Ian Abbott      2017-07-10  29   */

bc6245e5efd70c4 Ian Abbott 2017-07-10 @30 #define BUILD_BUG_ON_INVALID(e) ((void)(sizeof((__force long)(e))))
bc6245e5efd70c4 Ian Abbott 2017-07-10 31

---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/kbuild-all(a)lists.01.org

#9 lore raw

Re: [PATCH 06/16] mm/pagemap: Cleanup PREEMPT_COUNT leftovers

In reply to Uladzislau Rezki (Sony)
I love your patch! Perhaps something to improve:

Hi "Uladzislau,

I love your patch! Perhaps something to improve:

[auto build test WARNING on arm/for-next]
[also build test WARNING on tip/sched/core linus/master v5.10-rc1] [cannot apply to rcu/dev drm-intel/for-linux-next next-20201029] [If your patch is applied to the wrong git tree, kindly drop us a note. And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch]

url:    https://github.com/0day-ci/linux/commits/Uladzislau-Rezki-Sony/rcu-tree-Add-a-work-to-allocate-pages-from-regular-context/20201030-005247
base:   git://git.armlinux.org.uk/~rmk/linux-arm.git for-next
config: arm-randconfig-r004-20201029 (attached as .config)
compiler: clang version 12.0.0 (https://github.com/llvm/llvm-project 772aaa602383cf82795572ebcd86b0e660f3579f)

reproduce (this is a W=1 build):
wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
chmod +x ~/bin/make.cross
# install arm cross compiling tool for clang build
# apt-get install binutils-arm-linux-gnueabi
# https://github.com/0day-ci/linux/commit/1e0b66ce2b779463194f8b398b8e48caa95aabf4
git remote add linux-review https://github.com/0day-ci/linux
git fetch --no-tags linux-review Uladzislau-Rezki-Sony/rcu-tree-Add-a-work-to-allocate-pages-from-regular-context/20201030-005247 git checkout 1e0b66ce2b779463194f8b398b8e48caa95aabf4
# save the attached .config to linux build tree
COMPILER_INSTALL_PATH=$HOME/0day COMPILER=clang make.cross ARCH=arm

If you fix the issue, kindly add following tag as appropriate

All warnings (new ones prefixed by >>):

In file included from drivers/char/random.c:348:
In file included from include/trace/events/random.h:8:
In file included from include/linux/writeback.h:14:
In file included from include/linux/blk-cgroup.h:23:
In file included from include/linux/blkdev.h:14:
include/linux/pagemap.h:207:26: error: expected ';' after do/while statement

           VM_BUG_ON(preemptible())
                                   ^
                                   ;
>> drivers/char/random.c:1269:41: warning: shift count >= width of type [-Wshift-count-overflow]
           c_high = (sizeof(cycles) > 4) ? cycles >> 32 : 0;
                                                  ^  ~~
   drivers/char/random.c:1270:35: warning: shift count >= width of type [-Wshift-count-overflow]
           j_high = (sizeof(now) > 4) ? now >> 32 : 0;
                                            ^  ~~
   drivers/char/random.c:2296:6: warning: no previous prototype for function 'add_hwgenerator_randomness' [-Wmissing-prototypes]
   void add_hwgenerator_randomness(const char *buffer, size_t count,
        ^

drivers/char/random.c:2296:1: note: declare 'static' if the function is not intended to be used outside of this translation unit void add_hwgenerator_randomness(const char *buffer, size_t count,
^
static
3 warnings and 1 error generated.
--
In file included from drivers/usb/host/ehci-platform.c:37:
In file included from include/linux/usb/of.h:12:
In file included from include/linux/usb/otg.h:13:
In file included from include/linux/phy/phy.h:17:
In file included from include/linux/regulator/consumer.h:35:
In file included from include/linux/suspend.h:5:
In file included from include/linux/swap.h:9:
In file included from include/linux/memcontrol.h:22:
In file included from include/linux/writeback.h:14:
In file included from include/linux/blk-cgroup.h:23:
In file included from include/linux/blkdev.h:14:
include/linux/pagemap.h:207:26: error: expected ';' after do/while statement

           VM_BUG_ON(preemptible())
                                   ^
                                   ;
>> drivers/usb/host/ehci-platform.c:259:24: warning: shift count >= width of type [-Wshift-count-overflow]
                   pdata->dma_mask_64 ? DMA_BIT_MASK(64) : DMA_BIT_MASK(32));
                                        ^~~~~~~~~~~~~~~~
   include/linux/dma-mapping.h:76:54: note: expanded from macro 'DMA_BIT_MASK'
   #define DMA_BIT_MASK(n) (((n) == 64) ? ~0ULL : ((1ULL<<(n))-1))
                                                        ^ ~~~

1 warning and 1 error generated.
--
In file included from drivers/usb/host/xhci-plat.c:21:
In file included from include/linux/usb/of.h:12:
In file included from include/linux/usb/otg.h:13:
In file included from include/linux/phy/phy.h:17:
In file included from include/linux/regulator/consumer.h:35:
In file included from include/linux/suspend.h:5:
In file included from include/linux/swap.h:9:
In file included from include/linux/memcontrol.h:22:
In file included from include/linux/writeback.h:14:
In file included from include/linux/blk-cgroup.h:23:
In file included from include/linux/blkdev.h:14:
include/linux/pagemap.h:207:26: error: expected ';' after do/while statement

           VM_BUG_ON(preemptible())
                                   ^
                                   ;
>> drivers/usb/host/xhci-plat.c:222:10: warning: shift count >= width of type [-Wshift-count-overflow]
                                                      DMA_BIT_MASK(64));
                                                      ^~~~~~~~~~~~~~~~
   include/linux/dma-mapping.h:76:54: note: expanded from macro 'DMA_BIT_MASK'
   #define DMA_BIT_MASK(n) (((n) == 64) ? ~0ULL : ((1ULL<<(n))-1))
                                                        ^ ~~~
   drivers/usb/host/xhci-plat.c:224:43: warning: shift count >= width of type [-Wshift-count-overflow]
                   ret = dma_set_mask_and_coherent(sysdev, DMA_BIT_MASK(64));
                                                           ^~~~~~~~~~~~~~~~
   include/linux/dma-mapping.h:76:54: note: expanded from macro 'DMA_BIT_MASK'
   #define DMA_BIT_MASK(n) (((n) == 64) ? ~0ULL : ((1ULL<<(n))-1))
                                                        ^ ~~~

2 warnings and 1 error generated.
--
In file included from drivers/mmc/host/sdhci.c:24:
In file included from include/linux/regulator/consumer.h:35:
In file included from include/linux/suspend.h:5:
In file included from include/linux/swap.h:9:
In file included from include/linux/memcontrol.h:22:
In file included from include/linux/writeback.h:14:
In file included from include/linux/blk-cgroup.h:23:
In file included from include/linux/blkdev.h:14:
include/linux/pagemap.h:207:26: error: expected ';' after do/while statement

           VM_BUG_ON(preemptible())
                                   ^
                                   ;
>> drivers/mmc/host/sdhci.c:3953:40: warning: shift count >= width of type [-Wshift-count-overflow]
                   ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
                                                        ^~~~~~~~~~~~~~~~
   include/linux/dma-mapping.h:76:54: note: expanded from macro 'DMA_BIT_MASK'
   #define DMA_BIT_MASK(n) (((n) == 64) ? ~0ULL : ((1ULL<<(n))-1))
                                                        ^ ~~~
   drivers/mmc/host/sdhci.c:4254:20: warning: shift count >= width of type [-Wshift-count-overflow]
                   host->dma_mask = DMA_BIT_MASK(64);
                                    ^~~~~~~~~~~~~~~~
   include/linux/dma-mapping.h:76:54: note: expanded from macro 'DMA_BIT_MASK'
   #define DMA_BIT_MASK(n) (((n) == 64) ? ~0ULL : ((1ULL<<(n))-1))
                                                        ^ ~~~

2 warnings and 1 error generated.

vim +1269 drivers/char/random.c

ee3e00e9e7101c8 Theodore Ts'o     2014-06-15  1254  
775f4b297b78060 Theodore Ts'o     2012-07-02  1255  void add_interrupt_randomness(int irq, int irq_flags)
^1da177e4c3f415 Linus Torvalds    2005-04-16  1256  {
775f4b297b78060 Theodore Ts'o     2012-07-02  1257  	struct entropy_store	*r;
1b2a1a7e8ad1144 Christoph Lameter 2014-08-17  1258  	struct fast_pool	*fast_pool = this_cpu_ptr(&irq_randomness);
775f4b297b78060 Theodore Ts'o     2012-07-02  1259  	struct pt_regs		*regs = get_irq_regs();
775f4b297b78060 Theodore Ts'o     2012-07-02  1260  	unsigned long		now = jiffies;
655b226470b2295 Theodore Ts'o     2013-09-22  1261  	cycles_t		cycles = random_get_entropy();
43759d4f429c8d5 Theodore Ts'o     2014-06-14  1262  	__u32			c_high, j_high;
655b226470b2295 Theodore Ts'o     2013-09-22  1263  	__u64			ip;
83664a6928a420b H. Peter Anvin    2014-03-17  1264  	unsigned long		seed;
91fcb532efe366d Theodore Ts'o     2014-06-10  1265  	int			credit = 0;
655b226470b2295 Theodore Ts'o     2013-09-22  1266  
ee3e00e9e7101c8 Theodore Ts'o     2014-06-15  1267  	if (cycles == 0)
ee3e00e9e7101c8 Theodore Ts'o     2014-06-15  1268  		cycles = get_reg(fast_pool, regs);
655b226470b2295 Theodore Ts'o     2013-09-22 @1269  	c_high = (sizeof(cycles) > 4) ? cycles >> 32 : 0;
655b226470b2295 Theodore Ts'o     2013-09-22  1270  	j_high = (sizeof(now) > 4) ? now >> 32 : 0;
43759d4f429c8d5 Theodore Ts'o     2014-06-14  1271  	fast_pool->pool[0] ^= cycles ^ j_high ^ irq;
43759d4f429c8d5 Theodore Ts'o     2014-06-14  1272  	fast_pool->pool[1] ^= now ^ c_high;
655b226470b2295 Theodore Ts'o     2013-09-22  1273  	ip = regs ? instruction_pointer(regs) : _RET_IP_;
43759d4f429c8d5 Theodore Ts'o     2014-06-14  1274  	fast_pool->pool[2] ^= ip;
ee3e00e9e7101c8 Theodore Ts'o     2014-06-15  1275  	fast_pool->pool[3] ^= (sizeof(ip) > 4) ? ip >> 32 :
ee3e00e9e7101c8 Theodore Ts'o     2014-06-15  1276  		get_reg(fast_pool, regs);
3060d6fe2857064 Yinghai Lu        2008-08-19  1277  
43759d4f429c8d5 Theodore Ts'o     2014-06-14  1278  	fast_mix(fast_pool);
43759d4f429c8d5 Theodore Ts'o     2014-06-14  1279  	add_interrupt_bench(cycles);
3060d6fe2857064 Yinghai Lu        2008-08-19  1280  
43838a23a05fbd1 Theodore Ts'o     2018-04-11  1281  	if (unlikely(crng_init == 0)) {
e192be9d9a30555 Theodore Ts'o     2016-06-12  1282  		if ((fast_pool->count >= 64) &&
e192be9d9a30555 Theodore Ts'o     2016-06-12  1283  		    crng_fast_load((char *) fast_pool->pool,
e192be9d9a30555 Theodore Ts'o     2016-06-12  1284  				   sizeof(fast_pool->pool))) {
e192be9d9a30555 Theodore Ts'o     2016-06-12  1285  			fast_pool->count = 0;
e192be9d9a30555 Theodore Ts'o     2016-06-12  1286  			fast_pool->last = now;
e192be9d9a30555 Theodore Ts'o     2016-06-12  1287  		}
e192be9d9a30555 Theodore Ts'o     2016-06-12  1288  		return;
e192be9d9a30555 Theodore Ts'o     2016-06-12  1289  	}
e192be9d9a30555 Theodore Ts'o     2016-06-12  1290  
840f95077ffd640 Theodore Ts'o     2014-06-14  1291  	if ((fast_pool->count < 64) &&
840f95077ffd640 Theodore Ts'o     2014-06-14  1292  	    !time_after(now, fast_pool->last + HZ))
^1da177e4c3f415 Linus Torvalds    2005-04-16  1293  		return;
^1da177e4c3f415 Linus Torvalds    2005-04-16  1294  
e192be9d9a30555 Theodore Ts'o     2016-06-12  1295  	r = &input_pool;
840f95077ffd640 Theodore Ts'o     2014-06-14  1296  	if (!spin_trylock(&r->lock))
91fcb532efe366d Theodore Ts'o     2014-06-10  1297  		return;
83664a6928a420b H. Peter Anvin    2014-03-17  1298  
91fcb532efe366d Theodore Ts'o     2014-06-10  1299  	fast_pool->last = now;
85608f8e16c28f8 Theodore Ts'o     2014-06-10  1300  	__mix_pool_bytes(r, &fast_pool->pool, sizeof(fast_pool->pool));
83664a6928a420b H. Peter Anvin    2014-03-17  1301  
83664a6928a420b H. Peter Anvin    2014-03-17  1302  	/*
83664a6928a420b H. Peter Anvin    2014-03-17  1303  	 * If we have architectural seed generator, produce a seed and
48d6be955a7167b Theodore Ts'o     2014-07-17  1304  	 * add it to the pool.  For the sake of paranoia don't let the
48d6be955a7167b Theodore Ts'o     2014-07-17  1305  	 * architectural seed generator dominate the input from the
48d6be955a7167b Theodore Ts'o     2014-07-17  1306  	 * interrupt noise.
83664a6928a420b H. Peter Anvin    2014-03-17  1307  	 */
83664a6928a420b H. Peter Anvin    2014-03-17  1308  	if (arch_get_random_seed_long(&seed)) {
85608f8e16c28f8 Theodore Ts'o     2014-06-10  1309  		__mix_pool_bytes(r, &seed, sizeof(seed));
48d6be955a7167b Theodore Ts'o     2014-07-17  1310  		credit = 1;
83664a6928a420b H. Peter Anvin    2014-03-17  1311  	}
91fcb532efe366d Theodore Ts'o     2014-06-10  1312  	spin_unlock(&r->lock);
83664a6928a420b H. Peter Anvin    2014-03-17  1313  
ee3e00e9e7101c8 Theodore Ts'o     2014-06-15  1314  	fast_pool->count = 0;
83664a6928a420b H. Peter Anvin    2014-03-17  1315  
ee3e00e9e7101c8 Theodore Ts'o     2014-06-15  1316  	/* award one bit for the contents of the fast pool */
ee3e00e9e7101c8 Theodore Ts'o     2014-06-15  1317  	credit_entropy_bits(r, credit + 1);
^1da177e4c3f415 Linus Torvalds    2005-04-16  1318  }
4b44f2d18a33056 Stephan Mueller   2016-05-02  1319  EXPORT_SYMBOL_GPL(add_interrupt_randomness);
^1da177e4c3f415 Linus Torvalds    2005-04-16  1320  

---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/kbuild-all(a)lists.01.org

maintainer
#7 lore raw

Re: [PATCH 06/16] mm/pagemap: Cleanup PREEMPT_COUNT leftovers

In reply to Uladzislau Rezki
Quoted 34 lines · Uladzislau Rezki
Quoted 31 lines · Uladzislau Rezki (Sony)

From: Thomas Gleixner <[email protected]>

CONFIG_PREEMPT_COUNT is now unconditionally enabled and will be removed. Cleanup the leftovers before doing so.

---
 include/linux/pagemap.h | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)
include/linux/pagemap.h +1 -3
index c77b7c31b2e4..cbfbe2bcca75 100644
@@ -204,9 +204,7 @@ void release_pages(struct page **pages, int nr);
 static inline int __page_cache_add_speculative(struct page *page, int count)
 {
 #ifdef CONFIG_TINY_RCU
-# ifdef CONFIG_PREEMPT_COUNT
-	VM_BUG_ON(!in_atomic() && !irqs_disabled());
-# endif
+	VM_BUG_ON(preemptible())
 	/*
 	 * Preempt must be disabled here - we rely on rcu_read_lock doing
 	 * this for us.
Signature · 1 line
2.20.1

Hello, Paul.

Quoted 3 lines

Sorry for a small mistake, it was fixed by you before, whereas i took an old version of the patch that is question. Please use below one instead of posted one:

We have all been there and done that! ;-)

I will give this update a spin and see what happens.

Thanx, Paul

Quoted 32 lines
Author: Thomas Gleixner <[email protected]>
Date:   Mon Sep 14 19:25:00 2020 +0200

mm/pagemap: Cleanup PREEMPT_COUNT leftovers

CONFIG_PREEMPT_COUNT is now unconditionally enabled and will be removed. Cleanup the leftovers before doing so.

Cc Andrew Morton <[email protected]>
[ paulmck: Fix !SMP build error per kernel test robot feedback. ]
include/linux/pagemap.h +1 -4
index 7de11dcd534d..b3d9d9217ea0 100644
@@ -168,9 +168,7 @@ void release_pages(struct page **pages, int nr);
 static inline int __page_cache_add_speculative(struct page *page, int count)
 {
 #ifdef CONFIG_TINY_RCU
-# ifdef CONFIG_PREEMPT_COUNT
-       VM_BUG_ON(!in_atomic() && !irqs_disabled());
-# endif
+       VM_BUG_ON(preemptible());
        /*
         * Preempt must be disabled here - we rely on rcu_read_lock doing
         * this for us.
 
--

Vlad Rezki

#10 lore raw

Re: [PATCH 06/16] mm/pagemap: Cleanup PREEMPT_COUNT leftovers

In reply to Uladzislau Rezki (Sony)
I love your patch! Yet something to improve:

Hi "Uladzislau,

I love your patch! Yet something to improve:

[auto build test ERROR on arm/for-next]
[also build test ERROR on tip/sched/core linus/master v5.10-rc1] [cannot apply to rcu/dev drm-intel/for-linux-next next-20201029] [If your patch is applied to the wrong git tree, kindly drop us a note. And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch]

url:    https://github.com/0day-ci/linux/commits/Uladzislau-Rezki-Sony/rcu-tree-Add-a-work-to-allocate-pages-from-regular-context/20201030-005247
base:   git://git.armlinux.org.uk/~rmk/linux-arm.git for-next
config: xtensa-allyesconfig (attached as .config)
compiler: xtensa-linux-gcc (GCC) 9.3.0

reproduce (this is a W=1 build):
wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
chmod +x ~/bin/make.cross
# https://github.com/0day-ci/linux/commit/1e0b66ce2b779463194f8b398b8e48caa95aabf4
git remote add linux-review https://github.com/0day-ci/linux
git fetch --no-tags linux-review Uladzislau-Rezki-Sony/rcu-tree-Add-a-work-to-allocate-pages-from-regular-context/20201030-005247 git checkout 1e0b66ce2b779463194f8b398b8e48caa95aabf4
# save the attached .config to linux build tree
COMPILER_INSTALL_PATH=$HOME/0day COMPILER=gcc-9.3.0 make.cross ARCH=xtensa

If you fix the issue, kindly add following tag as appropriate

All errors (new ones prefixed by >>):

In file included from include/linux/gfp.h:5,

                    from include/linux/umh.h:4,
                    from include/linux/kmod.h:9,
                    from include/linux/module.h:16,
                    from drivers/staging/wlan-ng/p80211req.c:54:
   include/linux/pagemap.h: In function '__page_cache_add_speculative':
>> include/linux/mmdebug.h:20:2: error: expected ';' before 'do'
      20 |  do {        \
         |  ^~
   include/linux/pagemap.h:217:2: note: in expansion of macro 'VM_BUG_ON_PAGE'
     217 |  VM_BUG_ON_PAGE(page_count(page) == 0, page);
         |  ^~~~~~~~~~~~~~
--
   In file included from include/linux/percpu.h:5,
                    from include/linux/radix-tree.h:14,
                    from include/linux/fs.h:15,
                    from include/linux/dax.h:5,
                    from mm/filemap.c:15:
   include/linux/pagemap.h: In function '__page_cache_add_speculative':
>> include/linux/mmdebug.h:20:2: error: expected ';' before 'do'
      20 |  do {        \
         |  ^~
   include/linux/pagemap.h:217:2: note: in expansion of macro 'VM_BUG_ON_PAGE'
     217 |  VM_BUG_ON_PAGE(page_count(page) == 0, page);
         |  ^~~~~~~~~~~~~~

mm/filemap.c: At top level:
mm/filemap.c:830:14: warning: no previous prototype for '__add_to_page_cache_locked' [-Wmissing-prototypes]
830 | noinline int __add_to_page_cache_locked(struct page *page,
| ^~~~~~~~~~~~~~~~~~~~~~~~~~
--
In file included from include/linux/percpu.h:5,

                    from include/linux/radix-tree.h:14,
                    from include/linux/fs.h:15,
                    from include/linux/backing-dev.h:13,
                    from mm/truncate.c:12:
   include/linux/pagemap.h: In function '__page_cache_add_speculative':
>> include/linux/mmdebug.h:20:2: error: expected ';' before 'do'
      20 |  do {        \
         |  ^~
   include/linux/pagemap.h:217:2: note: in expansion of macro 'VM_BUG_ON_PAGE'
     217 |  VM_BUG_ON_PAGE(page_count(page) == 0, page);
         |  ^~~~~~~~~~~~~~

mm/truncate.c: At top level:
mm/truncate.c:531:15: warning: no previous prototype for '__invalidate_mapping_pages' [-Wmissing-prototypes]
531 | unsigned long __invalidate_mapping_pages(struct address_space *mapping,
| ^~~~~~~~~~~~~~~~~~~~~~~~~~
--
In file included from include/linux/percpu.h:5,
from include/linux/hrtimer.h:19,
from include/linux/sched.h:20,
from include/linux/cgroup.h:12,
from include/linux/memcontrol.h:13,
from mm/workingset.c:8:
include/linux/pagemap.h: In function '__page_cache_add_speculative':

>> include/linux/mmdebug.h:20:2: error: expected ';' before 'do'
      20 |  do {        \
         |  ^~
   include/linux/pagemap.h:217:2: note: in expansion of macro 'VM_BUG_ON_PAGE'
     217 |  VM_BUG_ON_PAGE(page_count(page) == 0, page);
         |  ^~~~~~~~~~~~~~
   mm/workingset.c: In function 'unpack_shadow':
   mm/workingset.c:201:15: warning: variable 'nid' set but not used [-Wunused-but-set-variable]
     201 |  int memcgid, nid;
         |               ^~~
--
   In file included from include/linux/mm.h:9,
                    from mm/vmalloc.c:12:
   include/linux/pagemap.h: In function '__page_cache_add_speculative':
>> include/linux/mmdebug.h:20:2: error: expected ';' before 'do'
      20 |  do {        \
         |  ^~
   include/linux/pagemap.h:217:2: note: in expansion of macro 'VM_BUG_ON_PAGE'
     217 |  VM_BUG_ON_PAGE(page_count(page) == 0, page);
         |  ^~~~~~~~~~~~~~

mm/vmalloc.c: At top level:
mm/vmalloc.c:1318:6: warning: no previous prototype for 'set_iounmap_nonlazy' [-Wmissing-prototypes]
1318 | void set_iounmap_nonlazy(void)
| ^~~~~~~~~~~~~~~~~~~
--
In file included from include/linux/mm.h:9,

                    from mm/page_alloc.c:19:
   include/linux/pagemap.h: In function '__page_cache_add_speculative':
>> include/linux/mmdebug.h:20:2: error: expected ';' before 'do'
      20 |  do {        \
         |  ^~
   include/linux/pagemap.h:217:2: note: in expansion of macro 'VM_BUG_ON_PAGE'
     217 |  VM_BUG_ON_PAGE(page_count(page) == 0, page);
         |  ^~~~~~~~~~~~~~

mm/page_alloc.c: At top level:
mm/page_alloc.c:3558:15: warning: no previous prototype for 'should_fail_alloc_page' [-Wmissing-prototypes]
3558 | noinline bool should_fail_alloc_page(gfp_t gfp_mask, unsigned int order)
| ^~~~~~~~~~~~~~~~~~~~~~
mm/page_alloc.c:6183:23: warning: no previous prototype for 'memmap_init' [-Wmissing-prototypes]
6183 | void __meminit __weak memmap_init(unsigned long size, int nid,
| ^~~~~~~~~~~
--
In file included from include/linux/percpu.h:5,
from include/linux/hrtimer.h:19,
from include/linux/sched.h:20,
from include/linux/cgroup.h:12,
from include/linux/memcontrol.h:13,
from mm/memcontrol.c:26:
include/linux/pagemap.h: In function '__page_cache_add_speculative':

>> include/linux/mmdebug.h:20:2: error: expected ';' before 'do'
      20 |  do {        \
         |  ^~
   include/linux/pagemap.h:217:2: note: in expansion of macro 'VM_BUG_ON_PAGE'
     217 |  VM_BUG_ON_PAGE(page_count(page) == 0, page);
         |  ^~~~~~~~~~~~~~
   In file included from arch/xtensa/include/asm/atomic.h:19,
                    from include/linux/atomic.h:7,
                    from include/linux/page_counter.h:5,
                    from mm/memcontrol.c:25:
   mm/memcontrol.c: In function 'memcg_reparent_objcgs':
   arch/xtensa/include/asm/cmpxchg.h:173:3: warning: value computed is not used [-Wunused-value]
     173 |  ((__typeof__(*(ptr)))__xchg((unsigned long)(x),(ptr),sizeof(*(ptr))))
         |  ~^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
   mm/memcontrol.c:328:2: note: in expansion of macro 'xchg'
     328 |  xchg(&objcg->memcg, parent);
         |  ^~~~
   arch/xtensa/include/asm/cmpxchg.h:173:3: warning: value computed is not used [-Wunused-value]
     173 |  ((__typeof__(*(ptr)))__xchg((unsigned long)(x),(ptr),sizeof(*(ptr))))
         |  ~^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
   mm/memcontrol.c:335:3: note: in expansion of macro 'xchg'
     335 |   xchg(&iter->memcg, parent);
         |   ^~~~
--
   In file included from include/linux/percpu.h:5,
                    from include/linux/hrtimer.h:19,
                    from include/linux/sched.h:20,
                    from include/linux/ratelimit.h:6,
                    from include/linux/dev_printk.h:16,
                    from include/linux/device.h:15,
                    from include/sound/core.h:10,
                    from drivers/gpu/drm/bridge/adv7511/adv7511_audio.c:9:
   include/linux/pagemap.h: In function '__page_cache_add_speculative':
>> include/linux/mmdebug.h:20:2: error: expected ';' before 'do'
      20 |  do {        \
         |  ^~
   include/linux/pagemap.h:217:2: note: in expansion of macro 'VM_BUG_ON_PAGE'
     217 |  VM_BUG_ON_PAGE(page_count(page) == 0, page);
         |  ^~~~~~~~~~~~~~

drivers/gpu/drm/bridge/adv7511/adv7511_audio.c: At top level:
drivers/gpu/drm/bridge/adv7511/adv7511_audio.c:58:5: warning: no previous prototype for 'adv7511_hdmi_hw_params' [-Wmissing-prototypes]
58 | int adv7511_hdmi_hw_params(struct device *dev, void *data,
| ^~~~~~~~~~~~~~~~~~~~~~
--
In file included from include/linux/percpu.h:5,
from include/linux/bpf.h:11,
from security/security.c:13:
include/linux/pagemap.h: In function '__page_cache_add_speculative':

>> include/linux/mmdebug.h:20:2: error: expected ';' before 'do'
      20 |  do {        \
         |  ^~
   include/linux/pagemap.h:217:2: note: in expansion of macro 'VM_BUG_ON_PAGE'
     217 |  VM_BUG_ON_PAGE(page_count(page) == 0, page);
         |  ^~~~~~~~~~~~~~
   At top level:
   include/linux/lsm_hook_defs.h:394:18: warning: 'perf_event_write_default' defined but not used [-Wunused-const-variable=]
     394 | LSM_HOOK(int, 0, perf_event_write, struct perf_event *event)
         |                  ^~~~~~~~~~~~~~~~
   security/security.c:681:32: note: in definition of macro 'LSM_RET_DEFAULT'
     681 | #define LSM_RET_DEFAULT(NAME) (NAME##_default)
         |                                ^~~~
   security/security.c:686:2: note: in expansion of macro 'DECLARE_LSM_RET_DEFAULT_int'
     686 |  DECLARE_LSM_RET_DEFAULT_##RET(DEFAULT, NAME)
         |  ^~~~~~~~~~~~~~~~~~~~~~~~
   include/linux/lsm_hook_defs.h:394:1: note: in expansion of macro 'LSM_HOOK'
     394 | LSM_HOOK(int, 0, perf_event_write, struct perf_event *event)
         | ^~~~~~~~
   include/linux/lsm_hook_defs.h:393:18: warning: 'perf_event_read_default' defined but not used [-Wunused-const-variable=]
     393 | LSM_HOOK(int, 0, perf_event_read, struct perf_event *event)
         |                  ^~~~~~~~~~~~~~~
   security/security.c:681:32: note: in definition of macro 'LSM_RET_DEFAULT'
     681 | #define LSM_RET_DEFAULT(NAME) (NAME##_default)
         |                                ^~~~
   security/security.c:686:2: note: in expansion of macro 'DECLARE_LSM_RET_DEFAULT_int'
     686 |  DECLARE_LSM_RET_DEFAULT_##RET(DEFAULT, NAME)
         |  ^~~~~~~~~~~~~~~~~~~~~~~~
   include/linux/lsm_hook_defs.h:393:1: note: in expansion of macro 'LSM_HOOK'
     393 | LSM_HOOK(int, 0, perf_event_read, struct perf_event *event)
         | ^~~~~~~~
   include/linux/lsm_hook_defs.h:391:18: warning: 'perf_event_alloc_default' defined but not used [-Wunused-const-variable=]
     391 | LSM_HOOK(int, 0, perf_event_alloc, struct perf_event *event)
         |                  ^~~~~~~~~~~~~~~~
   security/security.c:681:32: note: in definition of macro 'LSM_RET_DEFAULT'
     681 | #define LSM_RET_DEFAULT(NAME) (NAME##_default)
         |                                ^~~~
   security/security.c:686:2: note: in expansion of macro 'DECLARE_LSM_RET_DEFAULT_int'
     686 |  DECLARE_LSM_RET_DEFAULT_##RET(DEFAULT, NAME)
         |  ^~~~~~~~~~~~~~~~~~~~~~~~
   include/linux/lsm_hook_defs.h:391:1: note: in expansion of macro 'LSM_HOOK'
     391 | LSM_HOOK(int, 0, perf_event_alloc, struct perf_event *event)
         | ^~~~~~~~
   include/linux/lsm_hook_defs.h:390:18: warning: 'perf_event_open_default' defined but not used [-Wunused-const-variable=]
     390 | LSM_HOOK(int, 0, perf_event_open, struct perf_event_attr *attr, int type)
         |                  ^~~~~~~~~~~~~~~
   security/security.c:681:32: note: in definition of macro 'LSM_RET_DEFAULT'
     681 | #define LSM_RET_DEFAULT(NAME) (NAME##_default)
         |                                ^~~~
   security/security.c:686:2: note: in expansion of macro 'DECLARE_LSM_RET_DEFAULT_int'
     686 |  DECLARE_LSM_RET_DEFAULT_##RET(DEFAULT, NAME)
         |  ^~~~~~~~~~~~~~~~~~~~~~~~
   include/linux/lsm_hook_defs.h:390:1: note: in expansion of macro 'LSM_HOOK'
     390 | LSM_HOOK(int, 0, perf_event_open, struct perf_event_attr *attr, int type)
         | ^~~~~~~~
   include/linux/lsm_hook_defs.h:387:18: warning: 'locked_down_default' defined but not used [-Wunused-const-variable=]
     387 | LSM_HOOK(int, 0, locked_down, enum lockdown_reason what)
         |                  ^~~~~~~~~~~
   security/security.c:681:32: note: in definition of macro 'LSM_RET_DEFAULT'
     681 | #define LSM_RET_DEFAULT(NAME) (NAME##_default)
         |                                ^~~~
   security/security.c:686:2: note: in expansion of macro 'DECLARE_LSM_RET_DEFAULT_int'
     686 |  DECLARE_LSM_RET_DEFAULT_##RET(DEFAULT, NAME)
         |  ^~~~~~~~~~~~~~~~~~~~~~~~
   include/linux/lsm_hook_defs.h:387:1: note: in expansion of macro 'LSM_HOOK'
     387 | LSM_HOOK(int, 0, locked_down, enum lockdown_reason what)
         | ^~~~~~~~
   include/linux/lsm_hook_defs.h:383:18: warning: 'bpf_prog_alloc_security_default' defined but not used [-Wunused-const-variable=]
     383 | LSM_HOOK(int, 0, bpf_prog_alloc_security, struct bpf_prog_aux *aux)
         |                  ^~~~~~~~~~~~~~~~~~~~~~~
   security/security.c:681:32: note: in definition of macro 'LSM_RET_DEFAULT'
     681 | #define LSM_RET_DEFAULT(NAME) (NAME##_default)
         |                                ^~~~
   security/security.c:686:2: note: in expansion of macro 'DECLARE_LSM_RET_DEFAULT_int'
     686 |  DECLARE_LSM_RET_DEFAULT_##RET(DEFAULT, NAME)
         |  ^~~~~~~~~~~~~~~~~~~~~~~~
   include/linux/lsm_hook_defs.h:383:1: note: in expansion of macro 'LSM_HOOK'
     383 | LSM_HOOK(int, 0, bpf_prog_alloc_security, struct bpf_prog_aux *aux)
         | ^~~~~~~~
   include/linux/lsm_hook_defs.h:381:18: warning: 'bpf_map_alloc_security_default' defined but not used [-Wunused-const-variable=]
     381 | LSM_HOOK(int, 0, bpf_map_alloc_security, struct bpf_map *map)
         |                  ^~~~~~~~~~~~~~~~~~~~~~
   security/security.c:681:32: note: in definition of macro 'LSM_RET_DEFAULT'
     681 | #define LSM_RET_DEFAULT(NAME) (NAME##_default)
         |                                ^~~~
   security/security.c:686:2: note: in expansion of macro 'DECLARE_LSM_RET_DEFAULT_int'
     686 |  DECLARE_LSM_RET_DEFAULT_##RET(DEFAULT, NAME)
         |  ^~~~~~~~~~~~~~~~~~~~~~~~
   include/linux/lsm_hook_defs.h:381:1: note: in expansion of macro 'LSM_HOOK'
     381 | LSM_HOOK(int, 0, bpf_map_alloc_security, struct bpf_map *map)
         | ^~~~~~~~
   include/linux/lsm_hook_defs.h:380:18: warning: 'bpf_prog_default' defined but not used [-Wunused-const-variable=]
     380 | LSM_HOOK(int, 0, bpf_prog, struct bpf_prog *prog)
         |                  ^~~~~~~~
   security/security.c:681:32: note: in definition of macro 'LSM_RET_DEFAULT'
     681 | #define LSM_RET_DEFAULT(NAME) (NAME##_default)
         |                                ^~~~
   security/security.c:686:2: note: in expansion of macro 'DECLARE_LSM_RET_DEFAULT_int'
     686 |  DECLARE_LSM_RET_DEFAULT_##RET(DEFAULT, NAME)
         |  ^~~~~~~~~~~~~~~~~~~~~~~~
   include/linux/lsm_hook_defs.h:380:1: note: in expansion of macro 'LSM_HOOK'
--
   In file included from include/linux/gfp.h:5,
                    from include/linux/umh.h:4,
                    from include/linux/kmod.h:9,
                    from include/linux/module.h:16,
                    from drivers/staging/vt6655/device.h:19,
                    from drivers/staging/vt6655/device_main.c:37:
   include/linux/pagemap.h: In function '__page_cache_add_speculative':
>> include/linux/mmdebug.h:20:2: error: expected ';' before 'do'
      20 |  do {        \
         |  ^~
   include/linux/pagemap.h:217:2: note: in expansion of macro 'VM_BUG_ON_PAGE'
     217 |  VM_BUG_ON_PAGE(page_count(page) == 0, page);
         |  ^~~~~~~~~~~~~~
   In file included from drivers/staging/vt6655/device_main.c:43:
   drivers/staging/vt6655/rxtx.h: At top level:
   drivers/staging/vt6655/rxtx.h:114:1: warning: alignment 1 of 'struct vnt_rts_g' is less than 2 [-Wpacked-not-aligned]
     114 | } __packed;
         | ^
   drivers/staging/vt6655/rxtx.h:128:1: warning: alignment 1 of 'struct vnt_rts_g_fb' is less than 2 [-Wpacked-not-aligned]
     128 | } __packed;
         | ^
   drivers/staging/vt6655/rxtx.h:135:1: warning: alignment 1 of 'struct vnt_rts_ab' is less than 2 [-Wpacked-not-aligned]
     135 | } __packed;
         | ^
   drivers/staging/vt6655/rxtx.h:144:1: warning: alignment 1 of 'struct vnt_rts_a_fb' is less than 2 [-Wpacked-not-aligned]
     144 | } __packed;
         | ^
   drivers/staging/vt6655/rxtx.h:153:1: warning: alignment 1 of 'struct vnt_cts' is less than 2 [-Wpacked-not-aligned]
     153 | } __packed;
         | ^
   drivers/staging/vt6655/rxtx.h:163:1: warning: alignment 1 of 'struct vnt_cts_fb' is less than 2 [-Wpacked-not-aligned]
     163 | } __packed;
         | ^
--
   In file included from include/linux/gfp.h:5,
                    from include/linux/umh.h:4,
                    from include/linux/kmod.h:9,
                    from include/linux/module.h:16,
                    from drivers/staging/vt6655/device.h:19,
                    from drivers/staging/vt6655/upc.h:19,
                    from drivers/staging/vt6655/srom.c:32:
   include/linux/pagemap.h: In function '__page_cache_add_speculative':
>> include/linux/mmdebug.h:20:2: error: expected ';' before 'do'
      20 |  do {        \
         |  ^~
   include/linux/pagemap.h:217:2: note: in expansion of macro 'VM_BUG_ON_PAGE'
     217 |  VM_BUG_ON_PAGE(page_count(page) == 0, page);
         |  ^~~~~~~~~~~~~~
   In file included from drivers/staging/vt6655/srom.c:32:
   drivers/staging/vt6655/srom.c: In function 'SROMbyReadEmbedded':
   drivers/staging/vt6655/upc.h:45:16: warning: variable 'byData' set but not used [-Wunused-but-set-variable]
      45 |  unsigned char byData;     \
         |                ^~~~~~
   drivers/staging/vt6655/srom.c:85:4: note: in expansion of macro 'PCAvDelayByIO'
      85 |    PCAvDelayByIO(CB_DELAY_LOOP_WAIT);
         |    ^~~~~~~~~~~~~

..

vim +20 include/linux/mmdebug.h

309381feaee5642 Sasha Levin           2014-01-23  16  
59ea746337c69f6 Jiri Slaby            2008-06-12  17  #ifdef CONFIG_DEBUG_VM
59ea746337c69f6 Jiri Slaby            2008-06-12  18  #define VM_BUG_ON(cond) BUG_ON(cond)
309381feaee5642 Sasha Levin           2014-01-23  19  #define VM_BUG_ON_PAGE(cond, page)					\
e4f674229ce63da Dave Hansen           2014-06-04 @20  	do {								\
e4f674229ce63da Dave Hansen           2014-06-04  21  		if (unlikely(cond)) {					\
e4f674229ce63da Dave Hansen           2014-06-04  22  			dump_page(page, "VM_BUG_ON_PAGE(" __stringify(cond)")");\
e4f674229ce63da Dave Hansen           2014-06-04  23  			BUG();						\
e4f674229ce63da Dave Hansen           2014-06-04  24  		}							\
e4f674229ce63da Dave Hansen           2014-06-04  25  	} while (0)
fa3759ccd5651c4 Sasha Levin           2014-10-09  26  #define VM_BUG_ON_VMA(cond, vma)					\
fa3759ccd5651c4 Sasha Levin           2014-10-09  27  	do {								\
fa3759ccd5651c4 Sasha Levin           2014-10-09  28  		if (unlikely(cond)) {					\
fa3759ccd5651c4 Sasha Levin           2014-10-09  29  			dump_vma(vma);					\
fa3759ccd5651c4 Sasha Levin           2014-10-09  30  			BUG();						\
fa3759ccd5651c4 Sasha Levin           2014-10-09  31  		}							\
fa3759ccd5651c4 Sasha Levin           2014-10-09  32  	} while (0)
31c9afa6db122a5 Sasha Levin           2014-10-09  33  #define VM_BUG_ON_MM(cond, mm)						\
31c9afa6db122a5 Sasha Levin           2014-10-09  34  	do {								\
31c9afa6db122a5 Sasha Levin           2014-10-09  35  		if (unlikely(cond)) {					\
31c9afa6db122a5 Sasha Levin           2014-10-09  36  			dump_mm(mm);					\
31c9afa6db122a5 Sasha Levin           2014-10-09  37  			BUG();						\
31c9afa6db122a5 Sasha Levin           2014-10-09  38  		}							\
31c9afa6db122a5 Sasha Levin           2014-10-09  39  	} while (0)
91241681c62a5a6 Michal Hocko          2018-04-05  40  #define VM_WARN_ON(cond) (void)WARN_ON(cond)
91241681c62a5a6 Michal Hocko          2018-04-05  41  #define VM_WARN_ON_ONCE(cond) (void)WARN_ON_ONCE(cond)
91241681c62a5a6 Michal Hocko          2018-04-05  42  #define VM_WARN_ONCE(cond, format...) (void)WARN_ONCE(cond, format)
91241681c62a5a6 Michal Hocko          2018-04-05  43  #define VM_WARN(cond, format...) (void)WARN(cond, format)
59ea746337c69f6 Jiri Slaby            2008-06-12  44  #else
02602a18c32d76f Konstantin Khlebnikov 2012-05-29  45  #define VM_BUG_ON(cond) BUILD_BUG_ON_INVALID(cond)
309381feaee5642 Sasha Levin           2014-01-23  46  #define VM_BUG_ON_PAGE(cond, page) VM_BUG_ON(cond)
fa3759ccd5651c4 Sasha Levin           2014-10-09  47  #define VM_BUG_ON_VMA(cond, vma) VM_BUG_ON(cond)
31c9afa6db122a5 Sasha Levin           2014-10-09  48  #define VM_BUG_ON_MM(cond, mm) VM_BUG_ON(cond)
02a8efeda894d35 Andrew Morton         2014-06-04  49  #define VM_WARN_ON(cond) BUILD_BUG_ON_INVALID(cond)
02a8efeda894d35 Andrew Morton         2014-06-04  50  #define VM_WARN_ON_ONCE(cond) BUILD_BUG_ON_INVALID(cond)
ef6b571fb8920d5 Andrew Morton         2014-08-06  51  #define VM_WARN_ONCE(cond, format...) BUILD_BUG_ON_INVALID(cond)
a54f9aebaa9f0ea Aneesh Kumar K.V      2016-07-26  52  #define VM_WARN(cond, format...) BUILD_BUG_ON_INVALID(cond)
59ea746337c69f6 Jiri Slaby            2008-06-12  53  #endif
59ea746337c69f6 Jiri Slaby            2008-06-12  54  

---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/kbuild-all(a)lists.01.org

#29 lore raw
In reply to Uladzislau Rezki (Sony)
Quoted 22 lines · Uladzislau Rezki (Sony)

The current memmory-allocation interface presents to following difficulties that this patch is designed to overcome:

a) If built with CONFIG_PROVE_RAW_LOCK_NESTING, the lockdep will complain about violation("BUG: Invalid wait context") of the nesting rules. It does the raw_spinlock vs. spinlock nesting checks, i.e. it is not legal to acquire a spinlock_t while holding a raw_spinlock_t.

   Internally the kfree_rcu() uses raw_spinlock_t whereas the
   "page allocator" internally deals with spinlock_t to access
   to its zones. The code also can be broken from higher level
   of view:
   <snip>
       raw_spin_lock(&some_lock);
       kfree_rcu(some_pointer, some_field_offset);
   <snip>

b) If built with CONFIG_PREEMPT_RT. Please note, in that case spinlock_t is converted into sleepable variant. Invoking the page allocator from atomic contexts leads to "BUG: scheduling while atomic".

Quoted 8 lines

c) call_rcu() is invoked from raw atomic context and kfree_rcu() and kvfree_rcu() are expected to be called from atomic raw context as well.

Move out a page allocation from contexts which trigger kvfree_rcu() function to the separate worker. When a k[v]free_rcu() per-cpu page cache is empty a fallback mechanism is used and a special job is scheduled to refill the per-cpu cache.

Looks good, still reviewing here. BTW just for my education, I was wondering about Thomas's email:
https://lkml.org/lkml/2020/8/11/939

If slab allocations in pure raw-atomic context on RT is not allowed or recommended, should kfree_rcu() be allowed?
slab can have same issue right? If per-cpu cache is drained, it has to allocate page from buddy allocator and there's no GFP flag to tell it about context where alloc is happening from.

Or are we saying that we want to support kfree on RT from raw atomic atomic context, even though kmalloc is not supported? I hate to bring up this elephant in the room, but since I am a part of the people maintaining this code, I believe I would rather set some rules than supporting unsupported usages. :-\ (Once I know what is supported and what isn't that is). If indeed raw atomic kfree_rcu() is a bogus use case because of -RT, then we ought to put a giant warning than supporting it :-(.

But I don't stand in the way of this patch if all agree on it. thanks,

- Joel

Quoted 192 lines
---
 kernel/rcu/tree.c | 109 ++++++++++++++++++++++++++++------------------
 1 file changed, 66 insertions(+), 43 deletions(-)
kernel/rcu/tree.c +66 -43
index 06895ef85d69..f2da2a1cc716 100644
@@ -177,7 +177,7 @@ module_param(rcu_unlock_delay, int, 0444);
  * per-CPU. Object size is equal to one page. This value
  * can be changed at boot time.
  */
-static int rcu_min_cached_objs = 2;
+static int rcu_min_cached_objs = 5;
 module_param(rcu_min_cached_objs, int, 0444);
 
 /* Retrieve RCU kthreads priority for rcutorture */
@@ -3084,6 +3084,9 @@ struct kfree_rcu_cpu_work {
  *	In order to save some per-cpu space the list is singular.
  *	Even though it is lockless an access has to be protected by the
  *	per-cpu lock.
+ * @page_cache_work: A work to refill the cache when it is empty
+ * @work_in_progress: Indicates that page_cache_work is running
+ * @hrtimer: A hrtimer for scheduling a page_cache_work
  * @nr_bkv_objs: number of allocated objects at @bkvcache.
  *
  * This is a per-CPU structure.  The reason that it is not included in
@@ -3100,6 +3103,11 @@ struct kfree_rcu_cpu {
 	bool monitor_todo;
 	bool initialized;
 	int count;
+
+	struct work_struct page_cache_work;
+	atomic_t work_in_progress;
+	struct hrtimer hrtimer;
+
 	struct llist_head bkvcache;
 	int nr_bkv_objs;
 };
@@ -3217,10 +3225,10 @@ static void kfree_rcu_work(struct work_struct *work)
 			}
 			rcu_lock_release(&rcu_callback_map);
 
-			krcp = krc_this_cpu_lock(&flags);
+			raw_spin_lock_irqsave(&krcp->lock, flags);
 			if (put_cached_bnode(krcp, bkvhead[i]))
 				bkvhead[i] = NULL;
-			krc_this_cpu_unlock(krcp, flags);
+			raw_spin_unlock_irqrestore(&krcp->lock, flags);
 
 			if (bkvhead[i])
 				free_page((unsigned long) bkvhead[i]);
@@ -3347,6 +3355,57 @@ static void kfree_rcu_monitor(struct work_struct *work)
 		raw_spin_unlock_irqrestore(&krcp->lock, flags);
 }
 
+static enum hrtimer_restart
+schedule_page_work_fn(struct hrtimer *t)
+{
+	struct kfree_rcu_cpu *krcp =
+		container_of(t, struct kfree_rcu_cpu, hrtimer);
+
+	queue_work(system_highpri_wq, &krcp->page_cache_work);
+	return HRTIMER_NORESTART;
+}
+
+static void fill_page_cache_func(struct work_struct *work)
+{
+	struct kvfree_rcu_bulk_data *bnode;
+	struct kfree_rcu_cpu *krcp =
+		container_of(work, struct kfree_rcu_cpu,
+			page_cache_work);
+	unsigned long flags;
+	bool pushed;
+	int i;
+
+	for (i = 0; i < rcu_min_cached_objs; i++) {
+		bnode = (struct kvfree_rcu_bulk_data *)
+			__get_free_page(GFP_KERNEL | __GFP_NOWARN);
+
+		if (bnode) {
+			raw_spin_lock_irqsave(&krcp->lock, flags);
+			pushed = put_cached_bnode(krcp, bnode);
+			raw_spin_unlock_irqrestore(&krcp->lock, flags);
+
+			if (!pushed) {
+				free_page((unsigned long) bnode);
+				break;
+			}
+		}
+	}
+
+	atomic_set(&krcp->work_in_progress, 0);
+}
+
+static void
+run_page_cache_worker(struct kfree_rcu_cpu *krcp)
+{
+	if (rcu_scheduler_active == RCU_SCHEDULER_RUNNING &&
+			!atomic_xchg(&krcp->work_in_progress, 1)) {
+		hrtimer_init(&krcp->hrtimer, CLOCK_MONOTONIC,
+			HRTIMER_MODE_REL);
+		krcp->hrtimer.function = schedule_page_work_fn;
+		hrtimer_start(&krcp->hrtimer, 0, HRTIMER_MODE_REL);
+	}
+}
+
 static inline bool
 kvfree_call_rcu_add_ptr_to_bulk(struct kfree_rcu_cpu *krcp, void *ptr)
 {
@@ -3363,32 +3422,8 @@ kvfree_call_rcu_add_ptr_to_bulk(struct kfree_rcu_cpu *krcp, void *ptr)
 	if (!krcp->bkvhead[idx] ||
 			krcp->bkvhead[idx]->nr_records == KVFREE_BULK_MAX_ENTR) {
 		bnode = get_cached_bnode(krcp);
-		if (!bnode) {
-			/*
-			 * To keep this path working on raw non-preemptible
-			 * sections, prevent the optional entry into the
-			 * allocator as it uses sleeping locks. In fact, even
-			 * if the caller of kfree_rcu() is preemptible, this
-			 * path still is not, as krcp->lock is a raw spinlock.
-			 * With additional page pre-allocation in the works,
-			 * hitting this return is going to be much less likely.
-			 */
-			if (IS_ENABLED(CONFIG_PREEMPT_RT))
-				return false;
-
-			/*
-			 * NOTE: For one argument of kvfree_rcu() we can
-			 * drop the lock and get the page in sleepable
-			 * context. That would allow to maintain an array
-			 * for the CONFIG_PREEMPT_RT as well if no cached
-			 * pages are available.
-			 */
-			bnode = (struct kvfree_rcu_bulk_data *)
-				__get_free_page(GFP_NOWAIT | __GFP_NOWARN);
-		}
-
 		/* Switch to emergency path. */
-		if (unlikely(!bnode))
+		if (!bnode)
 			return false;
 
 		/* Initialize the new block. */
@@ -3452,12 +3487,10 @@ void kvfree_call_rcu(struct rcu_head *head, rcu_callback_t func)
 		goto unlock_return;
 	}
 
-	/*
-	 * Under high memory pressure GFP_NOWAIT can fail,
-	 * in that case the emergency path is maintained.
-	 */
 	success = kvfree_call_rcu_add_ptr_to_bulk(krcp, ptr);
 	if (!success) {
+		run_page_cache_worker(krcp);
+
 		if (head == NULL)
 			// Inline if kvfree_rcu(one_arg) call.
 			goto unlock_return;
@@ -4449,24 +4482,14 @@ static void __init kfree_rcu_batch_init(void)
 
 	for_each_possible_cpu(cpu) {
 		struct kfree_rcu_cpu *krcp = per_cpu_ptr(&krc, cpu);
-		struct kvfree_rcu_bulk_data *bnode;
 
 		for (i = 0; i < KFREE_N_BATCHES; i++) {
 			INIT_RCU_WORK(&krcp->krw_arr[i].rcu_work, kfree_rcu_work);
 			krcp->krw_arr[i].krcp = krcp;
 		}
 
-		for (i = 0; i < rcu_min_cached_objs; i++) {
-			bnode = (struct kvfree_rcu_bulk_data *)
-				__get_free_page(GFP_NOWAIT | __GFP_NOWARN);
-
-			if (bnode)
-				put_cached_bnode(krcp, bnode);
-			else
-				pr_err("Failed to preallocate for %d CPU!\n", cpu);
-		}
-
 		INIT_DELAYED_WORK(&krcp->monitor_work, kfree_rcu_monitor);
+		INIT_WORK(&krcp->page_cache_work, fill_page_cache_func);
 		krcp->initialized = true;
 	}
 	if (register_shrinker(&kfree_rcu_shrinker))
Signature · 1 line
2.20.1
author maintainer
#30 lore raw
Quoted 32 lines · Joel Fernandes
Quoted 22 lines · Uladzislau Rezki (Sony)

The current memmory-allocation interface presents to following difficulties that this patch is designed to overcome:

a) If built with CONFIG_PROVE_RAW_LOCK_NESTING, the lockdep will complain about violation("BUG: Invalid wait context") of the nesting rules. It does the raw_spinlock vs. spinlock nesting checks, i.e. it is not legal to acquire a spinlock_t while holding a raw_spinlock_t.

   Internally the kfree_rcu() uses raw_spinlock_t whereas the
   "page allocator" internally deals with spinlock_t to access
   to its zones. The code also can be broken from higher level
   of view:
   <snip>
       raw_spin_lock(&some_lock);
       kfree_rcu(some_pointer, some_field_offset);
   <snip>

b) If built with CONFIG_PREEMPT_RT. Please note, in that case spinlock_t is converted into sleepable variant. Invoking the page allocator from atomic contexts leads to "BUG: scheduling while atomic".

Quoted 8 lines

c) call_rcu() is invoked from raw atomic context and kfree_rcu() and kvfree_rcu() are expected to be called from atomic raw context as well.

Move out a page allocation from contexts which trigger kvfree_rcu() function to the separate worker. When a k[v]free_rcu() per-cpu page cache is empty a fallback mechanism is used and a special job is scheduled to refill the per-cpu cache.

Quoted 7 lines

Looks good, still reviewing here. BTW just for my education, I was wondering about Thomas's email:
https://lkml.org/lkml/2020/8/11/939

If slab allocations in pure raw-atomic context on RT is not allowed or recommended, should kfree_rcu() be allowed?

Thanks for reviewing, Joel :)

The decision was made that we need to support kfree_rcu() from "real atomic contexts", to align with how it used to be before. We can go and just convert our local locks to the spinlock_t variant but that was not Paul goal, it can be that some users need kfree_rcu() for raw atomics.

Quoted 5 lines

slab can have same issue right? If per-cpu cache is drained, it has to allocate page from buddy allocator and there's no GFP flag to tell it about context where alloc is happening from.

Sounds like that. Apart of that, it might turn out soon that we or somebody else will rise a question one more time about something GFP_RAW or GFP_NOLOCKS. So who knows..

Quoted 9 lines

Or are we saying that we want to support kfree on RT from raw atomic atomic context, even though kmalloc is not supported? I hate to bring up this elephant in the room, but since I am a part of the people maintaining this code, I believe I would rather set some rules than supporting unsupported usages. :-\ (Once I know what is supported and what isn't that is). If indeed raw atomic kfree_rcu() is a bogus use case because of -RT, then we ought to put a giant warning than supporting it :-(.

We discussed it several times, the conclusion was that we need to support kfree_rcu() from raw contexts. At least that was a clear signal from Paul to me. I think, if we obtain the preemtable(), so it becomes versatile, we can drop the patch that is in question later on in the future.

--
Vlad Rezki

#35 lore raw
In reply to Uladzislau Rezki (Sony)
Quoted 2 lines · Uladzislau Rezki (Sony)

The current memmory-allocation interface presents to following difficulties that this patch is designed to overcome

[...]

Quoted 28 lines
---
 kernel/rcu/tree.c | 109 ++++++++++++++++++++++++++++------------------
 1 file changed, 66 insertions(+), 43 deletions(-)
kernel/rcu/tree.c +4 -1
index 06895ef85d69..f2da2a1cc716 100644
@@ -177,7 +177,7 @@ module_param(rcu_unlock_delay, int, 0444);
  * per-CPU. Object size is equal to one page. This value
  * can be changed at boot time.
  */
-static int rcu_min_cached_objs = 2;
+static int rcu_min_cached_objs = 5;
 module_param(rcu_min_cached_objs, int, 0444);
 
 /* Retrieve RCU kthreads priority for rcutorture */
@@ -3084,6 +3084,9 @@ struct kfree_rcu_cpu_work {
  *	In order to save some per-cpu space the list is singular.
  *	Even though it is lockless an access has to be protected by the
  *	per-cpu lock.
+ * @page_cache_work: A work to refill the cache when it is empty
+ * @work_in_progress: Indicates that page_cache_work is running
+ * @hrtimer: A hrtimer for scheduling a page_cache_work
  * @nr_bkv_objs: number of allocated objects at @bkvcache.
  *
  * This is a per-CPU structure.  The reason that it is not included in

kernel/rcu/tree.c struct kfree_rcu_cpu { · line 3108

Quoted 6 lines
kernel/rcu/tree.c +3 -0
@@ -3100,6 +3103,11 @@ struct kfree_rcu_cpu {
 	bool monitor_todo;
 	bool initialized;
 	int count;
+
+	struct work_struct page_cache_work;
+	atomic_t work_in_progress;

Does it need to be atomic? run_page_cache_work() is only called under a lock. You can use xchg() there. And when you do the atomic_set, you can use WRITE_ONCE as it is a data-race.

Quoted 14 lines
kernel/rcu/tree.c +0 -4
@@ -4449,24 +4482,14 @@ static void __init kfree_rcu_batch_init(void)
 
 	for_each_possible_cpu(cpu) {
 		struct kfree_rcu_cpu *krcp = per_cpu_ptr(&krc, cpu);
-		struct kvfree_rcu_bulk_data *bnode;
 
 		for (i = 0; i < KFREE_N_BATCHES; i++) {
 			INIT_RCU_WORK(&krcp->krw_arr[i].rcu_work, kfree_rcu_work);
 			krcp->krw_arr[i].krcp = krcp;
 		}
 
-		for (i = 0; i < rcu_min_cached_objs; i++) {
-			bnode = (struct kvfree_rcu_bulk_data *)
-				__get_free_page(GFP_NOWAIT | __GFP_NOWARN);

kernel/rcu/tree.c static void __init kfree_rcu_batch_init(void) · line 4493

Quoted 10 lines
kernel/rcu/tree.c +1 -7
-
-			if (bnode)
-				put_cached_bnode(krcp, bnode);
-			else
-				pr_err("Failed to preallocate for %d CPU!\n", cpu);
-		}
-
 		INIT_DELAYED_WORK(&krcp->monitor_work, kfree_rcu_monitor);
+		INIT_WORK(&krcp->page_cache_work, fill_page_cache_func);
 		krcp->initialized = true;

During initialization, is it not better to still pre-allocate? That way you don't have to wait to get into a situation where you need to initially allocate.

AFAICS after the above line deletions, the bulk pages are initially empty.

thanks,

- Joel

Quoted 5 lines
kernel/rcu/tree.c +0 -0
 	}
 	if (register_shrinker(&kfree_rcu_shrinker))
Signature · 1 line
2.20.1
#20 lore raw

Re: [PATCH 15/16] rcu/tree: Allocate a page when caller is preemptible

In reply to Uladzislau Rezki (Sony)

Hi Vlad,
Few minor nits:

Quoted 51 lines · Uladzislau Rezki (Sony)

Given that CONFIG_PREEMPT_COUNT is unconditionally enabled by the earlier commits in this series, the preemptible() macro now properly detects preempt-disable code regions even in kernels built with CONFIG_PREEMPT_NONE.

This commit therefore uses preemptible() to determine whether allocation is possible at all for double-argument kvfree_rcu(). If !preemptible(), then allocation is not possible, and kvfree_rcu() falls back to using the less cache-friendly rcu_head approach. Even when preemptible(), the caller might be involved in reclaim, so the GFP_ flags used by double-argument kvfree_rcu() must avoid invoking reclaim processing.

Note that single-argument kvfree_rcu() must be invoked in sleepable contexts, and that its fallback is the relatively high latency synchronize_rcu(). Single-argument kvfree_rcu() therefore uses GFP_KERNEL|__GFP_RETRY_MAYFAIL to allow limited sleeping within the memory allocator.

[ paulmck: Add add_ptr_to_bulk_krc_lock header comment per Michal Hocko. ]
---
 kernel/rcu/tree.c | 48 ++++++++++++++++++++++++++++++-----------------
 1 file changed, 31 insertions(+), 17 deletions(-)
kernel/rcu/tree.c +13 -1
index f2da2a1cc716..3f9b016a44dc 100644
@@ -3406,37 +3406,55 @@ run_page_cache_worker(struct kfree_rcu_cpu *krcp)
 	}
 }
 
+// Record ptr in a page managed by krcp, with the pre-krc_this_cpu_lock()
+// state specified by flags.  If can_sleep is true, the caller must
+// be schedulable and not be holding any locks or mutexes that might be
+// acquired by the memory allocator or anything that it might invoke.
+// If !can_sleep, then if !preemptible() no allocation will be undertaken,
+// otherwise the allocation will use GFP_ATOMIC to avoid the remainder of
+// the aforementioned deadlock possibilities.  Returns true if ptr was
+// successfully recorded, else the caller must use a fallback.
 static inline bool
-kvfree_call_rcu_add_ptr_to_bulk(struct kfree_rcu_cpu *krcp, void *ptr)
+add_ptr_to_bulk_krc_lock(struct kfree_rcu_cpu **krcp,
+	unsigned long *flags, void *ptr, bool can_sleep)
 {
 	struct kvfree_rcu_bulk_data *bnode;
+	bool can_alloc_page = preemptible();
+	gfp_t gfp = (can_sleep ? GFP_KERNEL | __GFP_RETRY_MAYFAIL :
+		GFP_ATOMIC) | __GFP_NOWARN;
 	int idx;

kernel/rcu/tree.c run_page_cache_worker(struct kfree_rcu_cpu *krcp) · line 3433

Quoted 10 lines
kernel/rcu/tree.c +2 -2
 
-	if (unlikely(!krcp->initialized))
+	*krcp = krc_this_cpu_lock(flags);
+	if (unlikely(!(*krcp)->initialized))
 		return false;
 
-	lockdep_assert_held(&krcp->lock);
 	idx = !!is_vmalloc_addr(ptr);
 
 	/* Check if a new block is required. */

Maybe convert this comment also to //... like the new ones you added (and the ones below).

kernel/rcu/tree.c run_page_cache_worker(struct kfree_rcu_cpu *krcp) · line 3437

Quoted 8 lines
kernel/rcu/tree.c +4 -4
-	if (!krcp->bkvhead[idx] ||
-			krcp->bkvhead[idx]->nr_records == KVFREE_BULK_MAX_ENTR) {
-		bnode = get_cached_bnode(krcp);
-		/* Switch to emergency path. */
+	if (!(*krcp)->bkvhead[idx] ||
+			(*krcp)->bkvhead[idx]->nr_records == KVFREE_BULK_MAX_ENTR) {
+		bnode = get_cached_bnode(*krcp);
+		if (!bnode && can_alloc_page) {

I think you can directly put preemptible() here with a comment saying allocate only if preemptible and get rid of can_alloc_page.

kernel/rcu/tree.c run_page_cache_worker(struct kfree_rcu_cpu *krcp) · line 3443

Quoted 6 lines
kernel/rcu/tree.c +6 -0
+			krc_this_cpu_unlock(*krcp, *flags);
+			bnode = (struct kvfree_rcu_bulk_data *)
+				__get_free_page(gfp);
+			*krcp = krc_this_cpu_lock(flags);
+		}
+

I think the "Switch to emergency path" comment should stay here before the if().

thanks,

- Joel

Quoted 47 lines
kernel/rcu/tree.c +6 -10
 		if (!bnode)
 			return false;
 
 		/* Initialize the new block. */
 		bnode->nr_records = 0;
-		bnode->next = krcp->bkvhead[idx];
+		bnode->next = (*krcp)->bkvhead[idx];
 
 		/* Attach it to the head. */
-		krcp->bkvhead[idx] = bnode;
+		(*krcp)->bkvhead[idx] = bnode;
 	}
 
 	/* Finally insert. */
-	krcp->bkvhead[idx]->records
-		[krcp->bkvhead[idx]->nr_records++] = ptr;
+	(*krcp)->bkvhead[idx]->records
+		[(*krcp)->bkvhead[idx]->nr_records++] = ptr;
 
 	return true;
 }
@@ -3474,20 +3492,16 @@ void kvfree_call_rcu(struct rcu_head *head, rcu_callback_t func)
 		ptr = (unsigned long *) func;
 	}
 
-	krcp = krc_this_cpu_lock(&flags);
-
 	// Queue the object but don't yet schedule the batch.
 	if (debug_rcu_head_queue(ptr)) {
 		// Probable double kfree_rcu(), just leak.
 		WARN_ONCE(1, "%s(): Double-freed call. rcu_head %p\n",
 			  __func__, head);
 
-		// Mark as success and leave.
-		success = true;
-		goto unlock_return;
+		return;
 	}
 
-	success = kvfree_call_rcu_add_ptr_to_bulk(krcp, ptr);
+	success = add_ptr_to_bulk_krc_lock(&krcp, &flags, ptr, !head);
 	if (!success) {
 		run_page_cache_worker(krcp);
 
Signature · 1 line
2.20.1
maintainer
#31 lore raw
In reply to Uladzislau Rezki
Quoted 40 lines · Uladzislau Rezki
Quoted 32 lines · Joel Fernandes
Quoted 22 lines · Uladzislau Rezki (Sony)

The current memmory-allocation interface presents to following difficulties that this patch is designed to overcome:

a) If built with CONFIG_PROVE_RAW_LOCK_NESTING, the lockdep will complain about violation("BUG: Invalid wait context") of the nesting rules. It does the raw_spinlock vs. spinlock nesting checks, i.e. it is not legal to acquire a spinlock_t while holding a raw_spinlock_t.

   Internally the kfree_rcu() uses raw_spinlock_t whereas the
   "page allocator" internally deals with spinlock_t to access
   to its zones. The code also can be broken from higher level
   of view:
   <snip>
       raw_spin_lock(&some_lock);
       kfree_rcu(some_pointer, some_field_offset);
   <snip>

b) If built with CONFIG_PREEMPT_RT. Please note, in that case spinlock_t is converted into sleepable variant. Invoking the page allocator from atomic contexts leads to "BUG: scheduling while atomic".

Quoted 8 lines

c) call_rcu() is invoked from raw atomic context and kfree_rcu() and kvfree_rcu() are expected to be called from atomic raw context as well.

Move out a page allocation from contexts which trigger kvfree_rcu() function to the separate worker. When a k[v]free_rcu() per-cpu page cache is empty a fallback mechanism is used and a special job is scheduled to refill the per-cpu cache.

Quoted 7 lines

Looks good, still reviewing here. BTW just for my education, I was wondering about Thomas's email:
https://lkml.org/lkml/2020/8/11/939

If slab allocations in pure raw-atomic context on RT is not allowed or recommended, should kfree_rcu() be allowed?

Quoted 6 lines

Thanks for reviewing, Joel :)

The decision was made that we need to support kfree_rcu() from "real atomic contexts", to align with how it used to be before. We can go and just convert our local locks to the spinlock_t variant but that was not Paul goal, it can be that some users need kfree_rcu() for raw atomics.

People invoke call_rcu() from raw atomics, and so we should provide the same for kfree_rcu(). Yes, people could work around a raw-atomic prohibition, but such prohibitions incur constant costs over time in terms of development effort, increased bug rate, and increased complexity. Yes, this does increase all of those for RCU, but the relative increase is negligible, RCU being what it is.

Quoted 7 lines
Quoted 4 lines

slab can have same issue right? If per-cpu cache is drained, it has to allocate page from buddy allocator and there's no GFP flag to tell it about context where alloc is happening from.

Sounds like that. Apart of that, it might turn out soon that we or somebody else will rise a question one more time about something GFP_RAW or GFP_NOLOCKS. So who knows..

I would prefer that slab provide some way of dealing with raw atomic context, but the maintainers are thus far unconvinced.

Quoted 8 lines
Quoted 8 lines

Or are we saying that we want to support kfree on RT from raw atomic atomic context, even though kmalloc is not supported? I hate to bring up this elephant in the room, but since I am a part of the people maintaining this code, I believe I would rather set some rules than supporting unsupported usages. :-\ (Once I know what is supported and what isn't that is). If indeed raw atomic kfree_rcu() is a bogus use case because of -RT, then we ought to put a giant warning than supporting it :-(.

Quoted 4 lines

We discussed it several times, the conclusion was that we need to support kfree_rcu() from raw contexts. At least that was a clear signal from Paul to me. I think, if we obtain the preemtable(), so it becomes versatile, we can drop the patch that is in question later on in the future.

Given a universally meaningful preemptible(), we could directly call the allocator in some cases. It might (or might not) still make sense to defer the allocation when preemptible() indicated that a direct call to the allocator was unsafe.

Thanx, Paul

author maintainer
#21 lore raw

Re: [PATCH 15/16] rcu/tree: Allocate a page when caller is preemptible

In reply to Joel Fernandes

Hello, Joel.

Quoted 59 lines · Uladzislau Rezki (Sony)
Quoted 57 lines

Given that CONFIG_PREEMPT_COUNT is unconditionally enabled by the earlier commits in this series, the preemptible() macro now properly detects preempt-disable code regions even in kernels built with CONFIG_PREEMPT_NONE.

This commit therefore uses preemptible() to determine whether allocation is possible at all for double-argument kvfree_rcu(). If !preemptible(), then allocation is not possible, and kvfree_rcu() falls back to using the less cache-friendly rcu_head approach. Even when preemptible(), the caller might be involved in reclaim, so the GFP_ flags used by double-argument kvfree_rcu() must avoid invoking reclaim processing.

Note that single-argument kvfree_rcu() must be invoked in sleepable contexts, and that its fallback is the relatively high latency synchronize_rcu(). Single-argument kvfree_rcu() therefore uses GFP_KERNEL|__GFP_RETRY_MAYFAIL to allow limited sleeping within the memory allocator.

[ paulmck: Add add_ptr_to_bulk_krc_lock header comment per Michal Hocko. ]
---
 kernel/rcu/tree.c | 48 ++++++++++++++++++++++++++++++-----------------
 1 file changed, 31 insertions(+), 17 deletions(-)
kernel/rcu/tree.c +15 -2
index f2da2a1cc716..3f9b016a44dc 100644
@@ -3406,37 +3406,55 @@ run_page_cache_worker(struct kfree_rcu_cpu *krcp)
 	}
 }
 
+// Record ptr in a page managed by krcp, with the pre-krc_this_cpu_lock()
+// state specified by flags.  If can_sleep is true, the caller must
+// be schedulable and not be holding any locks or mutexes that might be
+// acquired by the memory allocator or anything that it might invoke.
+// If !can_sleep, then if !preemptible() no allocation will be undertaken,
+// otherwise the allocation will use GFP_ATOMIC to avoid the remainder of
+// the aforementioned deadlock possibilities.  Returns true if ptr was
+// successfully recorded, else the caller must use a fallback.
 static inline bool
-kvfree_call_rcu_add_ptr_to_bulk(struct kfree_rcu_cpu *krcp, void *ptr)
+add_ptr_to_bulk_krc_lock(struct kfree_rcu_cpu **krcp,
+	unsigned long *flags, void *ptr, bool can_sleep)
 {
 	struct kvfree_rcu_bulk_data *bnode;
+	bool can_alloc_page = preemptible();
+	gfp_t gfp = (can_sleep ? GFP_KERNEL | __GFP_RETRY_MAYFAIL :
+		GFP_ATOMIC) | __GFP_NOWARN;
 	int idx;
 
-	if (unlikely(!krcp->initialized))
+	*krcp = krc_this_cpu_lock(flags);
+	if (unlikely(!(*krcp)->initialized))
 		return false;
 
Quoted 8 lines

kernel/rcu/tree.c run_page_cache_worker(struct kfree_rcu_cpu *krcp) · line 3433

Quoted 4 lines
kernel/rcu/tree.c +0 -1
-	lockdep_assert_held(&krcp->lock);
 	idx = !!is_vmalloc_addr(ptr);
 
 	/* Check if a new block is required. */

Maybe convert this comment also to //... like the new ones you added (and the ones below).

No, problem. I can convert it.

Quoted 12 lines

kernel/rcu/tree.c run_page_cache_worker(struct kfree_rcu_cpu *krcp) · line 3437

Quoted 8 lines
kernel/rcu/tree.c +4 -4
-	if (!krcp->bkvhead[idx] ||
-			krcp->bkvhead[idx]->nr_records == KVFREE_BULK_MAX_ENTR) {
-		bnode = get_cached_bnode(krcp);
-		/* Switch to emergency path. */
+	if (!(*krcp)->bkvhead[idx] ||
+			(*krcp)->bkvhead[idx]->nr_records == KVFREE_BULK_MAX_ENTR) {
+		bnode = get_cached_bnode(*krcp);
+		if (!bnode && can_alloc_page) {

I think you can directly put preemptible() here with a comment saying allocate only if preemptible and get rid of can_alloc_page.

Not really. We check preemtable() before acquiring the internal lock, otherwise it will always return "false". Thus, it is checked on the entry in the beginning.

Quoted 10 lines

kernel/rcu/tree.c run_page_cache_worker(struct kfree_rcu_cpu *krcp) · line 3443

Quoted 6 lines
kernel/rcu/tree.c +6 -0
+			krc_this_cpu_unlock(*krcp, *flags);
+			bnode = (struct kvfree_rcu_bulk_data *)
+				__get_free_page(gfp);
+			*krcp = krc_this_cpu_lock(flags);
+		}
+

I think the "Switch to emergency path" comment should stay here before the if().

Right. We need to keep it. Will take it back.

--
Vlad Rezki

author maintainer
#36 lore raw
In reply to Joel Fernandes
Quoted 34 lines · Joel Fernandes
Quoted 2 lines · Uladzislau Rezki (Sony)

The current memmory-allocation interface presents to following difficulties that this patch is designed to overcome

[...]

Quoted 30 lines
---
 kernel/rcu/tree.c | 109 ++++++++++++++++++++++++++++------------------
 1 file changed, 66 insertions(+), 43 deletions(-)
kernel/rcu/tree.c +4 -1
index 06895ef85d69..f2da2a1cc716 100644
@@ -177,7 +177,7 @@ module_param(rcu_unlock_delay, int, 0444);
  * per-CPU. Object size is equal to one page. This value
  * can be changed at boot time.
  */
-static int rcu_min_cached_objs = 2;
+static int rcu_min_cached_objs = 5;
 module_param(rcu_min_cached_objs, int, 0444);
 
 /* Retrieve RCU kthreads priority for rcutorture */
@@ -3084,6 +3084,9 @@ struct kfree_rcu_cpu_work {
  *	In order to save some per-cpu space the list is singular.
  *	Even though it is lockless an access has to be protected by the
  *	per-cpu lock.
+ * @page_cache_work: A work to refill the cache when it is empty
+ * @work_in_progress: Indicates that page_cache_work is running
+ * @hrtimer: A hrtimer for scheduling a page_cache_work
  * @nr_bkv_objs: number of allocated objects at @bkvcache.
  *
  * This is a per-CPU structure.  The reason that it is not included in
@@ -3100,6 +3103,11 @@ struct kfree_rcu_cpu {
 	bool monitor_todo;
 	bool initialized;
Quoted 9 lines

kernel/rcu/tree.c struct kfree_rcu_cpu { · line 3108

Quoted 4 lines
kernel/rcu/tree.c +3 -0
 	int count;
+
+	struct work_struct page_cache_work;
+	atomic_t work_in_progress;

Does it need to be atomic? run_page_cache_work() is only called under a lock. You can use xchg() there. And when you do the atomic_set, you can use WRITE_ONCE as it is a data-race.

We can use xchg together with *_ONCE() macro. Could you please clarify what is your concern about using atomic_t? Both xchg() and atomic_xchg() guarantee atamarity. Same as WRITE_ONCE() or atomic_set().

Quoted 20 lines
Quoted 20 lines
kernel/rcu/tree.c +0 -10
@@ -4449,24 +4482,14 @@ static void __init kfree_rcu_batch_init(void)
 
 	for_each_possible_cpu(cpu) {
 		struct kfree_rcu_cpu *krcp = per_cpu_ptr(&krc, cpu);
-		struct kvfree_rcu_bulk_data *bnode;
 
 		for (i = 0; i < KFREE_N_BATCHES; i++) {
 			INIT_RCU_WORK(&krcp->krw_arr[i].rcu_work, kfree_rcu_work);
 			krcp->krw_arr[i].krcp = krcp;
 		}
 
-		for (i = 0; i < rcu_min_cached_objs; i++) {
-			bnode = (struct kvfree_rcu_bulk_data *)
-				__get_free_page(GFP_NOWAIT | __GFP_NOWARN);
-
-			if (bnode)
-				put_cached_bnode(krcp, bnode);
-			else
-				pr_err("Failed to preallocate for %d CPU!\n", cpu);
-		}
Quoted 9 lines

kernel/rcu/tree.c static void __init kfree_rcu_batch_init(void) · line 4493

Quoted 4 lines
kernel/rcu/tree.c +1 -1
-
 		INIT_DELAYED_WORK(&krcp->monitor_work, kfree_rcu_monitor);
+		INIT_WORK(&krcp->page_cache_work, fill_page_cache_func);
 		krcp->initialized = true;

During initialization, is it not better to still pre-allocate? That way you don't have to wait to get into a situation where you need to initially allocate.

Since we have a worker that does it when a cache is empty there is no a high need in doing it during initialization phase. If we can reduce an amount of code it is always good :)

Thanks, Joel.

--
Vlad Rezki

author maintainer
#32 lore raw
In reply to Paul E. McKenney
Quoted 48 lines · Paul E. McKenney
Quoted 40 lines · Uladzislau Rezki
Quoted 32 lines · Joel Fernandes
Quoted 22 lines · Uladzislau Rezki (Sony)

The current memmory-allocation interface presents to following difficulties that this patch is designed to overcome:

a) If built with CONFIG_PROVE_RAW_LOCK_NESTING, the lockdep will complain about violation("BUG: Invalid wait context") of the nesting rules. It does the raw_spinlock vs. spinlock nesting checks, i.e. it is not legal to acquire a spinlock_t while holding a raw_spinlock_t.

   Internally the kfree_rcu() uses raw_spinlock_t whereas the
   "page allocator" internally deals with spinlock_t to access
   to its zones. The code also can be broken from higher level
   of view:
   <snip>
       raw_spin_lock(&some_lock);
       kfree_rcu(some_pointer, some_field_offset);
   <snip>

b) If built with CONFIG_PREEMPT_RT. Please note, in that case spinlock_t is converted into sleepable variant. Invoking the page allocator from atomic contexts leads to "BUG: scheduling while atomic".

Quoted 8 lines

c) call_rcu() is invoked from raw atomic context and kfree_rcu() and kvfree_rcu() are expected to be called from atomic raw context as well.

Move out a page allocation from contexts which trigger kvfree_rcu() function to the separate worker. When a k[v]free_rcu() per-cpu page cache is empty a fallback mechanism is used and a special job is scheduled to refill the per-cpu cache.

Quoted 7 lines

Looks good, still reviewing here. BTW just for my education, I was wondering about Thomas's email:
https://lkml.org/lkml/2020/8/11/939

If slab allocations in pure raw-atomic context on RT is not allowed or recommended, should kfree_rcu() be allowed?

Quoted 6 lines

Thanks for reviewing, Joel :)

The decision was made that we need to support kfree_rcu() from "real atomic contexts", to align with how it used to be before. We can go and just convert our local locks to the spinlock_t variant but that was not Paul goal, it can be that some users need kfree_rcu() for raw atomics.

Quoted 7 lines

People invoke call_rcu() from raw atomics, and so we should provide the same for kfree_rcu(). Yes, people could work around a raw-atomic prohibition, but such prohibitions incur constant costs over time in terms of development effort, increased bug rate, and increased complexity. Yes, this does increase all of those for RCU, but the relative increase is negligible, RCU being what it is.

I see your point.

Quoted 11 lines
Quoted 7 lines
Quoted 4 lines

slab can have same issue right? If per-cpu cache is drained, it has to allocate page from buddy allocator and there's no GFP flag to tell it about context where alloc is happening from.

Sounds like that. Apart of that, it might turn out soon that we or somebody else will rise a question one more time about something GFP_RAW or GFP_NOLOCKS. So who knows..

I would prefer that slab provide some way of dealing with raw atomic context, but the maintainers are thus far unconvinced.

I think, when preempt_rt is fully integrated to the kernel, we might get new users with such demand. So, it is not a closed topic so far, IMHO.

Quoted 13 lines
Quoted 8 lines
Quoted 8 lines

Or are we saying that we want to support kfree on RT from raw atomic atomic context, even though kmalloc is not supported? I hate to bring up this elephant in the room, but since I am a part of the people maintaining this code, I believe I would rather set some rules than supporting unsupported usages. :-\ (Once I know what is supported and what isn't that is). If indeed raw atomic kfree_rcu() is a bogus use case because of -RT, then we ought to put a giant warning than supporting it :-(.

Quoted 4 lines

We discussed it several times, the conclusion was that we need to support kfree_rcu() from raw contexts. At least that was a clear signal from Paul to me. I think, if we obtain the preemtable(), so it becomes versatile, we can drop the patch that is in question later on in the future.

Quoted 5 lines

Given a universally meaningful preemptible(), we could directly call the allocator in some cases. It might (or might not) still make sense to defer the allocation when preemptible() indicated that a direct call to the allocator was unsafe.

I do not have a strong opinion here. Giving the fact that maintaining of such "deferring" is not considered as a big effort, i think, we can live with it.

--
Vlad Rezki

maintainer
#33 lore raw
Quoted 66 lines · Uladzislau Rezki
Quoted 48 lines · Paul E. McKenney
Quoted 40 lines · Uladzislau Rezki
Quoted 32 lines · Joel Fernandes
Quoted 22 lines · Uladzislau Rezki (Sony)

The current memmory-allocation interface presents to following difficulties that this patch is designed to overcome:

a) If built with CONFIG_PROVE_RAW_LOCK_NESTING, the lockdep will complain about violation("BUG: Invalid wait context") of the nesting rules. It does the raw_spinlock vs. spinlock nesting checks, i.e. it is not legal to acquire a spinlock_t while holding a raw_spinlock_t.

   Internally the kfree_rcu() uses raw_spinlock_t whereas the
   "page allocator" internally deals with spinlock_t to access
   to its zones. The code also can be broken from higher level
   of view:
   <snip>
       raw_spin_lock(&some_lock);
       kfree_rcu(some_pointer, some_field_offset);
   <snip>

b) If built with CONFIG_PREEMPT_RT. Please note, in that case spinlock_t is converted into sleepable variant. Invoking the page allocator from atomic contexts leads to "BUG: scheduling while atomic".

Quoted 8 lines

c) call_rcu() is invoked from raw atomic context and kfree_rcu() and kvfree_rcu() are expected to be called from atomic raw context as well.

Move out a page allocation from contexts which trigger kvfree_rcu() function to the separate worker. When a k[v]free_rcu() per-cpu page cache is empty a fallback mechanism is used and a special job is scheduled to refill the per-cpu cache.

Quoted 7 lines

Looks good, still reviewing here. BTW just for my education, I was wondering about Thomas's email:
https://lkml.org/lkml/2020/8/11/939

If slab allocations in pure raw-atomic context on RT is not allowed or recommended, should kfree_rcu() be allowed?

Quoted 6 lines

Thanks for reviewing, Joel :)

The decision was made that we need to support kfree_rcu() from "real atomic contexts", to align with how it used to be before. We can go and just convert our local locks to the spinlock_t variant but that was not Paul goal, it can be that some users need kfree_rcu() for raw atomics.

Quoted 7 lines

People invoke call_rcu() from raw atomics, and so we should provide the same for kfree_rcu(). Yes, people could work around a raw-atomic prohibition, but such prohibitions incur constant costs over time in terms of development effort, increased bug rate, and increased complexity. Yes, this does increase all of those for RCU, but the relative increase is negligible, RCU being what it is.

I see your point.

Quoted 8 lines
Quoted 7 lines
Quoted 4 lines

slab can have same issue right? If per-cpu cache is drained, it has to allocate page from buddy allocator and there's no GFP flag to tell it about context where alloc is happening from.

Sounds like that. Apart of that, it might turn out soon that we or somebody else will rise a question one more time about something GFP_RAW or GFP_NOLOCKS. So who knows..

Quoted 5 lines
Quoted 3 lines

I would prefer that slab provide some way of dealing with raw atomic context, but the maintainers are thus far unconvinced.

I think, when preempt_rt is fully integrated to the kernel, we might get new users with such demand. So, it is not a closed topic so far, IMHO.

Agreed! ;-)

Quoted 13 lines
Quoted 13 lines
Quoted 12 lines
Quoted 8 lines

Or are we saying that we want to support kfree on RT from raw atomic atomic context, even though kmalloc is not supported? I hate to bring up this elephant in the room, but since I am a part of the people maintaining this code, I believe I would rather set some rules than supporting unsupported usages. :-\ (Once I know what is supported and what isn't that is). If indeed raw atomic kfree_rcu() is a bogus use case because of -RT, then we ought to put a giant warning than supporting it :-(.

We discussed it several times, the conclusion was that we need to support kfree_rcu() from raw contexts. At least that was a clear signal from Paul to me. I think, if we obtain the preemtable(), so it becomes versatile, we can drop the patch that is in question later on in the future.

Quoted 8 lines
Quoted 5 lines

Given a universally meaningful preemptible(), we could directly call the allocator in some cases. It might (or might not) still make sense to defer the allocation when preemptible() indicated that a direct call to the allocator was unsafe.

I do not have a strong opinion here. Giving the fact that maintaining of such "deferring" is not considered as a big effort, i think, we can live with it.

And agreed here as well. If this were instead a large body of complex code, I might feel otherwise. But as it is, why worry?

Thanx, Paul

#22 lore raw

Re: [PATCH 15/16] rcu/tree: Allocate a page when caller is preemptible

In reply to Uladzislau Rezki

On Wed, Nov 04, 2020 at 12:39:31PM +0100, Uladzislau Rezki wrote:
[..]

Quoted 12 lines
Quoted 12 lines

kernel/rcu/tree.c run_page_cache_worker(struct kfree_rcu_cpu *krcp) · line 3437

Quoted 8 lines
kernel/rcu/tree.c +4 -4
-	if (!krcp->bkvhead[idx] ||
-			krcp->bkvhead[idx]->nr_records == KVFREE_BULK_MAX_ENTR) {
-		bnode = get_cached_bnode(krcp);
-		/* Switch to emergency path. */
+	if (!(*krcp)->bkvhead[idx] ||
+			(*krcp)->bkvhead[idx]->nr_records == KVFREE_BULK_MAX_ENTR) {
+		bnode = get_cached_bnode(*krcp);
+		if (!bnode && can_alloc_page) {

I think you can directly put preemptible() here with a comment saying allocate only if preemptible and get rid of can_alloc_page.

Quoted 3 lines

Not really. We check preemtable() before acquiring the internal lock, otherwise it will always return "false". Thus, it is checked on the entry in the beginning.

You are right. Sorry. Sounds good to me.

- Joel

author maintainer
#34 lore raw
In reply to Paul E. McKenney
Quoted 93 lines · Paul E. McKenney
Quoted 66 lines · Uladzislau Rezki
Quoted 48 lines · Paul E. McKenney
Quoted 40 lines · Uladzislau Rezki
Quoted 32 lines · Joel Fernandes
Quoted 22 lines · Uladzislau Rezki (Sony)

The current memmory-allocation interface presents to following difficulties that this patch is designed to overcome:

a) If built with CONFIG_PROVE_RAW_LOCK_NESTING, the lockdep will complain about violation("BUG: Invalid wait context") of the nesting rules. It does the raw_spinlock vs. spinlock nesting checks, i.e. it is not legal to acquire a spinlock_t while holding a raw_spinlock_t.

   Internally the kfree_rcu() uses raw_spinlock_t whereas the
   "page allocator" internally deals with spinlock_t to access
   to its zones. The code also can be broken from higher level
   of view:
   <snip>
       raw_spin_lock(&some_lock);
       kfree_rcu(some_pointer, some_field_offset);
   <snip>

b) If built with CONFIG_PREEMPT_RT. Please note, in that case spinlock_t is converted into sleepable variant. Invoking the page allocator from atomic contexts leads to "BUG: scheduling while atomic".

Quoted 8 lines

c) call_rcu() is invoked from raw atomic context and kfree_rcu() and kvfree_rcu() are expected to be called from atomic raw context as well.

Move out a page allocation from contexts which trigger kvfree_rcu() function to the separate worker. When a k[v]free_rcu() per-cpu page cache is empty a fallback mechanism is used and a special job is scheduled to refill the per-cpu cache.

Quoted 7 lines

Looks good, still reviewing here. BTW just for my education, I was wondering about Thomas's email:
https://lkml.org/lkml/2020/8/11/939

If slab allocations in pure raw-atomic context on RT is not allowed or recommended, should kfree_rcu() be allowed?

Quoted 6 lines

Thanks for reviewing, Joel :)

The decision was made that we need to support kfree_rcu() from "real atomic contexts", to align with how it used to be before. We can go and just convert our local locks to the spinlock_t variant but that was not Paul goal, it can be that some users need kfree_rcu() for raw atomics.

Quoted 7 lines

People invoke call_rcu() from raw atomics, and so we should provide the same for kfree_rcu(). Yes, people could work around a raw-atomic prohibition, but such prohibitions incur constant costs over time in terms of development effort, increased bug rate, and increased complexity. Yes, this does increase all of those for RCU, but the relative increase is negligible, RCU being what it is.

I see your point.

Quoted 8 lines
Quoted 7 lines
Quoted 4 lines

slab can have same issue right? If per-cpu cache is drained, it has to allocate page from buddy allocator and there's no GFP flag to tell it about context where alloc is happening from.

Sounds like that. Apart of that, it might turn out soon that we or somebody else will rise a question one more time about something GFP_RAW or GFP_NOLOCKS. So who knows..

Quoted 5 lines
Quoted 3 lines

I would prefer that slab provide some way of dealing with raw atomic context, but the maintainers are thus far unconvinced.

I think, when preempt_rt is fully integrated to the kernel, we might get new users with such demand. So, it is not a closed topic so far, IMHO.

Agreed! ;-)

Quoted 18 lines
Quoted 18 lines
Quoted 8 lines
Quoted 8 lines

Or are we saying that we want to support kfree on RT from raw atomic atomic context, even though kmalloc is not supported? I hate to bring up this elephant in the room, but since I am a part of the people maintaining this code, I believe I would rather set some rules than supporting unsupported usages. :-\ (Once I know what is supported and what isn't that is). If indeed raw atomic kfree_rcu() is a bogus use case because of -RT, then we ought to put a giant warning than supporting it :-(.

Quoted 4 lines

We discussed it several times, the conclusion was that we need to support kfree_rcu() from raw contexts. At least that was a clear signal from Paul to me. I think, if we obtain the preemtable(), so it becomes versatile, we can drop the patch that is in question later on in the future.

Given a universally meaningful preemptible(), we could directly call the allocator in some cases. It might (or might not) still make sense to defer the allocation when preemptible() indicated that a direct call to the allocator was unsafe.

Quoted 7 lines
Quoted 3 lines

I do not have a strong opinion here. Giving the fact that maintaining of such "deferring" is not considered as a big effort, i think, we can live with it.

And agreed here as well. If this were instead a large body of complex code, I might feel otherwise. But as it is, why worry?

Agreed! I do not consider it as extra complexity.

--
Vlad Rezki

#37 lore raw
In reply to Uladzislau Rezki
Quoted 44 lines · Uladzislau Rezki
Quoted 34 lines · Joel Fernandes
Quoted 2 lines · Uladzislau Rezki (Sony)

The current memmory-allocation interface presents to following difficulties that this patch is designed to overcome

[...]

Quoted 30 lines
---
 kernel/rcu/tree.c | 109 ++++++++++++++++++++++++++++------------------
 1 file changed, 66 insertions(+), 43 deletions(-)
kernel/rcu/tree.c +4 -1
index 06895ef85d69..f2da2a1cc716 100644
@@ -177,7 +177,7 @@ module_param(rcu_unlock_delay, int, 0444);
  * per-CPU. Object size is equal to one page. This value
  * can be changed at boot time.
  */
-static int rcu_min_cached_objs = 2;
+static int rcu_min_cached_objs = 5;
 module_param(rcu_min_cached_objs, int, 0444);
 
 /* Retrieve RCU kthreads priority for rcutorture */
@@ -3084,6 +3084,9 @@ struct kfree_rcu_cpu_work {
  *	In order to save some per-cpu space the list is singular.
  *	Even though it is lockless an access has to be protected by the
  *	per-cpu lock.
+ * @page_cache_work: A work to refill the cache when it is empty
+ * @work_in_progress: Indicates that page_cache_work is running
+ * @hrtimer: A hrtimer for scheduling a page_cache_work
  * @nr_bkv_objs: number of allocated objects at @bkvcache.
  *
  * This is a per-CPU structure.  The reason that it is not included in
@@ -3100,6 +3103,11 @@ struct kfree_rcu_cpu {
 	bool monitor_todo;
 	bool initialized;
Quoted 9 lines

kernel/rcu/tree.c struct kfree_rcu_cpu { · line 3108

Quoted 4 lines
kernel/rcu/tree.c +3 -0
 	int count;
+
+	struct work_struct page_cache_work;
+	atomic_t work_in_progress;

Does it need to be atomic? run_page_cache_work() is only called under a lock. You can use xchg() there. And when you do the atomic_set, you can use WRITE_ONCE as it is a data-race.

Quoted 3 lines

We can use xchg together with *_ONCE() macro. Could you please clarify what is your concern about using atomic_t? Both xchg() and atomic_xchg() guarantee atamarity. Same as WRITE_ONCE() or atomic_set().

Right, whether there's lock or not does not matter as xchg() is also atomic-swap.

atomic_t is a more complex type though, I would directly use int since atomic_t is not needed here and there's no lost-update issue here. It could be matter of style as well.

BTW I did think atomic_xchg() adds additional memory barriers
but I could not find that to be the case in the implementation. Is that not the case? Docs says "atomic_xchg must provide explicit memory barriers around the operation.".

Quoted 29 lines
Quoted 20 lines
Quoted 20 lines
kernel/rcu/tree.c +0 -10
@@ -4449,24 +4482,14 @@ static void __init kfree_rcu_batch_init(void)
 
 	for_each_possible_cpu(cpu) {
 		struct kfree_rcu_cpu *krcp = per_cpu_ptr(&krc, cpu);
-		struct kvfree_rcu_bulk_data *bnode;
 
 		for (i = 0; i < KFREE_N_BATCHES; i++) {
 			INIT_RCU_WORK(&krcp->krw_arr[i].rcu_work, kfree_rcu_work);
 			krcp->krw_arr[i].krcp = krcp;
 		}
 
-		for (i = 0; i < rcu_min_cached_objs; i++) {
-			bnode = (struct kvfree_rcu_bulk_data *)
-				__get_free_page(GFP_NOWAIT | __GFP_NOWARN);
-
-			if (bnode)
-				put_cached_bnode(krcp, bnode);
-			else
-				pr_err("Failed to preallocate for %d CPU!\n", cpu);
-		}
Quoted 9 lines

kernel/rcu/tree.c static void __init kfree_rcu_batch_init(void) · line 4493

Quoted 4 lines
kernel/rcu/tree.c +1 -1
-
 		INIT_DELAYED_WORK(&krcp->monitor_work, kfree_rcu_monitor);
+		INIT_WORK(&krcp->page_cache_work, fill_page_cache_func);
 		krcp->initialized = true;

During initialization, is it not better to still pre-allocate? That way you don't have to wait to get into a situation where you need to initially allocate.

Quoted 3 lines

Since we have a worker that does it when a cache is empty there is no a high need in doing it during initialization phase. If we can reduce an amount of code it is always good :)

I am all for not having more code than needed. But you would hit synchronize_rcu() slow path immediately on first headless kfree_rcu() right? That seems like a step back from the current code :)

thanks,

- Joel

author maintainer
#38 lore raw
Quoted 20 lines
Quoted 13 lines
Quoted 4 lines
Quoted 4 lines

* This is a per-CPU structure. The reason that it is not included in

kernel/rcu/tree.c +0 -0
@@ -3100,6 +3103,11 @@ struct kfree_rcu_cpu {
 	bool monitor_todo;
 	bool initialized;
Quoted 9 lines

kernel/rcu/tree.c struct kfree_rcu_cpu { · line 3108

Quoted 4 lines
kernel/rcu/tree.c +3 -0
 	int count;
+
+	struct work_struct page_cache_work;
+	atomic_t work_in_progress;

Does it need to be atomic? run_page_cache_work() is only called under a lock. You can use xchg() there. And when you do the atomic_set, you can use WRITE_ONCE as it is a data-race.

Quoted 3 lines

We can use xchg together with *_ONCE() macro. Could you please clarify what is your concern about using atomic_t? Both xchg() and atomic_xchg() guarantee atamarity. Same as WRITE_ONCE() or atomic_set().

Right, whether there's lock or not does not matter as xchg() is also atomic-swap.

Quoted 9 lines

atomic_t is a more complex type though, I would directly use int since atomic_t is not needed here and there's no lost-update issue here. It could be matter of style as well.

BTW I did think atomic_xchg() adds additional memory barriers
but I could not find that to be the case in the implementation. Is that not the case? Docs says "atomic_xchg must provide explicit memory barriers around the operation.".

In most of the systems atmoc_xchg() is same as xchg() and atomic_set() is same as WRITE_ONCE(). But there are exceptions, for example "parisc"

*** arch/parisc/include/asm/atomic.h:
<snip>
...

#define _atomic_spin_lock_irqsave(l,f) do { \
    arch_spinlock_t *s = ATOMIC_HASH(l); \
    local_irq_save(f);   \
    arch_spin_lock(s);   \
} while(0)
...
static __inline__ void atomic_set(atomic_t *v, int i)
{
     unsigned long flags;
     _atomic_spin_lock_irqsave(v, flags);

     v->counter = i;

     _atomic_spin_unlock_irqrestore(v, flags);
}

<snip>

I will switch to xchg() and WRITE_ONCE(), because of such specific ARCHs.

Quoted 29 lines
Quoted 29 lines
Quoted 29 lines
Quoted 14 lines
kernel/rcu/tree.c +0 -4
@@ -4449,24 +4482,14 @@ static void __init kfree_rcu_batch_init(void)
 
 	for_each_possible_cpu(cpu) {
 		struct kfree_rcu_cpu *krcp = per_cpu_ptr(&krc, cpu);
-		struct kvfree_rcu_bulk_data *bnode;
 
 		for (i = 0; i < KFREE_N_BATCHES; i++) {
 			INIT_RCU_WORK(&krcp->krw_arr[i].rcu_work, kfree_rcu_work);
 			krcp->krw_arr[i].krcp = krcp;
 		}
 
-		for (i = 0; i < rcu_min_cached_objs; i++) {
-			bnode = (struct kvfree_rcu_bulk_data *)
-				__get_free_page(GFP_NOWAIT | __GFP_NOWARN);

kernel/rcu/tree.c static void __init kfree_rcu_batch_init(void) · line 4493

Quoted 10 lines
kernel/rcu/tree.c +1 -7
-
-			if (bnode)
-				put_cached_bnode(krcp, bnode);
-			else
-				pr_err("Failed to preallocate for %d CPU!\n", cpu);
-		}
-
 		INIT_DELAYED_WORK(&krcp->monitor_work, kfree_rcu_monitor);
+		INIT_WORK(&krcp->page_cache_work, fill_page_cache_func);
 		krcp->initialized = true;

During initialization, is it not better to still pre-allocate? That way you don't have to wait to get into a situation where you need to initially allocate.

Quoted 8 lines
Quoted 3 lines

Since we have a worker that does it when a cache is empty there is no a high need in doing it during initialization phase. If we can reduce an amount of code it is always good :)

I am all for not having more code than needed. But you would hit synchronize_rcu() slow path immediately on first headless kfree_rcu() right? That seems like a step back from the current code :)

As for slow path and hitting the synchronize_rcu() immediately. Yes, a slow hit "counter" will be increased by 1, the difference between two variants will be N and N + 1 times. I do not consider N + 1 as a big difference and impact on performance.

Should we guarantee that a first user does not hit a fallback path that invokes synchronize_rcu()? If not, i would rather remove redundant code.

Any thoughts here?

Thanks!

--
Vlad Rezki