Reduce ratelimit's false-positive misses
Subsystems (6)
cover Reduce ratelimit's false-positive misses
Hello!
This RFC series replaces open-coded uses of the ratelimit_state structure with formal APIs, counts all rate-limit misses, replaces jiffies=0 special case with a flag, provides a ___ratelimit() lockless fastpath to (almost) eliminate false-positive misses, and adds a simple test. The lockless fastpath also provides a 10-20% speedup on my x86 laptop, though your mileage may vary.
The key point of this series is the reduction of false-positive misses. Patches 4-7 move in this direction within the fully locked confines of the current design, and patch 8 introduces the lockless fastpath.
The individual patches are as follows:
1. Create functions to handle ratelimit_state internals.
2. random: Avoid open-coded use of ratelimit_state structure's
->missed field.
3. drm/i915: Avoid open-coded use of ratelimit_state structure's
->missed field.
4. drm/amd/pm: Avoid open-coded use of ratelimit_state structure's
internals.
5. Convert the ->missed field to atomic_t.
6. Count misses due to lock contention.
7. Avoid jiffies=0 special case.
8. Reduce ratelimit's false-positive misses.
9. Add trivial kunit test for ratelimit.
Thanx, Paul
------------------------------------------------------------------------
b/drivers/char/random.c | 9 - b/drivers/gpu/drm/amd/pm/amdgpu_pm.c | 11 -- b/drivers/gpu/drm/i915/i915_perf.c | 8 - b/include/linux/ratelimit.h | 40 ++++++- b/include/linux/ratelimit_types.h | 2 b/lib/Kconfig.debug | 11 ++ b/lib/Makefile | 1 b/lib/ratelimit.c | 8 - b/lib/test_ratelimit.c | 77 ++++++++++++++ include/linux/ratelimit.h | 13 -- include/linux/ratelimit_types.h | 4 lib/ratelimit.c | 186 +++++++++++++++++++++++++++-------- 12 files changed, 296 insertions(+), 74 deletions(-)
RFC PATCH 1/9 ratelimit: Create functions to handle ratelimit_state internals
A number of ratelimit uses do open-coded access to the ratelimit_state structure's ->missed field. This works, but is a bit messy and makes it more annoying to make changes to this field.
Therefore, provide a ratelimit_state_inc_miss() function that increments out the ->missed field ratelimit_state_get_miss() function that reads out the ->missed field of the specified ratelimit_state structure, and a ratelimit_state_reset_miss() function that reads out that field, but that also resets its value to zero. These functions will replace client-code open-coded uses of ->miss.
In addition, a new ratelimit_state_reset_interval() encapsulates what was previously open-coded lock acquisition and resetting.
[ paulmck: Apply kernel test robot feedback. ]
--- include/linux/ratelimit.h | 40 ++++++++++++++++++++++++++++++++++----- lib/ratelimit.c | 8 ++++---- 2 files changed, 39 insertions(+), 9 deletions(-)
include/linux/ratelimit.h +35 -5
@@ -22,16 +22,46 @@ static inline void ratelimit_default_init(struct ratelimit_state *rs) DEFAULT_RATELIMIT_BURST); } +static inline void ratelimit_state_inc_miss (struct ratelimit_state *rs) +{ + rs->missed++; +} + +static inline int ratelimit_state_get_miss (struct ratelimit_state *rs) +{ +return rs->missed; +} + +static inline int ratelimit_state_reset_miss (struct ratelimit_state *rs) +{ +int ret = rs->missed; + + rs->missed =0 ; +return ret; +} + +static inline void ratelimit_state_reset_interval (struct ratelimit_state *rs,int interval_init) +{ +unsigned long flags; + +raw_spin_lock_irqsave (&rs->lock, flags); + rs->interval = interval_init; + rs->begin =0 ; + rs->printed =0 ; +ratelimit_state_reset_miss (rs); +raw_spin_unlock_irqrestore (&rs->lock, flags); +} +static inline void ratelimit_state_exit (struct ratelimit_state *rs) { +int m; +if (!(rs->flags &RATELIMIT_MSG_ON_RELEASE ))return ; -if (rs->missed) { -pr_warn ("%s: %d output lines suppressed due to ratelimiting\n" , - current->comm, rs->missed); - rs->missed =0 ; - } + m =ratelimit_state_reset_miss (rs); +if (m) +pr_warn ("%s: %d output lines suppressed due to ratelimiting\n" , current->comm, m); } static inline void
lib/ratelimit.c +4 -4
@@ -51,12 +51,12 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func) rs->begin = jiffies;if (time_is_before_jiffies (rs->begin + interval)) { -if (rs->missed) { +int m =ratelimit_state_reset_miss (rs); + +if (m) {if (!(rs->flags &RATELIMIT_MSG_ON_RELEASE )) {printk_deferred (KERN_WARNING -"%s: %d callbacks suppressed\n" , - func, rs->missed); - rs->missed =0 ; +"%s: %d callbacks suppressed\n" , func, m); } } rs->begin = jiffies;
Signature · 1 line
2.40.1
RFC PATCH 2/9 random: Avoid open-coded use of ratelimit_state structure's ->missed field
In reply to Paul E. McKenneyThe _credit_init_bits() function directly accesses the ratelimit_state structure's ->missed field, which work, but which also makes it more difficult to change this field. Therefore, make use of the ratelimit_state_get_miss() and ratelimit_state_inc_miss() functions instead of directly accessing the ->missed field.
"Jason A. Donenfeld" <[email protected]>
--- drivers/char/random.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-)
drivers/char/random.c +5 -4
@@ -726,6 +726,7 @@ static void __cold _credit_init_bits(size_t bits) static DECLARE_WORK(set_ready, crng_set_ready);unsigned int new, orig, add;unsigned long flags; +int m;if (!bits)return ;
@@ -748,9 +749,9 @@ static void __cold _credit_init_bits(size_t bits)wake_up_interruptible (&crng_init_wait);kill_fasync (&fasync,SIGIO ,POLL_IN );pr_notice ("crng init done\n" ); -if (urandom_warning.missed) -pr_notice ("%d urandom warning(s) missed due to ratelimiting\n" , - urandom_warning.missed); + m =ratelimit_state_get_miss (&urandom_warning); +if (m) +pr_notice ("%d urandom warning(s) missed due to ratelimiting\n" , m); }else if (orig <POOL_EARLY_BITS && new >=POOL_EARLY_BITS ) {spin_lock_irqsave (&base_crng.lock, flags);/* Check if crng_init is CRNG_EMPTY, to avoid race with crng_reseed(). */
@@ -1466,7 +1467,7 @@ static ssize_t urandom_read_iter(struct kiocb *kiocb, struct iov_iter *iter)if (!crng_ready ()) {if (!ratelimit_disable && maxwarn <=0 ) - ++urandom_warning.missed; +ratelimit_state_inc_miss (&urandom_warning);else if (ratelimit_disable ||__ratelimit (&urandom_warning)) { --maxwarn; pr_notice("%s: uninitialized urandom read (%zu bytes read)\n",
Signature · 1 line
2.40.1
RFC PATCH 3/9 drm/i915: Avoid open-coded use of ratelimit_state structure's ->missed field
In reply to Paul E. McKenneyThe i915_oa_stream_destroy() function directly accesses the ratelimit_state structure's ->missed field, which work, but which also makes it more difficult to change this field. Therefore, make use of the ratelimit_state_get_miss() function instead of directly accessing the ->missed field.
--- drivers/gpu/drm/i915/i915_perf.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-)
drivers/gpu/drm/i915/i915_perf.c +4 -4
@@ -1663,6 +1663,7 @@ static void i915_oa_stream_destroy(struct i915_perf_stream *stream)struct i915_perf *perf = stream->perf;struct intel_gt *gt = stream->engine->gt;struct i915_perf_group *g = stream->engine->oa_group; +int m;if (WARN_ON (stream != g->exclusive_stream))return ;
@@ -1687,10 +1688,9 @@ static void i915_oa_stream_destroy(struct i915_perf_stream *stream)free_oa_configs (stream);free_noa_wait (stream); -if (perf->spurious_report_rs.missed) { -gt_notice (gt,"%d spurious OA report notices suppressed due to ratelimiting\n" , - perf->spurious_report_rs.missed); - } + m =ratelimit_state_get_miss (&perf->spurious_report_rs); +if (m) +gt_notice (gt,"%d spurious OA report notices suppressed due to ratelimiting\n" , m); }static void gen7_init_oa_buffer (struct i915_perf_stream *stream)
Signature · 1 line
2.40.1
RFC PATCH 4/9 drm/amd/pm: Avoid open-coded use of ratelimit_state structure's internals
In reply to Paul E. McKenneyThe amdgpu_set_thermal_throttling_logging() function directly accesses the ratelimit_state structure's ->missed field, which work, but which also makes it more difficult to change this field. Therefore, make use of the ratelimit_state_reset_miss() function instead of directly accessing the ->missed field.
Nevertheless, open-coded use of ->burst and ->interval is still permitted, for example, for runtime sysfs adjustment of these fields.
--- drivers/gpu/drm/amd/pm/amdgpu_pm.c | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-)
drivers/gpu/drm/amd/pm/amdgpu_pm.c +2 -9
@@ -1663,7 +1663,6 @@ static ssize_t amdgpu_set_thermal_throttling_logging(struct device *dev,struct drm_device *ddev =dev_get_drvdata (dev);struct amdgpu_device *adev =drm_to_adev (ddev);long throttling_logging_interval; -unsigned long flags;int ret =0 ; ret =kstrtol (buf,0 , &throttling_logging_interval);
@@ -1674,18 +1673,12 @@ static ssize_t amdgpu_set_thermal_throttling_logging(struct device *dev,return -EINVAL ;if (throttling_logging_interval >0 ) { -raw_spin_lock_irqsave (&adev->throttling_logging_rs.lock, flags);/* * Reset the ratelimit timer internals. * This can effectively restart the timer. */ - adev->throttling_logging_rs.interval = - (throttling_logging_interval -1 ) *HZ ; - adev->throttling_logging_rs.begin =0 ; - adev->throttling_logging_rs.printed =0 ; - adev->throttling_logging_rs.missed =0 ; -raw_spin_unlock_irqrestore (&adev->throttling_logging_rs.lock, flags); - +ratelimit_state_reset_interval (&adev->throttling_logging_rs, + (throttling_logging_interval -1 ) *HZ );atomic_set (&adev->throttling_logging_enabled,1 ); }else {atomic_set (&adev->throttling_logging_enabled,0 );
Signature · 1 line
2.40.1
RE: [PATCH RFC 4/9] drm/amd/pm: Avoid open-coded use of ratelimit_state structure's internals
[Public]
Quoted 18 lines
-----Original Message-----
From: Paul E. McKenney <[email protected]>
Sent: Thursday, April 3, 2025 5:15 PM
To: [email protected]
Cc: [email protected]; Paul E. McKenney <[email protected]>; kernel test robot <[email protected]>; Feng, Kenneth <[email protected]>; Deucher, Alexander <[email protected]>; Koenig, Christian <[email protected]>; Xinhui Pan <[email protected]>; David Airlie <[email protected]>; Simona Vetter <[email protected]>; amd- [email protected]; [email protected]
Subject: [PATCH RFC 4/9] drm/amd/pm: Avoid open-coded use ofratelimit_statestructure's internalsThe
amdgpu_set_thermal_throttling_logging()function directly accesses theratelimit_statestructure's ->missed field, which work, but which also makes it more difficult to change this field. Therefore, make use of theratelimit_state_reset_miss()function instead of directly accessing the ->missed field.
Quoted 7 lines
Nevertheless, open-coded use of ->burst and ->interval is still permitted, for example, for runtime sysfs adjustment of these fields.
Reported-by kernel test robot <[email protected]>Signed-off-by Paul E. McKenney <[email protected]>
Feel free to take this via whatever tree makes sense for the larger series. Otherwise, let me know, and I will pick it up.
Alex
Quoted 50 lines
Cc Kenneth Feng <[email protected]>Cc Alex Deucher <[email protected]>Cc "Christian König" <[email protected]>Cc Xinhui Pan <[email protected]>Cc David Airlie <[email protected]>Cc Simona Vetter <[email protected]>Cc <[email protected]>Cc <[email protected]>--- drivers/gpu/drm/amd/pm/amdgpu_pm.c | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-)drivers/gpu/drm/amd/pm/amdgpu_pm.c +0
-1@@ -1663,7 +1663,6 @@ static ssize_t amdgpu_set_thermal_throttling_logging(struct device *dev, struct drm_device *ddev = dev_get_drvdata(dev);struct amdgpu_device *adev =drm_to_adev (ddev);long throttling_logging_interval; -unsigned long flags;int ret =0 ;ret = kstrtol(buf, 0, &throttling_logging_interval); @@ -1674,18 +1673,12 @@ static ssize_t amdgpu_set_thermal_throttling_logging(struct device *dev, return -EINVAL; if (throttling_logging_interval > 0) {drivers/gpu/drm/amd/pm/amdgpu_pm.c +0
-7-raw_spin_lock_irqsave (&adev->throttling_logging_rs.lock, flags);/* * Reset the ratelimit timer internals. * This can effectively restart the timer. */ - adev->throttling_logging_rs.interval = - (throttling_logging_interval -1 ) *HZ ; - adev->throttling_logging_rs.begin =0 ; - adev->throttling_logging_rs.printed =0 ; - adev->throttling_logging_rs.missed =0 ; - raw_spin_unlock_irqrestore(&adev->throttling_logging_rs.lock,flags);
drivers/gpu/drm/amd/pm/amdgpu_pm.c +2
-2- +ratelimit_state_reset_interval (&adev->throttling_logging_rs, + (throttling_logging_interval -1 ) *HZ );atomic_set (&adev->throttling_logging_enabled,1 ); }else {atomic_set (&adev->throttling_logging_enabled,0 ); --2.40.1
Re: [PATCH RFC 4/9] drm/amd/pm: Avoid open-coded use of ratelimit_state structure's internals
Quoted 29 lines · Alexander
[Public]
Quoted 18 lines
-----Original Message-----
From: Paul E. McKenney <[email protected]>
Sent: Thursday, April 3, 2025 5:15 PM
To: [email protected]
Cc: [email protected]; Paul E. McKenney <[email protected]>; kernel test robot <[email protected]>; Feng, Kenneth <[email protected]>; Deucher, Alexander <[email protected]>; Koenig, Christian <[email protected]>; Xinhui Pan <[email protected]>; David Airlie <[email protected]>; Simona Vetter <[email protected]>; amd- [email protected]; [email protected]
Subject: [PATCH RFC 4/9] drm/amd/pm: Avoid open-coded use ofratelimit_statestructure's internalsThe
amdgpu_set_thermal_throttling_logging()function directly accesses theratelimit_statestructure's ->missed field, which work, but which also makes it more difficult to change this field. Therefore, make use of theratelimit_state_reset_miss()function instead of directly accessing the ->missed field.Quoted 7 lines
Nevertheless, open-coded use of ->burst and ->interval is still permitted, for example, for runtime sysfs adjustment of these fields.
Reported-by kernel test robot <[email protected]>Signed-off-by Paul E. McKenney <[email protected]>
Quoted 3 lines
Acked-by Alex Deucher <[email protected]>Feel free to take this via whatever tree makes sense for the larger series. Otherwise, let me know, and I will pick it up.
Thank you, and yes, this does depend on a patch that is earlier in this series, so I will be happy to carry it.
Thanx, Paul
Quoted 54 lines
Alex
Quoted 50 lines
Cc Kenneth Feng <[email protected]>Cc Alex Deucher <[email protected]>Cc "Christian König" <[email protected]>Cc Xinhui Pan <[email protected]>Cc David Airlie <[email protected]>Cc Simona Vetter <[email protected]>Cc <[email protected]>Cc <[email protected]>--- drivers/gpu/drm/amd/pm/amdgpu_pm.c | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-)drivers/gpu/drm/amd/pm/amdgpu_pm.c +0
-1@@ -1663,7 +1663,6 @@ static ssize_t amdgpu_set_thermal_throttling_logging(struct device *dev, struct drm_device *ddev = dev_get_drvdata(dev);struct amdgpu_device *adev =drm_to_adev (ddev);long throttling_logging_interval; -unsigned long flags;int ret =0 ;ret = kstrtol(buf, 0, &throttling_logging_interval); @@ -1674,18 +1673,12 @@ static ssize_t amdgpu_set_thermal_throttling_logging(struct device *dev, return -EINVAL; if (throttling_logging_interval > 0) {drivers/gpu/drm/amd/pm/amdgpu_pm.c +0
-7-raw_spin_lock_irqsave (&adev->throttling_logging_rs.lock, flags);/* * Reset the ratelimit timer internals. * This can effectively restart the timer. */ - adev->throttling_logging_rs.interval = - (throttling_logging_interval -1 ) *HZ ; - adev->throttling_logging_rs.begin =0 ; - adev->throttling_logging_rs.printed =0 ; - adev->throttling_logging_rs.missed =0 ; - raw_spin_unlock_irqrestore(&adev->throttling_logging_rs.lock,flags);
drivers/gpu/drm/amd/pm/amdgpu_pm.c +2
-2- +ratelimit_state_reset_interval (&adev->throttling_logging_rs, + (throttling_logging_interval -1 ) *HZ );atomic_set (&adev->throttling_logging_enabled,1 ); }else {atomic_set (&adev->throttling_logging_enabled,0 ); --2.40.1
RFC PATCH 5/9 ratelimit: Convert the ->missed field to atomic_t
In reply to Paul E. McKenneyThe ratelimit_state structure's ->missed field is sometimes incremented locklessly, and it would be good to avoid lost counts. This is also needed to count the number of misses due to try-lock failure. Therefore, convert the rratelimit_state structure's ->missed field to atomic_t.
--- include/linux/ratelimit.h | 9 +++------ include/linux/ratelimit_types.h | 2 +- lib/ratelimit.c | 2 +- 3 files changed, 5 insertions(+), 8 deletions(-)
include/linux/ratelimit.h +3 -6
@@ -24,20 +24,17 @@ static inline void ratelimit_default_init(struct ratelimit_state *rs)static inline void ratelimit_state_inc_miss (struct ratelimit_state *rs) { - rs->missed++; +atomic_inc (&rs->missed); }static inline int ratelimit_state_get_miss (struct ratelimit_state *rs) { -return rs->missed; +return atomic_read (&rs->missed); }static inline int ratelimit_state_reset_miss (struct ratelimit_state *rs) { -int ret = rs->missed; - - rs->missed =0 ; -return ret; +return atomic_xchg_relaxed (&rs->missed,0 ); }static inline void ratelimit_state_reset_interval (struct ratelimit_state *rs,int interval_init)
include/linux/ratelimit_types.h +1 -1
@@ -18,7 +18,7 @@ struct ratelimit_state {int interval;int burst;int printed; -int missed; +atomic_t missed;unsigned int flags;unsigned long begin; };
lib/ratelimit.c +1 -1
@@ -66,7 +66,7 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func) rs->printed++; ret =1 ; }else { - rs->missed++; +ratelimit_state_inc_miss (rs); ret =0 ; }raw_spin_unlock_irqrestore (&rs->lock, flags);
Signature · 1 line
2.40.1
RFC PATCH 6/9 ratelimit: Count misses due to lock contention
In reply to Paul E. McKenneyThe ___ratelimit() function simply returns zero ("do ratelimiting") if the trylock fails, but does not adjust the ->missed field. This means that the resulting dropped printk()s are dropped silently, which could seriously confuse people trying to do console-log-based debugging. Therefore, increment the ->missed field upon trylock failure.
--- lib/ratelimit.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-)
lib/ratelimit.c +3 -1
@@ -44,8 +44,10 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func)* in addition to the one that will be printed by * the entity that is holding the lock already: */ -if (!raw_spin_trylock_irqsave (&rs->lock, flags)) +if (!raw_spin_trylock_irqsave (&rs->lock, flags)) { +ratelimit_state_inc_miss (rs);return 0 ; + }if (!rs->begin) rs->begin = jiffies;
Signature · 1 line
2.40.1
RFC PATCH 7/9 ratelimit: Avoid jiffies=0 special case
In reply to Paul E. McKenneyThe ___ratelimit() function special-cases the jiffies-counter value of zero as "uninitialized". This works well on 64-bit systems, where the jiffies counter is not going to return to zero for more than half a billion years on systems with HZ=1000, but similar 32-bit systems take less than 50 days to wrap the jiffies counter. And although the consequences of wrapping the jiffies counter seem to be limited to minor confusion on the duration of the rate-limiting interval that happens to end at time zero, it is almost no work to avoid this confusion.
Therefore, introduce a RATELIMIT_INITIALIZED bit to the ratelimit_state structure's ->flags field so that a ->begin value of zero is no longer special.
--- include/linux/ratelimit.h | 2 +- include/linux/ratelimit_types.h | 1 + lib/ratelimit.c | 4 +++- 3 files changed, 5 insertions(+), 2 deletions(-)
include/linux/ratelimit.h +1 -1
@@ -43,7 +43,7 @@ static inline void ratelimit_state_reset_interval(struct ratelimit_state *rs, inraw_spin_lock_irqsave (&rs->lock, flags); rs->interval = interval_init; - rs->begin =0 ; + rs->flags &= ~RATELIMIT_INITIALIZED ; rs->printed =0 ;ratelimit_state_reset_miss (rs);raw_spin_unlock_irqrestore (&rs->lock, flags);
include/linux/ratelimit_types.h +1 -0
@@ -11,6 +11,7 @@/* issue num suppressed message on exit */ #define RATELIMIT_MSG_ON_RELEASE BIT(0) +#define RATELIMIT_INITIALIZED BIT(1)struct ratelimit_state {raw_spinlock_t lock;/* protect the state */
lib/ratelimit.c +3 -1
@@ -49,8 +49,10 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func)return 0 ; } -if (!rs->begin) +if (!(rs->flags &RATELIMIT_INITIALIZED )) { rs->begin = jiffies; + rs->flags |=RATELIMIT_INITIALIZED ; + }if (time_is_before_jiffies (rs->begin + interval)) {int m =ratelimit_state_reset_miss (rs);
Signature · 1 line
2.40.1
RFC PATCH 8/9 ratelimit: Reduce ratelimit's false-positive misses
In reply to Paul E. McKenneyThe current ratelimit implementation can suffer from false-positive misses. That is, ___ratelimit() might return zero (causing the caller to invoke rate limiting, for example, by dropping printk()s) even when the current burst has not yet been consumed. This happens when one CPU holds a given ratelimit structure's lock and some other CPU concurrently invokes ___ratelimit(). The fact that the lock is a raw irq-disabled spinlock might make low-contention trylock failure seem unlikely, but vCPU preemption, NMIs, and firmware interrupts can greatly extend the trylock-failure window.
Avoiding these false-positive misses is especially important when correlating console logged hardware failures with other information.
Therefore, instead of attempting to acquire the lock on each call to ___ratelimit(), construct a lockless fastpath and only acquire the lock when retriggering (for the next burst) or when resynchronizing (due to either a long idle period or due to ratelimiting having been disabled). This reduces the number of lock-hold periods that can be extended by vCPU preemption, NMIs and firmware interrupts, but also means that these extensions must be of much longer durations (generally moving from milliseconds to seconds) before they can result in false-positive drops.
In addition, the lockless fastpath gets a 10-20% speedup compared to the old fully locked code on my x86 laptop. Your mileage will of course vary depending on your hardware, workload, and configuration.
[ paulmck: Apply feedback from Dan Carpenter. ]
--- include/linux/ratelimit.h | 2 +- include/linux/ratelimit_types.h | 3 +- lib/ratelimit.c | 176 +++++++++++++++++++++++++------- 3 files changed, 144 insertions(+), 37 deletions(-)
include/linux/ratelimit.h +1 -1
@@ -44,7 +44,7 @@ static inline void ratelimit_state_reset_interval(struct ratelimit_state *rs, inraw_spin_lock_irqsave (&rs->lock, flags); rs->interval = interval_init; rs->flags &= ~RATELIMIT_INITIALIZED ; - rs->printed =0 ; +atomic_set (&rs->rs_n_left, rs->burst);ratelimit_state_reset_miss (rs);raw_spin_unlock_irqrestore (&rs->lock, flags); }
include/linux/ratelimit_types.h +2 -1
@@ -18,7 +18,7 @@ struct ratelimit_state {int interval;int burst; -int printed; +atomic_t rs_n_left;atomic_t missed;unsigned int flags;unsigned long begin;
@@ -28,6 +28,7 @@ struct ratelimit_state { .lock = __RAW_SPIN_LOCK_UNLOCKED(name.lock), \ .interval = interval_init, \ .burst = burst_init, \ + .rs_n_left = ATOMIC_INIT(burst_init), \ .flags = flags_init, \ }
lib/ratelimit.c +141 -35
196 lines not on this page · raw message
Signature · 1 line
2.40.1
Re: [PATCH RFC 8/9] ratelimit: Reduce ratelimit's false-positive misses
Quoted 23 lines · Paul E. McKenney
The current ratelimit implementation can suffer from false-positive misses. That is,
___ratelimit()might return zero (causing the caller to invoke rate limiting, for example, by dropping printk()s) even when the current burst has not yet been consumed. This happens when one CPU holds a given ratelimit structure's lock and some other CPU concurrently invokes___ratelimit(). The fact that the lock is a raw irq-disabled spinlock might make low-contention trylock failure seem unlikely, but vCPU preemption, NMIs, and firmware interrupts can greatly extend the trylock-failure window.Avoiding these false-positive misses is especially important when correlating console logged hardware failures with other information.
Therefore, instead of attempting to acquire the lock on each call to
___ratelimit(), construct a lockless fastpath and only acquire the lock when retriggering (for the next burst) or when resynchronizing (due to either a long idle period or due to ratelimiting having been disabled). This reduces the number of lock-hold periods that can be extended by vCPU preemption, NMIs and firmware interrupts, but also means that these extensions must be of much longer durations (generally moving from milliseconds to seconds) before they can result in false-positive drops.
Quoted 4 lines
In addition, the lockless fastpath gets a 10-20% speedup compared to the old fully locked code on my x86 laptop. Your mileage will of course vary depending on your hardware, workload, and configuration.
First a nit: the func returns an int with 1 or 0, perhaps one extra patch to make it bool can be squeezed in here?
One of the previous patches fixes a bug on 32-bit archs.
Maybe it will sound silly, but my suggestion below hinges on it: is this patchset written with 32-bit kernels in mind?
If not, I wonder if the 32-bit stuff can stay with the locked variant and the 64-bit can get a lockless fast path which issues 8-byte cmpxchg on the event count + (to be introduced) sequence counter.
I think that would be significantly easier to reason about as it would guarantee no changes are made if someone is reconfiguring the struct, while providing the same win from single-threaded standpoint.
I think you know what you mean, but just in case here is a pseudocode draft of the fast path:
#define RATELIMIT_NEED_INIT BIT(31)
#define RATELIMIT_IN_FLUX BIT(0)
struct ratelimit_state_change {
int events_left;
unsigned int seq;
};struct ratelimit_state {
raw_spinlock_t lock;
int interval;
int burst;
int missed;
struct ratelimit_state_change rsc;
unsigned long begin;
};seq = READ_ONCE(rs->rsc.seq);smp_rmb();
if (seq & (RATELIMIT_NEED_INIT | RATELIMIT_IN_FLUX))
goto bad;
begin = READ_ONCE(rs->begin);
burst = READ_ONCE(rs->burst);
interval = READ_ONCE(rs->interval);events_left = READ_ONCE(rs->rsc.events_left;smp_rmb();
/* checks if we can cmpxchg go here */
....
/* now the work */
struct ratelimit_state_change new = {
.events_left = events_left - 1;
.seq = seq;
}if (try_cmpxchg64_relaxed(&rs->rsc, ......)) {
return true; /* succeeded */
}
/* ... retry based on what we got, most likely only ->events_left has changed */
On the stock kernel the struct is 32 bytes. I'm combining flags and the new seq field to avoid growing it.
This does cut down on available seq size, but it should be plenty as is. This also means the slowpath will have to be careful to not blindly ++ it to not walk into flags, but I think that's easier to handle that races. ;)
That said this is merely a suggestion, I'm not going to push for it.
I recognize this swaps atomic_dec into an cmpxchg loop which in principle will have worse throughput in face of multiple CPUs messing with it. However, the fast path in both your and my variant issues loads prior to the atomic op which already do most of the damage, so I don't think this bit matters that much.
Signature · 1 line
Mateusz Guzik <mjguzik gmail.com>
Re: [PATCH RFC 8/9] ratelimit: Reduce ratelimit's false-positive misses
Quoted 24 lines · Mateusz Guzik
Quoted 23 lines · Paul E. McKenney
The current ratelimit implementation can suffer from false-positive misses. That is,
___ratelimit()might return zero (causing the caller to invoke rate limiting, for example, by dropping printk()s) even when the current burst has not yet been consumed. This happens when one CPU holds a given ratelimit structure's lock and some other CPU concurrently invokes___ratelimit(). The fact that the lock is a raw irq-disabled spinlock might make low-contention trylock failure seem unlikely, but vCPU preemption, NMIs, and firmware interrupts can greatly extend the trylock-failure window.Avoiding these false-positive misses is especially important when correlating console logged hardware failures with other information.
Therefore, instead of attempting to acquire the lock on each call to
___ratelimit(), construct a lockless fastpath and only acquire the lock when retriggering (for the next burst) or when resynchronizing (due to either a long idle period or due to ratelimiting having been disabled). This reduces the number of lock-hold periods that can be extended by vCPU preemption, NMIs and firmware interrupts, but also means that these extensions must be of much longer durations (generally moving from milliseconds to seconds) before they can result in false-positive drops.
Quoted 3 lines
Quoted 3 lines
In addition, the lockless fastpath gets a 10-20% speedup compared to the old fully locked code on my x86 laptop. Your mileage will of course vary depending on your hardware, workload, and configuration.
Thank you for digging into this!!!
Quoted 2 lines
First a nit: the func returns an int with 1 or 0, perhaps one extra patch to make it bool can be squeezed in here?
I can do that. Patch below.
Quoted 4 lines
One of the previous patches fixes a bug on 32-bit archs.
Maybe it will sound silly, but my suggestion below hinges on it: is this patchset written with 32-bit kernels in mind?
Yes, that bug fix is reflected in the lockless-fastpath patch. It no longer treats ->begin==0 as special. The reason that this is 32-bit specific is that at 1000HZ, a 32-bit counter wraps every 50 days or so, which is well within the range of possible uptimes. Wrapping for 64-bit counter takes way longer.
Quoted 50 lines
If not, I wonder if the 32-bit stuff can stay with the locked variant and the 64-bit can get a lockless fast path which issues 8-byte cmpxchg on the event count + (to be introduced) sequence counter.
I think that would be significantly easier to reason about as it would guarantee no changes are made if someone is reconfiguring the struct, while providing the same win from single-threaded standpoint.
I think you know what you mean, but just in case here is a pseudocode draft of the fast path:
#define
RATELIMIT_NEED_INITBIT(31)
#defineRATELIMIT_IN_FLUXBIT(0)struct ratelimit_state_change { int events_left; unsigned int seq; };struct ratelimit_state { raw_spinlock_t lock; int interval; int burst; int missed; struct ratelimit_state_change rsc; unsigned long begin; };seq =
READ_ONCE(rs->rsc.seq);smp_rmb();
if (seq & (RATELIMIT_NEED_INIT|RATELIMIT_IN_FLUX))goto bad;begin =
READ_ONCE(rs->begin);
burst =READ_ONCE(rs->burst);
interval =READ_ONCE(rs->interval);events_left= READ_ONCE(rs->rsc.events_left;smp_rmb();
/* checks if we can cmpxchg go here */
....
/* now the work */struct ratelimit_state_change new = { .events_left = events_left - 1; .seq = seq; }if (
try_cmpxchg64_relaxed(&rs->rsc, ......)) {
return true; /* succeeded */
}
/* ... retry based on what we got, most likely only ->events_left has changed */
Quoted 7 lines
On the stock kernel the struct is 32 bytes. I'm combining flags and the new seq field to avoid growing it.
This does cut down on available seq size, but it should be plenty as is. This also means the slowpath will have to be careful to not blindly ++ it to not walk into flags, but I think that's easier to handle that races. ;)
In theory, something sort of like this that used a 16-byte cmpxchg and packed the ->begin, ->rs_n_left, and ->flags fields together could simplify this quite a bit. But not every system has a 16-byte cmpxchg on the on hand and packing into 8 bytes (let alone a 32-bit system's 4 bytes) would require painful tradeoffs. But in practice...
Quoted 7 lines
That said this is merely a suggestion, I'm not going to push for it.
I recognize this swaps
atomic_decinto an cmpxchg loop which in principle will have worse throughput in face of multiple CPUs messing with it. However, the fast path in both your and my variant issues loads prior to the atomic op which already do most of the damage, so I don't think this bit matters that much.
...as you say, the full-load throughput of cmpxchg() is lacking compared to that of atomic_dec_return(). And large systems might have serious ___ratelimit() call rates. Worse yet, the forward-progress properties of cmpxchg() are lacking compared to those of atomic_dec_return(), so I am not at all sold on this packing approach, even for systems providing 16-byte cmpxchg operations.
Yes, if a given ratelimit_state structure is mostly throttling, the load-only fastpath is there, but the quadratic overload behavior of cmpxchg() would apply during the non-throttling phases.
Never say never, of course, but we would need to see real issues with the atomic_dec_return() approach before it would make sense to take on the packing approach.
Thanx, Paul
Quoted 2 lines
Signature · 1 line
Mateusz Guzik <mjguzik gmail.com>
------------------------------------------------------------------------
include/linux/ratelimit_types.h +1 -1
@@ -43,7 +43,7 @@ struct ratelimit_state {struct ratelimit_state name = \ RATELIMIT_STATE_INIT(name, interval_init, burst_init) \ -extern int ___ratelimit (struct ratelimit_state *rs,const char *func); +extern bool ___ratelimit (struct ratelimit_state *rs,const char *func);#define __ratelimit (state) ___ratelimit(state, __func__)#endif /* _LINUX_RATELIMIT_TYPES_H */
lib/ratelimit.c +1 -1
@@ -24,7 +24,7 @@* 0 means callbacks will be suppressed. * 1 means go ahead and do it. */ -int ___ratelimit (struct ratelimit_state *rs,const char *func) +bool ___ratelimit (struct ratelimit_state *rs,const char *func) {unsigned long begin;int burst =READ_ONCE (rs->burst);
Re: [PATCH RFC 8/9] ratelimit: Reduce ratelimit's false-positive misses
Quoted 30 lines · Paul E. McKenney
Quoted 24 lines · Mateusz Guzik
Quoted 23 lines · Paul E. McKenney
The current ratelimit implementation can suffer from false-positive misses. That is,
___ratelimit()might return zero (causing the caller to invoke rate limiting, for example, by dropping printk()s) even when the current burst has not yet been consumed. This happens when one CPU holds a given ratelimit structure's lock and some other CPU concurrently invokes___ratelimit(). The fact that the lock is a raw irq-disabled spinlock might make low-contention trylock failure seem unlikely, but vCPU preemption, NMIs, and firmware interrupts can greatly extend the trylock-failure window.Avoiding these false-positive misses is especially important when correlating console logged hardware failures with other information.
Therefore, instead of attempting to acquire the lock on each call to
___ratelimit(), construct a lockless fastpath and only acquire the lock when retriggering (for the next burst) or when resynchronizing (due to either a long idle period or due to ratelimiting having been disabled). This reduces the number of lock-hold periods that can be extended by vCPU preemption, NMIs and firmware interrupts, but also means that these extensions must be of much longer durations (generally moving from milliseconds to seconds) before they can result in false-positive drops.Quoted 3 lines
Quoted 3 lines
In addition, the lockless fastpath gets a 10-20% speedup compared to the old fully locked code on my x86 laptop. Your mileage will of course vary depending on your hardware, workload, and configuration.
Quoted 7 lines
Thank you for digging into this!!!
Quoted 2 lines
First a nit: the func returns an int with 1 or 0, perhaps one extra patch to make it bool can be squeezed in here?
I can do that. Patch below.
thanks
Quoted 69 lines
Quoted 4 lines
One of the previous patches fixes a bug on 32-bit archs.
Maybe it will sound silly, but my suggestion below hinges on it: is this patchset written with 32-bit kernels in mind?
Yes, that bug fix is reflected in the lockless-fastpath patch. It no longer treats ->begin==0 as special. The reason that this is 32-bit specific is that at 1000HZ, a 32-bit counter wraps every 50 days or so, which is well within the range of possible uptimes. Wrapping for 64-bit counter takes way longer.
Quoted 50 lines
If not, I wonder if the 32-bit stuff can stay with the locked variant and the 64-bit can get a lockless fast path which issues 8-byte cmpxchg on the event count + (to be introduced) sequence counter.
I think that would be significantly easier to reason about as it would guarantee no changes are made if someone is reconfiguring the struct, while providing the same win from single-threaded standpoint.
I think you know what you mean, but just in case here is a pseudocode draft of the fast path:
#define
RATELIMIT_NEED_INITBIT(31)
#defineRATELIMIT_IN_FLUXBIT(0)struct ratelimit_state_change { int events_left; unsigned int seq; };struct ratelimit_state { raw_spinlock_t lock; int interval; int burst; int missed; struct ratelimit_state_change rsc; unsigned long begin; };seq =
READ_ONCE(rs->rsc.seq);smp_rmb();
if (seq & (RATELIMIT_NEED_INIT|RATELIMIT_IN_FLUX))goto bad;begin =
READ_ONCE(rs->begin);
burst =READ_ONCE(rs->burst);
interval =READ_ONCE(rs->interval);events_left= READ_ONCE(rs->rsc.events_left;smp_rmb();
/* checks if we can cmpxchg go here */
....
/* now the work */struct ratelimit_state_change new = { .events_left = events_left - 1; .seq = seq; }if (
try_cmpxchg64_relaxed(&rs->rsc, ......)) {
return true; /* succeeded */
}
/* ... retry based on what we got, most likely only ->events_left has changed */Quoted 7 lines
On the stock kernel the struct is 32 bytes. I'm combining flags and the new seq field to avoid growing it.
This does cut down on available seq size, but it should be plenty as is. This also means the slowpath will have to be careful to not blindly ++ it to not walk into flags, but I think that's easier to handle that races. ;)
Quoted 6 lines
In theory, something sort of like this that used a 16-byte cmpxchg and packed the ->begin, ->rs_n_left, and ->flags fields together could simplify this quite a bit. But not every system has a 16-byte cmpxchg on the on hand and packing into 8 bytes (let alone a 32-bit system's 4 bytes) would require painful tradeoffs. But in practice...
well cmpxchg16b has atrocious performance and I would not recommend ;)
Quoted 8 lines
Quoted 7 lines
That said this is merely a suggestion, I'm not going to push for it.
I recognize this swaps
atomic_decinto an cmpxchg loop which in principle will have worse throughput in face of multiple CPUs messing with it. However, the fast path in both your and my variant issues loads prior to the atomic op which already do most of the damage, so I don't think this bit matters that much.
Quoted 7 lines
...as you say, the full-load throughput of
cmpxchg()is lacking compared to that ofatomic_dec_return(). And large systems might have serious___ratelimit()call rates. Worse yet, the forward-progress properties ofcmpxchg()are lacking compared to those ofatomic_dec_return(), so I am not at all sold on this packing approach, even for systems providing 16-byte cmpxchg operations.
Well in my proposal this is 8-byte cmpxchg, not 16 with the sequence counter validating the rest of the state has not changed.
Quoted 4 lines
Yes, if a given
ratelimit_statestructure is mostly throttling, the load-only fastpath is there, but the quadratic overload behavior ofcmpxchg()would apply during the non-throttling phases.
It is indeed non-ideal, but if you really need good perf here, then I would argue literally just one instance of the counter is already bad.
Quoted 4 lines
Never say never, of course, but we would need to see real issues with the
atomic_dec_return()approach before it would make sense to take on the packing approach.
I claim my proposal is simpler to reason about as you get an invariant nobody changes the event count from under you and they always operate on a fully populated state.
All that said, this was a suggestion on the side which requires work to implement.
On the other hand your variant is already written and I'm by no means trying to block it. I am not in position to ACK it either and afaics ratelimit is virtually unmaintained anyway. I guess it's your call what to do with it.
Re: [PATCH RFC 8/9] ratelimit: Reduce ratelimit's false-positive misses
Quoted 110 lines · Mateusz Guzik
Quoted 30 lines · Paul E. McKenney
Quoted 24 lines · Mateusz Guzik
Quoted 23 lines · Paul E. McKenney
The current ratelimit implementation can suffer from false-positive misses. That is,
___ratelimit()might return zero (causing the caller to invoke rate limiting, for example, by dropping printk()s) even when the current burst has not yet been consumed. This happens when one CPU holds a given ratelimit structure's lock and some other CPU concurrently invokes___ratelimit(). The fact that the lock is a raw irq-disabled spinlock might make low-contention trylock failure seem unlikely, but vCPU preemption, NMIs, and firmware interrupts can greatly extend the trylock-failure window.Avoiding these false-positive misses is especially important when correlating console logged hardware failures with other information.
Therefore, instead of attempting to acquire the lock on each call to
___ratelimit(), construct a lockless fastpath and only acquire the lock when retriggering (for the next burst) or when resynchronizing (due to either a long idle period or due to ratelimiting having been disabled). This reduces the number of lock-hold periods that can be extended by vCPU preemption, NMIs and firmware interrupts, but also means that these extensions must be of much longer durations (generally moving from milliseconds to seconds) before they can result in false-positive drops.Quoted 3 lines
Quoted 3 lines
In addition, the lockless fastpath gets a 10-20% speedup compared to the old fully locked code on my x86 laptop. Your mileage will of course vary depending on your hardware, workload, and configuration.
Quoted 7 lines
Thank you for digging into this!!!
Quoted 2 lines
First a nit: the func returns an int with 1 or 0, perhaps one extra patch to make it bool can be squeezed in here?
I can do that. Patch below.
thanks
Quoted 69 lines
Quoted 4 lines
One of the previous patches fixes a bug on 32-bit archs.
Maybe it will sound silly, but my suggestion below hinges on it: is this patchset written with 32-bit kernels in mind?
Yes, that bug fix is reflected in the lockless-fastpath patch. It no longer treats ->begin==0 as special. The reason that this is 32-bit specific is that at 1000HZ, a 32-bit counter wraps every 50 days or so, which is well within the range of possible uptimes. Wrapping for 64-bit counter takes way longer.
Quoted 57 lines
If not, I wonder if the 32-bit stuff can stay with the locked variant and the 64-bit can get a lockless fast path which issues 8-byte cmpxchg on the event count + (to be introduced) sequence counter.
I think that would be significantly easier to reason about as it would guarantee no changes are made if someone is reconfiguring the struct, while providing the same win from single-threaded standpoint.
I think you know what you mean, but just in case here is a pseudocode draft of the fast path:
#define
RATELIMIT_NEED_INITBIT(31)
#defineRATELIMIT_IN_FLUXBIT(0)struct ratelimit_state_change { int events_left; unsigned int seq; };struct ratelimit_state { raw_spinlock_t lock; int interval; int burst; int missed; struct ratelimit_state_change rsc; unsigned long begin; };seq =
READ_ONCE(rs->rsc.seq);smp_rmb();
if (seq & (RATELIMIT_NEED_INIT|RATELIMIT_IN_FLUX))goto bad;begin =
READ_ONCE(rs->begin);
burst =READ_ONCE(rs->burst);
interval =READ_ONCE(rs->interval);events_left= READ_ONCE(rs->rsc.events_left;smp_rmb();
/* checks if we can cmpxchg go here */
....
/* now the work */struct ratelimit_state_change new = { .events_left = events_left - 1; .seq = seq; }if (
try_cmpxchg64_relaxed(&rs->rsc, ......)) {
return true; /* succeeded */
}
/* ... retry based on what we got, most likely only ->events_left has changed */On the stock kernel the struct is 32 bytes. I'm combining flags and the new seq field to avoid growing it.
This does cut down on available seq size, but it should be plenty as is. This also means the slowpath will have to be careful to not blindly ++ it to not walk into flags, but I think that's easier to handle that races. ;)
Quoted 7 lines
Quoted 5 lines
In theory, something sort of like this that used a 16-byte cmpxchg and packed the ->begin, ->rs_n_left, and ->flags fields together could simplify this quite a bit. But not every system has a 16-byte cmpxchg on the on hand and packing into 8 bytes (let alone a 32-bit system's 4 bytes) would require painful tradeoffs. But in practice...
well cmpxchg16b has atrocious performance and I would not recommend ;)
Atrocious even compared to a spinlock round trip?
Quoted 15 lines
Quoted 8 lines
Quoted 7 lines
That said this is merely a suggestion, I'm not going to push for it.
I recognize this swaps
atomic_decinto an cmpxchg loop which in principle will have worse throughput in face of multiple CPUs messing with it. However, the fast path in both your and my variant issues loads prior to the atomic op which already do most of the damage, so I don't think this bit matters that much.Quoted 6 lines
...as you say, the full-load throughput of
cmpxchg()is lacking compared to that ofatomic_dec_return(). And large systems might have serious___ratelimit()call rates. Worse yet, the forward-progress properties ofcmpxchg()are lacking compared to those ofatomic_dec_return(), so I am not at all sold on this packing approach, even for systems providing 16-byte cmpxchg operations.
Quoted 2 lines
Well in my proposal this is 8-byte cmpxchg, not 16 with the sequence counter validating the rest of the state has not changed.
Let me make sure that I understand what you are proposing.
Is the idea is to identify the kthread that will reset for the next interval? Or to avoid a race where a kthread tests ->rs_n_left before such a reset, but does its atomic_dec_return() after that same reset?
Or am I missing your point completely? ;-)
Quoted 6 lines
Quoted 3 lines
Yes, if a given
ratelimit_statestructure is mostly throttling, the load-only fastpath is there, but the quadratic overload behavior ofcmpxchg()would apply during the non-throttling phases.It is indeed non-ideal, but if you really need good perf here, then I would argue literally just one instance of the counter is already bad.
I do not believe that we need great performance, but it would be good to avoid quadratic overhead on systems with hundreds (let alone thousands) of CPUs.
Quoted 7 lines
Quoted 3 lines
Never say never, of course, but we would need to see real issues with the
atomic_dec_return()approach before it would make sense to take on the packing approach.I claim my proposal is simpler to reason about as you get an invariant nobody changes the event count from under you and they always operate on a fully populated state.
Me, I was being satified with the throttling being exact in the long term and being almost always exact on a per-interval basis. ;-)
Quoted 7 lines
All that said, this was a suggestion on the side which requires work to implement.
On the other hand your variant is already written and I'm by no means trying to block it. I am not in position to ACK it either and afaics ratelimit is virtually unmaintained anyway. I guess it's your call what to do with it.
Again, thank you for looking this over!
You have given me several ideas for improvement, including your cmpxchg() approach to guarantee exact per-interval throttling (if that was your intent), my cmpxchg16b() to go completely lockless with that same guarantee, and a combination of atomic_dec_return() and cmpxchg16b() to go completely lockless with linear (instead of quadratic) cmpxchg16b() performance and long-term exact throttling.
Can't argue with that! ;-)
Thanx, Paul
Re: [PATCH RFC 8/9] ratelimit: Reduce ratelimit's false-positive misses
In reply to Paul E. McKenneyQuoted 115 lines · Paul E. McKenney
The current ratelimit implementation can suffer from false-positive misses. That is,
___ratelimit()might return zero (causing the caller to invoke rate limiting, for example, by dropping printk()s) even when the current burst has not yet been consumed. This happens when one CPU holds a given ratelimit structure's lock and some other CPU concurrently invokes___ratelimit(). The fact that the lock is a raw irq-disabled spinlock might make low-contention trylock failure seem unlikely, but vCPU preemption, NMIs, and firmware interrupts can greatly extend the trylock-failure window.Avoiding these false-positive misses is especially important when correlating console logged hardware failures with other information.
Therefore, instead of attempting to acquire the lock on each call to
___ratelimit(), construct a lockless fastpath and only acquire the lock when retriggering (for the next burst) or when resynchronizing (due to either a long idle period or due to ratelimiting having been disabled). This reduces the number of lock-hold periods that can be extended by vCPU preemption, NMIs and firmware interrupts, but also means that these extensions must be of much longer durations (generally moving from milliseconds to seconds) before they can result in false-positive drops.In addition, the lockless fastpath gets a 10-20% speedup compared to the old fully locked code on my x86 laptop. Your mileage will of course vary depending on your hardware, workload, and configuration.
[ paulmck: Apply feedback from Dan Carpenter. ]
lib/ratelimit.c +53
-19@@ -26,55 +26,161 @@*/ int ___ratelimit (struct ratelimit_state *rs,const char *func) { -/* Paired with WRITE_ONCE() in .proc_handler(). -* Changing two values seperately could be inconsistent -* and some message could be lost. (See: net_ratelimit_state). -*/ -int interval =READ_ONCE (rs->interval); +unsigned long begin;int burst =READ_ONCE (rs->burst); +int delta =0 ;unsigned long flags; -int ret; - -if (!interval) -return 1 ; +bool gotlock = false; +bool initialized; +int interval =READ_ONCE (rs->interval); +unsigned long j; +int n_left;/* -* If we contend on this state's lock then almost -* by definition we are too busy to print a message, -* in addition to the one that will be printed by -* the entity that is holding the lock already: +* If the burst or interval settings mark this ratelimit_state +* structure as disabled, then clear the RATELIMIT_INITIALIZED bit +* in ->flags to force resetting of the ratelimiting interval when +* this ratelimit_state structure is next re-enabled. */ -if (!raw_spin_trylock_irqsave (&rs->lock, flags)) { -ratelimit_state_inc_miss (rs); -return 0 ; +if (burst <=0 || interval <=0 ) { +if ((READ_ONCE (rs->flags) &RATELIMIT_INITIALIZED ) && +raw_spin_trylock_irqsave (&rs->lock, flags)) { +if (READ_ONCE (rs->flags) &RATELIMIT_INITIALIZED ) +smp_store_release (&rs->flags, rs->flags & ~RATELIMIT_INITIALIZED ); +raw_spin_unlock_irqrestore (&rs->lock, flags); + } +return true; } -if (!(rs->flags &RATELIMIT_INITIALIZED )) { - rs->begin = jiffies; - rs->flags |=RATELIMIT_INITIALIZED ; +/* +* If this structure has just now been ratelimited, but not yet +* reset for the next rate-limiting interval, take an early and +* low-cost exit. +*/ +if (atomic_read_acquire (&rs->rs_n_left) <=0 )/* Pair with release. */ + goto limited; + +/* +* If this structure is marked as initialized and has been +* recently used, pick up its ->begin field. Otherwise, pick up +* the current time and attempt to re-initialized the structure. +*/ + j = jiffies; + initialized =smp_load_acquire (&rs->flags) &RATELIMIT_INITIALIZED ;/* Pair with release. */ +if (initialized) { + begin =READ_ONCE (rs->begin); + }else { +/* +* Uninitialized or long idle, so reset ->begin and +* mark initialized. If we fail to acquire the lock, +* let the lock holder do the work. +*/ + begin = j; +if (raw_spin_trylock_irqsave (&rs->lock, flags)) { +if (!(READ_ONCE (rs->flags) &RATELIMIT_INITIALIZED )) { + begin = jiffies; + j = begin; +WRITE_ONCE (rs->begin, begin); +smp_store_release (&rs->flags,/* Pair with acquire. */ + rs->flags |RATELIMIT_INITIALIZED ); + initialized = true; + } +raw_spin_unlock_irqrestore (&rs->lock, flags);
lib/ratelimit.c line 96
Quoted 10 lines
lib/ratelimit.c +6
-2+ } } -if (time_is_before_jiffies (rs->begin + interval)) { -int m =ratelimit_state_reset_miss (rs); +/* +* If this structure is still in the interval in which has +* already hit the rate limit, take an early and low-cost exit. +*/ +if (initialized &&time_before (begin -2 * interval, j) &&time_before (j, begin))
How do we know that the previous interval was ratelimited, please? I have to admit that I still do not fully get the new logic ;-)
Quoted 28 lines
lib/ratelimit.c +22
-5+ goto limited; -if (m) { -if (!(rs->flags &RATELIMIT_MSG_ON_RELEASE )) { -printk_deferred (KERN_WARNING -"%s: %d callbacks suppressed\n" , func, m); - } +/* +* Register another request, and take an early (but not low-cost) +* exit if rate-limiting just nowcame into effect. +*/ + n_left =atomic_dec_return (&rs->rs_n_left); +if (n_left <0 ) + goto limited;/* Just now started ratelimiting. */ +if (n_left >0 ) { +/* +* Otherwise, there is not yet any rate limiting for the +* current interval, and furthermore there is at least one +* last count remaining. But check to see if initialization +* is required or if we have run off the end of the interval +* without rate limiting having been imposed. Either way, +* we eventually return @true to tell our caller to go ahead. +*/ +if (initialized && +time_before (begin - interval, j) &&time_before (j, begin + interval)) +return true;/* Nothing special to do. */ +if (!raw_spin_trylock_irqsave (&rs->lock, flags)) +return true;/* Let lock holder do special work. */
lib/ratelimit.c line 129
Quoted 10 lines
lib/ratelimit.c +10
-0+ interval =READ_ONCE (rs->interval); + begin = rs->begin; + initialized =smp_load_acquire (&rs->flags) &RATELIMIT_INITIALIZED ; +if (interval <=0 || + (initialized && +time_before (begin - interval, j) &&time_before (j, begin + interval))) { +/* +* Someone else beat us to the special work, +* so release the lock and return. +*/
I am confused by the comment. IMHO, we are here when we do not need to reset the interval. It happens either when the interval is infinite (interval < 0) or when we are still in the currrent interval and more calls are allowed (n_left > 0).
Quoted 15 lines
lib/ratelimit.c +6
-7+raw_spin_unlock_irqrestore (&rs->lock, flags); +return true; } - rs->begin = jiffies; - rs->printed =0 ; + +/* We have the lock and will do initialization. */ + gotlock = true; + delta =-1 ; } -if (burst && burst > rs->printed) { - rs->printed++; - ret =1 ; - }else { -ratelimit_state_inc_miss (rs);
lib/ratelimit.c line 146
Quoted 10 lines
lib/ratelimit.c +9
-1- ret =0 ; +if (!gotlock) { +/* +* We get here if we got the last count (n_left == 0), +* so that rate limiting is in effect for the next caller. +* We will return @true to tell our caller to go ahead, +* but first we acquire the lock and set things up for +* the next rate-limiting interval. +*/ +raw_spin_lock_irqsave (&rs->lock, flags);
I do not have a good feeling about this. It could cause a deadlock in panic() when other CPUs are stopped or in NMI. And the ratelimit used to be "safe" in this context.
Honestly, the new code is a bit hard to follow for me. The original logic was more strightforward. I wonder if we could reduce the problem "just" by checking the remaining allowed count.
I mean to introduce the atomic rs_n_left and check it when it is not possible to take the lock. But keep most of the current logic. Something like:
From 1df6d1a79645dd5c0ad7cef7c94b802de202057e Mon Sep 17 00:00:00 2001
From: Petr Mladek <[email protected]>
Date: Tue, 8 Apr 2025 16:00:22 +0200
Subject: [PATCH 1/2] ratelimit: Reduce ratelimit's false-positive
(alternative)
--- include/linux/ratelimit.h | 2 +- include/linux/ratelimit_types.h | 2 +- lib/ratelimit.c | 51 ++++++++++++++++++++++++--------- 3 files changed, 40 insertions(+), 15 deletions(-)
include/linux/ratelimit.h +1 -1
@@ -44,7 +44,7 @@ static inline void ratelimit_state_reset_interval(struct ratelimit_state *rs, inraw_spin_lock_irqsave (&rs->lock, flags); rs->interval = interval_init; rs->flags &= ~RATELIMIT_INITIALIZED ; - rs->printed =0 ; +atomic_set (&rs->rs_n_left, rs->burst);ratelimit_state_reset_miss (rs);raw_spin_unlock_irqrestore (&rs->lock, flags); }
include/linux/ratelimit_types.h +1 -1
@@ -18,7 +18,7 @@ struct ratelimit_state {int interval;int burst; -int printed; +atomic_t rs_n_left;atomic_t missed;unsigned int flags;unsigned long begin;
lib/ratelimit.c +38 -13
77 lines not on this page · raw message
Signature · 9 lines
2.49.0
It is still racy. But I think that all versions were somehow racy.
BTW: The new selftest fails with it. There seems to be two kind of
problems:
1. Timing. The test seems to be a bit sensitive on timing.
It helped me to do not go that close to the "interval":lib/test_ratelimit.c +4 -4
@@ -24,19 +24,19 @@ static void test_ratelimit_smoke(struct kunit *test)test_ratelimited (test, true);test_ratelimited (test, false); -schedule_timeout_idle (TESTRL_INTERVAL -20 ); +schedule_timeout_idle (TESTRL_INTERVAL -40 );test_ratelimited (test, false); -schedule_timeout_idle (30 ); +schedule_timeout_idle (50 );test_ratelimited (test, true);schedule_timeout_idle (2 *TESTRL_INTERVAL );test_ratelimited (test, true);test_ratelimited (test, true); -schedule_timeout_idle (TESTRL_INTERVAL -20 ); +schedule_timeout_idle (TESTRL_INTERVAL -40 );test_ratelimited (test, true); -schedule_timeout_idle (30 ); +schedule_timeout_idle (50 );test_ratelimited (test, true);test_ratelimited (test, true);test_ratelimited (test, true);
2. It still failed on burst = 0 test:
// Test disabling. testrl.burst = 0; test_ratelimited(test, true);
I haven't tried the selftest with the original code. But it looks to me like it always returned "false" here as well. But I agree that it would make more sense to return "true" here.
Best Regards,
Petr
Re: [PATCH RFC 8/9] ratelimit: Reduce ratelimit's false-positive misses
Quoted 23 lines · Petr Mladek
Quoted 22 lines · Paul E. McKenney
The current ratelimit implementation can suffer from false-positive misses. That is,
___ratelimit()might return zero (causing the caller to invoke rate limiting, for example, by dropping printk()s) even when the current burst has not yet been consumed. This happens when one CPU holds a given ratelimit structure's lock and some other CPU concurrently invokes___ratelimit(). The fact that the lock is a raw irq-disabled spinlock might make low-contention trylock failure seem unlikely, but vCPU preemption, NMIs, and firmware interrupts can greatly extend the trylock-failure window.Avoiding these false-positive misses is especially important when correlating console logged hardware failures with other information.
Therefore, instead of attempting to acquire the lock on each call to
___ratelimit(), construct a lockless fastpath and only acquire the lock when retriggering (for the next burst) or when resynchronizing (due to either a long idle period or due to ratelimiting having been disabled). This reduces the number of lock-hold periods that can be extended by vCPU preemption, NMIs and firmware interrupts, but also means that these extensions must be of much longer durations (generally moving from milliseconds to seconds) before they can result in false-positive drops.
Quoted 5 lines
Quoted 5 lines
In addition, the lockless fastpath gets a 10-20% speedup compared to the old fully locked code on my x86 laptop. Your mileage will of course vary depending on your hardware, workload, and configuration.
[ paulmck: Apply feedback from Dan Carpenter. ]
Thank you for digging into this!
Quoted 90 lines
Quoted 90 lines
lib/ratelimit.c +54
-19@@ -26,55 +26,161 @@*/ int ___ratelimit (struct ratelimit_state *rs,const char *func) { -/* Paired with WRITE_ONCE() in .proc_handler(). -* Changing two values seperately could be inconsistent -* and some message could be lost. (See: net_ratelimit_state). -*/ -int interval =READ_ONCE (rs->interval); +unsigned long begin;int burst =READ_ONCE (rs->burst); +int delta =0 ;unsigned long flags; -int ret; - -if (!interval) -return 1 ; +bool gotlock = false; +bool initialized; +int interval =READ_ONCE (rs->interval); +unsigned long j; +int n_left;/* -* If we contend on this state's lock then almost -* by definition we are too busy to print a message, -* in addition to the one that will be printed by -* the entity that is holding the lock already: +* If the burst or interval settings mark this ratelimit_state +* structure as disabled, then clear the RATELIMIT_INITIALIZED bit +* in ->flags to force resetting of the ratelimiting interval when +* this ratelimit_state structure is next re-enabled. */ -if (!raw_spin_trylock_irqsave (&rs->lock, flags)) { -ratelimit_state_inc_miss (rs); -return 0 ; +if (burst <=0 || interval <=0 ) { +if ((READ_ONCE (rs->flags) &RATELIMIT_INITIALIZED ) && +raw_spin_trylock_irqsave (&rs->lock, flags)) { +if (READ_ONCE (rs->flags) &RATELIMIT_INITIALIZED ) +smp_store_release (&rs->flags, rs->flags & ~RATELIMIT_INITIALIZED ); +raw_spin_unlock_irqrestore (&rs->lock, flags); + } +return true; } -if (!(rs->flags &RATELIMIT_INITIALIZED )) { - rs->begin = jiffies; - rs->flags |=RATELIMIT_INITIALIZED ; +/* +* If this structure has just now been ratelimited, but not yet +* reset for the next rate-limiting interval, take an early and +* low-cost exit. +*/ +if (atomic_read_acquire (&rs->rs_n_left) <=0 )/* Pair with release. */ + goto limited; + +/* +* If this structure is marked as initialized and has been +* recently used, pick up its ->begin field. Otherwise, pick up +* the current time and attempt to re-initialized the structure. +*/ + j = jiffies; + initialized =smp_load_acquire (&rs->flags) &RATELIMIT_INITIALIZED ;/* Pair with release. */ +if (initialized) { + begin =READ_ONCE (rs->begin); + }else { +/* +* Uninitialized or long idle, so reset ->begin and +* mark initialized. If we fail to acquire the lock, +* let the lock holder do the work. +*/ + begin = j; +if (raw_spin_trylock_irqsave (&rs->lock, flags)) { +if (!(READ_ONCE (rs->flags) &RATELIMIT_INITIALIZED )) { + begin = jiffies; + j = begin; +WRITE_ONCE (rs->begin, begin); +smp_store_release (&rs->flags,/* Pair with acquire. */ + rs->flags |RATELIMIT_INITIALIZED ); + initialized = true; + } +raw_spin_unlock_irqrestore (&rs->lock, flags); + } }
Quoted 10 lines
lib/ratelimit.c line 96
Quoted 7 lines
lib/ratelimit.c +5
-2-if (time_is_before_jiffies (rs->begin + interval)) { -int m =ratelimit_state_reset_miss (rs); +/* +* If this structure is still in the interval in which has +* already hit the rate limit, take an early and low-cost exit. +*/ +if (initialized &&time_before (begin -2 * interval, j) &&time_before (j, begin))How do we know that the previous interval was ratelimited, please? I have to admit that I still do not fully get the new logic ;-)
Because otherwise "begin" would still be before "j". The fact that "begin" is after "j" means that we have already hit the limit and we are reset for the new interval that will start at time "begin".
Quoted 35 lines
Quoted 35 lines
lib/ratelimit.c +29
-5+ goto limited; -if (m) { -if (!(rs->flags &RATELIMIT_MSG_ON_RELEASE )) { -printk_deferred (KERN_WARNING -"%s: %d callbacks suppressed\n" , func, m); - } +/* +* Register another request, and take an early (but not low-cost) +* exit if rate-limiting just nowcame into effect. +*/ + n_left =atomic_dec_return (&rs->rs_n_left); +if (n_left <0 ) + goto limited;/* Just now started ratelimiting. */ +if (n_left >0 ) { +/* +* Otherwise, there is not yet any rate limiting for the +* current interval, and furthermore there is at least one +* last count remaining. But check to see if initialization +* is required or if we have run off the end of the interval +* without rate limiting having been imposed. Either way, +* we eventually return @true to tell our caller to go ahead. +*/ +if (initialized && +time_before (begin - interval, j) &&time_before (j, begin + interval)) +return true;/* Nothing special to do. */ +if (!raw_spin_trylock_irqsave (&rs->lock, flags)) +return true;/* Let lock holder do special work. */ + interval =READ_ONCE (rs->interval); + begin = rs->begin; + initialized =smp_load_acquire (&rs->flags) &RATELIMIT_INITIALIZED ; +if (interval <=0 || + (initialized && +time_before (begin - interval, j) &&time_before (j, begin + interval))) { +/*
Quoted 8 lines
lib/ratelimit.c line 129
Quoted 3 lines
lib/ratelimit.c +3
-0+* Someone else beat us to the special work, +* so release the lock and return. +*/ I am confused by the comment. IMHO, we are here when we do not need to reset the interval. It happens either when the interval is infinite (interval < 0) or when we are still in the currrent interval and more calls are allowed (
n_left> 0).
We ran off the end of the previous window before exhausting its ->rs_n_left. So the counts we didn't use during that interval we now lose and we start over with ->rs_n_left reset to ->burst.
Except that in this case, by the time we actually acquired the lock, someone else had already done that reset for us.
Quoted 21 lines
Quoted 21 lines
lib/ratelimit.c +11
-8+raw_spin_unlock_irqrestore (&rs->lock, flags); +return true; } - rs->begin = jiffies; - rs->printed =0 ; + +/* We have the lock and will do initialization. */ + gotlock = true; + delta =-1 ; } -if (burst && burst > rs->printed) { - rs->printed++; - ret =1 ; - }else { -ratelimit_state_inc_miss (rs); - ret =0 ; +if (!gotlock) { +/* +* We get here if we got the last count (n_left == 0), +* so that rate limiting is in effect for the next caller. +* We will return @true to tell our caller to go ahead,
Quoted 8 lines
lib/ratelimit.c line 146
Quoted 4 lines
lib/ratelimit.c +4
-0+* but first we acquire the lock and set things up for +* the next rate-limiting interval. +*/ +raw_spin_lock_irqsave (&rs->lock, flags);I do not have a good feeling about this. It could cause a deadlock in
panic()when other CPUs are stopped or in NMI. And the ratelimit used to be "safe" in this context.
Ah, then this needs to be a trylock. I did miss that point, thank you.
Quoted 3 lines
Honestly, the new code is a bit hard to follow for me. The original logic was more strightforward. I wonder if we could reduce the problem "just" by checking the remaining allowed count.
And I could repair this, but let's look at what you have.
Quoted 116 lines
I mean to introduce the atomic
rs_n_leftand check it when it is not possible to take the lock. But keep most of the current logic. Something like:Quoted 1 line
From
1df6d1a79645dd5c0ad7cef7c94b802de202057eMon Sep 17 00:00:00 2001From: Petr Mladek <[email protected]>
Date: Tue, 8 Apr 2025 16:00:22 +0200
Subject: [PATCH 1/2] ratelimit: Reduce ratelimit's false-positive
(alternative)--- include/linux/ratelimit.h | 2 +- include/linux/ratelimit_types.h | 2 +- lib/ratelimit.c | 51 ++++++++++++++++++++++++--------- 3 files changed, 40 insertions(+), 15 deletions(-)include/linux/ratelimit.h +1
-1@@ -44,7 +44,7 @@ static inline void ratelimit_state_reset_interval(struct ratelimit_state *rs, inraw_spin_lock_irqsave (&rs->lock, flags); rs->interval = interval_init; rs->flags &= ~RATELIMIT_INITIALIZED ; - rs->printed =0 ; +atomic_set (&rs->rs_n_left, rs->burst);ratelimit_state_reset_miss (rs);raw_spin_unlock_irqrestore (&rs->lock, flags); }include/linux/ratelimit_types.h +1
-1@@ -18,7 +18,7 @@ struct ratelimit_state {int interval;int burst; -int printed; +atomic_t rs_n_left;atomic_t missed;unsigned int flags;unsigned long begin;lib/ratelimit.c +32
-13@@ -39,12 +39,22 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func)return 1 ;/* -* If we contend on this state's lock then almost -* by definition we are too busy to print a message, -* in addition to the one that will be printed by -* the entity that is holding the lock already: +* If we contend on this state's lock then just check if +* the current burst is used or not. It might cause +* false positive when we are past the interval and +* the current lock owner is just about to reset it. */ if (!raw_spin_trylock_irqsave (&rs->lock, flags)) { +unsigned int rs_flags =READ_ONCE (rs->flags); + +if (rs_flags &RATELIMIT_INITIALIZED && burst) { +int n_left; + + n_left =atomic_dec_return (&rs->rs_n_left); +if (n_left >=0 ) +return 1 ; + } +ratelimit_state_inc_miss (rs);return 0 ; }@@ -52,27 +62,42 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func)if (!(rs->flags &RATELIMIT_INITIALIZED )) { rs->begin = jiffies; rs->flags |=RATELIMIT_INITIALIZED ; +atomic_set (&rs->rs_n_left, rs->burst); }if (time_is_before_jiffies (rs->begin + interval)) { -int m =ratelimit_state_reset_miss (rs); +int m; +/* +* Reset rs_n_left ASAP to reduce false positives +* in parallel calls, see above. +*/ +atomic_set (&rs->rs_n_left, rs->burst); + rs->begin = jiffies; + + m =ratelimit_state_reset_miss (rs);if (m) {if (!(rs->flags &RATELIMIT_MSG_ON_RELEASE )) {printk_deferred (KERN_WARNING "%s: %d callbacks suppressed\n" , func, m); } } - rs->begin = jiffies; - rs->printed =0 ; } -if (burst && burst > rs->printed) { - rs->printed++; - ret =1 ; - }else { -ratelimit_state_inc_miss (rs); - ret =0 ; +if (burst) { +int n_left; + +/* The burst might have been taken by a parallel call. */ + n_left =atomic_dec_return (&rs->rs_n_left); +if (n_left >=0 ) { + ret =1 ; + goto unlock_ret;
lib/ratelimit.c int ___ratelimit(struct ratelimit_state *rs, const char *func) · line 103
Quoted 10 lines
lib/ratelimit.c +6
-0+ } } + +ratelimit_state_inc_miss (rs); + ret =0 ; + +unlock_ret:raw_spin_unlock_irqrestore (&rs->lock, flags);return ret;
This looks like a plausible approach, and it certainly is a simpler change. I will look more closely.
Quoted 28 lines
2.49.0
It is still racy. But I think that all versions were somehow racy.
BTW: The new selftest fails with it. There seems to be two kind of
problems:1. Timing. The test seems to be a bit sensitive on timing. It helped me to do not go that close to the "interval":
lib/test_ratelimit.c +2
-2@@ -24,19 +24,19 @@ static void test_ratelimit_smoke(struct kunit *test)test_ratelimited (test, true);test_ratelimited (test, false); -schedule_timeout_idle (TESTRL_INTERVAL -20 ); +schedule_timeout_idle (TESTRL_INTERVAL -40 );test_ratelimited (test, false); -schedule_timeout_idle (30 ); +schedule_timeout_idle (50 );test_ratelimited (test, true);schedule_timeout_idle (2 *TESTRL_INTERVAL );test_ratelimited (test, true);
lib/test_ratelimit.c static void test_ratelimit_smoke(struct kunit *test) · line 42
Quoted 10 lines
lib/test_ratelimit.c +2
-2test_ratelimited (test, true); -schedule_timeout_idle (TESTRL_INTERVAL -20 ); +schedule_timeout_idle (TESTRL_INTERVAL -40 );test_ratelimited (test, true); -schedule_timeout_idle (30 ); +schedule_timeout_idle (50 );test_ratelimited (test, true);test_ratelimited (test, true);test_ratelimited (test, true);
Fair enough. I am not fond of my fixed delays, but have not yet come up with anything better.
Quoted 9 lines
2. It still failed on burst = 0 test:
// Test disabling. testrl.burst = 0; test_ratelimited(test, true);I haven't tried the selftest with the original code. But it looks to me like it always returned "false" here as well. But I agree that it would make more sense to return "true" here.
Thank you again, and I will look into this more carefully. My plan is to both trylock-ify my code and look carefully at your code. If all else turns out to be equal, your simpler code of course wins.
Thanx, Paul
RFC PATCH 9/9 lib: Add trivial kunit test for ratelimit
In reply to Paul E. McKenneyAdd a simple single-threaded smoke test for lib/ratelimit.c
--- lib/Kconfig.debug | 11 +++++++ lib/Makefile | 1 + lib/test_ratelimit.c | 77 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 89 insertions(+) create mode 100644 lib/test_ratelimit.c
lib/Kconfig.debug +11 -0
@@ -3166,6 +3166,17 @@ config TEST_OBJPOOL If unsure, say N. +config TEST_RATELIMIT + tristate "Test module for correctness and stress of ratelimit" if !KUNIT_ALL_TESTS + depends on KUNIT + default KUNIT_ALL_TESTS + help + This builds the "test_ratelimit" module that should be used + for correctness verification and concurrent testings of rate + limiting. + + If unsure, say N. + config INT_POW_TEST tristate "Integer exponentiation (int_pow) test" if !KUNIT_ALL_TESTS depends on KUNIT
lib/Makefile +1 -0
@@ -112,6 +112,7 @@ obj-$(CONFIG_TEST_REF_TRACKER) += test_ref_tracker.o CFLAGS_test_fprobe.o += $(CC_FLAGS_FTRACE) obj-$(CONFIG_FPROBE_SANITY_TEST) += test_fprobe.o obj-$(CONFIG_TEST_OBJPOOL) += test_objpool.o +obj-$(CONFIG_TEST_RATELIMIT) += test_ratelimit.o obj-$(CONFIG_TEST_FPU) += test_fpu.o test_fpu-y := test_fpu_glue.o test_fpu_impl.o
lib/test_ratelimit.c new file +77 -0
77 lines not on this page · raw message
Signature · 1 line
2.40.1
Hello!
This v2 series replaces open-coded uses of the ratelimit_state structure with formal APIs, counts all rate-limit misses, replaces jiffies=0 special case with a flag, provides a ___ratelimit() trylock-failure fastpath to (almost) eliminate false-positive misses, and adds a simple test.
The key point of this series is the reduction of false-positive misses.
The individual patches are as follows:
1. Add trivial kunit test for ratelimit.
2. Create functions to handle ratelimit_state internals.
3. Avoid open-coded use of ratelimit_state structure's ->missed
field.
4. Avoid open-coded use of ratelimit_state structure's ->missed
field.
5. Avoid open-coded use of ratelimit_state structure's internals.
6. Convert the ->missed field to atomic_t.
7. Count misses due to lock contention.
8. Avoid jiffies=0 special case.
9. Reduce ___ratelimit() false-positive rate limiting, courtesy of
Petr Mladek.
10. Allow zero ->burst to disable ratelimiting.
11. Force re-initialization when rate-limiting re-enabled.
12. Don't flush misses counter if RATELIMIT_MSG_ON_RELEASE.
13. Avoid atomic decrement if already rate-limited.
14. Avoid atomic decrement under lock if already rate-limited.
Changes since v1 (RFC):
o Switch from lockless fastpath to carrying out needed updates
upon trylock failure, per Petr Mladek feedback. This greatly simplifies the code and is a much smaller change from the current code. There is a small performance penalty compared to the lockless fastpath, but not enough to matter.
o Never unconditionally acquire the lock, again per Petr Mladek
feedback.
o Better define effects of non-positive burst values (always
ratelimit) and non-positive interval values (never ratelimit when the burst value is positive).
o The changes from Petr's original are supplied as five incremental
patches, but could easily be folded into Petr's original if desired. (Left to my lazy self, they stay as-is.)
Thanx, Paul
------------------------------------------------------------------------
b/drivers/char/random.c | 9 +-- b/drivers/gpu/drm/amd/pm/amdgpu_pm.c | 11 --- b/drivers/gpu/drm/i915/i915_perf.c | 8 +- b/include/linux/ratelimit.h | 40 ++++++++++++- b/include/linux/ratelimit_types.h | 2 b/lib/Kconfig.debug | 11 +++ b/lib/Makefile | 1 b/lib/ratelimit.c | 8 +- b/lib/test_ratelimit.c | 79 +++++++++++++++++++++++++++ include/linux/ratelimit.h | 13 +--- include/linux/ratelimit_types.h | 3 - lib/ratelimit.c | 102 +++++++++++++++++++++++++---------- 12 files changed, 223 insertions(+), 64 deletions(-)
PATCH v2 1/14 lib: Add trivial kunit test for ratelimit
Add a simple single-threaded smoke test for lib/ratelimit.c
To run on x86:
make ARCH=x86_64 mrproper ./tools/testing/kunit/kunit.py run --arch x86_64 --kconfig_add CONFIG_TEST_RATELIMIT=y --kconfig_add CONFIG_SMP=y lib_ratelimit
[ paulmck: Apply timeout feedback from Petr Mladek. ]
--- lib/Kconfig.debug | 11 ++++++ lib/Makefile | 1 + lib/test_ratelimit.c | 79 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 91 insertions(+) create mode 100644 lib/test_ratelimit.c
lib/Kconfig.debug +11 -0
@@ -3232,6 +3232,17 @@ config TEST_OBJPOOL If unsure, say N. +config TEST_RATELIMIT + tristate "Test module for correctness and stress of ratelimit" if !KUNIT_ALL_TESTS + depends on KUNIT + default KUNIT_ALL_TESTS + help + This builds the "test_ratelimit" module that should be used + for correctness verification and concurrent testings of rate + limiting. + + If unsure, say N. + config INT_POW_KUNIT_TEST tristate "Integer exponentiation (int_pow) test" if !KUNIT_ALL_TESTS depends on KUNIT
lib/Makefile +1 -0
@@ -103,6 +103,7 @@ obj-$(CONFIG_TEST_HMM) += test_hmm.o obj-$(CONFIG_TEST_FREE_PAGES) += test_free_pages.o obj-$(CONFIG_TEST_REF_TRACKER) += test_ref_tracker.o obj-$(CONFIG_TEST_OBJPOOL) += test_objpool.o +obj-$(CONFIG_TEST_RATELIMIT) += test_ratelimit.o obj-$(CONFIG_TEST_FPU) += test_fpu.o test_fpu-y := test_fpu_glue.o test_fpu_impl.o
lib/test_ratelimit.c new file +79 -0
79 lines not on this page · raw message
Signature · 1 line
2.40.1
Re: [PATCH v2 ratelimit 01/14] lib: Add trivial kunit test for ratelimit
Hi,
I have been recently involved in a conversion of printf/scanf selftests to KUnit. And I seem that there are some naming conventions. Adding Kees into Cc.
Quoted 32 lines · Paul E. McKenney
Add a simple single-threaded smoke test for
lib/ratelimit.cTo run on x86:
make ARCH=x86_64 mrproper ./tools/testing/kunit/kunit.py run --arch x86_64 --kconfig_add CONFIG_TEST_RATELIMIT=y --kconfig_add CONFIG_SMP=y lib_ratelimit[ paulmck: Apply timeout feedback from Petr Mladek. ]
Signed-off-by Paul E. McKenney <[email protected]>Cc Andrew Morton <[email protected]>Cc Kuniyuki Iwashima <[email protected]>Cc Mateusz Guzik <[email protected]>Cc Petr Mladek <[email protected]>Cc Steven Rostedt <[email protected]>Cc John Ogness <[email protected]>Cc Sergey Senozhatsky <[email protected]>Cc Jon Pan-Doh <[email protected]>Cc Bjorn Helgaas <[email protected]>Cc Karolina Stolarek <[email protected]>--- lib/Kconfig.debug | 11 ++++++ lib/Makefile | 1 + lib/test_ratelimit.c | 79 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 91 insertions(+) create mode 100644 lib/test_ratelimit.clib/Kconfig.debug +0
-0
lib/Kconfig.debug config TEST_OBJPOOL · line 3235
Quoted 4 lines
lib/Kconfig.debug +1
-0@@ -3232,6 +3232,17 @@ config TEST_OBJPOOL If unsure, say N. +config TEST_RATELIMIT
Most KUnit tests seems to follow the pattern <ITEM>_KUNIT_TEST. You might want to use:
config RATELIMIT_KUNIT_TEST
lib/Kconfig.debug config TEST_OBJPOOL · line 3236
Quoted 1 line
lib/Kconfig.debug +1
-0+ tristate "Test module for correctness and stress of ratelimit" if !KUNIT_ALL_TESTS
tristate "KUnit Test for correctness and stress of ratelimit" if !KUNIT_ALL_TESTS
lib/test_ratelimit.c
Quoted 14 lines
lib/Kconfig.debug +9
-0+ depends on KUNIT + default KUNIT_ALL_TESTS + help + This builds the "test_ratelimit" module that should be used + for correctness verification and concurrent testings of rate + limiting. + + If unsure, say N. + config INT_POW_KUNIT_TEST tristate "Integer exponentiation (int_pow) test" if !KUNIT_ALL_TESTS depends on KUNITlib/test_ratelimit.c new file +0
-0
The commit db6fe4d61ece241 ("lib: Move KUnit tests into tests/ subdirectory") moved many kunit test modules to lib/tests.
Also they renamed the printf/scanf test modules to <item>_kunit.c, so this probably should be:
lib/tests/ratelimit_kunit.c
Quoted 49 lines
lib/test_ratelimit.c new file +48
-0@@ -0,0 +1,79 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include <kunit/test.h> + +#include <linux/ratelimit.h> +#include <linux/module.h> + +/* a simple boot-time regression test */ + +#define TESTRL_INTERVAL (5 * HZ) +static DEFINE_RATELIMIT_STATE(testrl, TESTRL_INTERVAL, 3); + +#define test_ratelimited (test, expected) \ + KUNIT_ASSERT_EQ(test, ___ratelimit(&testrl, "test_ratelimit_smoke"), (expected)); + +static void test_ratelimit_smoke (struct kunit *test) +{ +// Check settings. +KUNIT_ASSERT_GE (test,TESTRL_INTERVAL ,100 ); + +// Test normal operation. +test_ratelimited (test, true); +test_ratelimited (test, true); +test_ratelimited (test, true); +test_ratelimited (test, false); + +schedule_timeout_idle (TESTRL_INTERVAL -40 ); +test_ratelimited (test, false); + +schedule_timeout_idle (50 ); +test_ratelimited (test, true); + +schedule_timeout_idle (2 *TESTRL_INTERVAL ); +test_ratelimited (test, true); +test_ratelimited (test, true); + +schedule_timeout_idle (TESTRL_INTERVAL -40 ); +test_ratelimited (test, true); +schedule_timeout_idle (50 ); +test_ratelimited (test, true); +test_ratelimited (test, true); +test_ratelimited (test, true); +test_ratelimited (test, false); + +// Test disabling. + testrl.burst =0 ; +test_ratelimited (test, false); + testrl.burst =2 ;
lib/test_ratelimit.c line 58
Quoted 10 lines
lib/test_ratelimit.c new file +10
-0+ testrl.interval =0 ; +test_ratelimited (test, true); +test_ratelimited (test, true); +test_ratelimited (test, true); +test_ratelimited (test, true); +test_ratelimited (test, true); +test_ratelimited (test, true); +test_ratelimited (test, true); + +// Testing re-enabling.
BTW: I had to add the following sleep to make sure that the next call
was past the interval. Otherwise, the test failed.
schedule_timeout_idle(TESTRL_INTERVAL);
That said, this was needed when testing the original ___ratelimit() implementation. It is not needed with the entire patchset applied. I guess that it was solved by the 11th patch "[PATCH 11/14] ratelimit:
Force re-initialization when rate-limiting re-enabled".
lib/test_ratelimit.c line 67
Quoted 9 lines
lib/test_ratelimit.c new file +9
-0+ testrl.interval =TESTRL_INTERVAL ; +test_ratelimited (test, true); +test_ratelimited (test, true); +test_ratelimited (test, false); +test_ratelimited (test, false); +} + +static struct kunit_case sort_test_cases[] = { +KUNIT_CASE (test_ratelimit_smoke),
The test printed:
[ 75.479585] # test_ratelimit_smoke: Test should be marked slow (runtime: 25.326793644s)
The warning has gone when using:
KUNIT_CASE_SLOW(test_ratelimit_smoke),
lib/test_ratelimit.c line 74
Quoted 7 lines
lib/test_ratelimit.c new file +7
-0+ {} +}; + +static struct kunit_suite ratelimit_test_suite = { + .name ="lib_ratelimit" , + .test_cases = sort_test_cases, +};
Best Regards,
Petr
Paul E. McKenney
Reviewed-by Petr MladekRe: [PATCH v2 ratelimit 01/14] lib: Add trivial kunit test for ratelimit
Quoted 5 lines · Petr Mladek
Hi,
I have been recently involved in a conversion of printf/scanf selftests to KUnit. And I seem that there are some naming conventions. Adding Kees into Cc.
Thank you for the review and feedback! Updated patch is at the end of this email.
Thanx, Paul
Quoted 133 lines · Paul E. McKenney
Quoted 32 lines
Add a simple single-threaded smoke test for
lib/ratelimit.cTo run on x86:
make ARCH=x86_64 mrproper ./tools/testing/kunit/kunit.py run --arch x86_64 --kconfig_add CONFIG_TEST_RATELIMIT=y --kconfig_add CONFIG_SMP=y lib_ratelimit[ paulmck: Apply timeout feedback from Petr Mladek. ]
Signed-off-by Paul E. McKenney <[email protected]>Cc Andrew Morton <[email protected]>Cc Kuniyuki Iwashima <[email protected]>Cc Mateusz Guzik <[email protected]>Cc Petr Mladek <[email protected]>Cc Steven Rostedt <[email protected]>Cc John Ogness <[email protected]>Cc Sergey Senozhatsky <[email protected]>Cc Jon Pan-Doh <[email protected]>Cc Bjorn Helgaas <[email protected]>Cc Karolina Stolarek <[email protected]>--- lib/Kconfig.debug | 11 ++++++ lib/Makefile | 1 + lib/test_ratelimit.c | 79 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 91 insertions(+) create mode 100644 lib/test_ratelimit.clib/Kconfig.debug +0
-0lib/Kconfig.debug config TEST_OBJPOOL · line 3235
Quoted 4 lines
lib/Kconfig.debug +1
-0@@ -3232,6 +3232,17 @@ config TEST_OBJPOOL If unsure, say N. +config TEST_RATELIMITMost KUnit tests seems to follow the pattern <ITEM>_KUNIT_TEST. You might want to use:
config
RATELIMIT_KUNIT_TESTlib/Kconfig.debug config TEST_OBJPOOL · line 3236
Quoted 1 line
lib/Kconfig.debug +1
-0+ tristate "Test module for correctness and stress of ratelimit" if !KUNIT_ALL_TESTStristate "KUnit Test for correctness and stress of ratelimit" if !KUNIT_ALL_TESTS
lib/test_ratelimit.c
Quoted 14 lines
lib/Kconfig.debug +9
-0+ depends on KUNIT + default KUNIT_ALL_TESTS + help + This builds the "test_ratelimit" module that should be used + for correctness verification and concurrent testings of rate + limiting. + + If unsure, say N. + config INT_POW_KUNIT_TEST tristate "Integer exponentiation (int_pow) test" if !KUNIT_ALL_TESTS depends on KUNITlib/test_ratelimit.c new file +0
-0The commit
db6fe4d61ece241("lib: Move KUnit tests into tests/ subdirectory") moved many kunit test modules to lib/tests.Also they renamed the printf/scanf test modules to <item>_kunit.c, so this probably should be:
lib/tests/ratelimit_kunit.cQuoted 49 lines
lib/test_ratelimit.c new file +48
-0@@ -0,0 +1,79 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include <kunit/test.h> + +#include <linux/ratelimit.h> +#include <linux/module.h> + +/* a simple boot-time regression test */ + +#define TESTRL_INTERVAL (5 * HZ) +static DEFINE_RATELIMIT_STATE(testrl, TESTRL_INTERVAL, 3); + +#define test_ratelimited (test, expected) \ + KUNIT_ASSERT_EQ(test, ___ratelimit(&testrl, "test_ratelimit_smoke"), (expected)); + +static void test_ratelimit_smoke (struct kunit *test) +{ +// Check settings. +KUNIT_ASSERT_GE (test,TESTRL_INTERVAL ,100 ); + +// Test normal operation. +test_ratelimited (test, true); +test_ratelimited (test, true); +test_ratelimited (test, true); +test_ratelimited (test, false); + +schedule_timeout_idle (TESTRL_INTERVAL -40 ); +test_ratelimited (test, false); + +schedule_timeout_idle (50 ); +test_ratelimited (test, true); + +schedule_timeout_idle (2 *TESTRL_INTERVAL ); +test_ratelimited (test, true); +test_ratelimited (test, true); + +schedule_timeout_idle (TESTRL_INTERVAL -40 ); +test_ratelimited (test, true); +schedule_timeout_idle (50 ); +test_ratelimited (test, true); +test_ratelimited (test, true); +test_ratelimited (test, true); +test_ratelimited (test, false); + +// Test disabling. + testrl.burst =0 ; +test_ratelimited (test, false); + testrl.burst =2 ;lib/test_ratelimit.c line 58
Quoted 10 lines
lib/test_ratelimit.c new file +10
-0+ testrl.interval =0 ; +test_ratelimited (test, true); +test_ratelimited (test, true); +test_ratelimited (test, true); +test_ratelimited (test, true); +test_ratelimited (test, true); +test_ratelimited (test, true); +test_ratelimited (test, true); + +// Testing re-enabling. BTW: I had to add the following sleep to make sure that the next call
was past the interval. Otherwise, the test failed.
Quoted 6 lines
schedule_timeout_idle(TESTRL_INTERVAL);That said, this was needed when testing the original
___ratelimit()implementation. It is not needed with the entire patchset applied. I guess that it was solved by the 11th patch "[PATCH 11/14] ratelimit:
Force re-initialization when rate-limiting re-enabled".
Yes, the intent is that the series fixes this test failure.
Quoted 18 lines
lib/test_ratelimit.c line 67
Quoted 9 lines
lib/test_ratelimit.c new file +9
-0+ testrl.interval =TESTRL_INTERVAL ; +test_ratelimited (test, true); +test_ratelimited (test, true); +test_ratelimited (test, false); +test_ratelimited (test, false); +} + +static struct kunit_case sort_test_cases[] = { +KUNIT_CASE (test_ratelimit_smoke),The test printed:
[ 75.479585] # test_ratelimit_smoke: Test should be marked slow (runtime: 25.326793644s)The warning has gone when using:
KUNIT_CASE_SLOW(test_ratelimit_smoke),
Quoted 7 lines
lib/test_ratelimit.c line 74
Quoted 7 lines
lib/test_ratelimit.c new file +7
-0+ {} +}; + +static struct kunit_suite ratelimit_test_suite = { + .name ="lib_ratelimit" , + .test_cases = sort_test_cases, +};
------------------------------------------------------------------------
lib: Add trivial kunit test for ratelimit
Add a simple single-threaded smoke test for lib/ratelimit.c
To run on x86:
make ARCH=x86_64 mrproper
./tools/testing/kunit/kunit.py run --arch x86_64 --kconfig_add CONFIG_RATELIMIT_KUNIT_TEST=y --kconfig_add CONFIG_SMP=y lib_ratelimit
This will fail on old ___ratelimit(), and subsequent patches provide the fixes that are required.
[ paulmck: Apply timeout and kunit feedback from Petr Mladek. ]
lib/Kconfig.debug +11 -0
@@ -3232,6 +3232,17 @@ config TEST_OBJPOOL If unsure, say N. +config RATELIMIT_KUNIT_TEST + tristate "KUnit Test for correctness and stress of ratelimit" if !KUNIT_ALL_TESTS + depends on KUNIT + default KUNIT_ALL_TESTS + help + This builds the "test_ratelimit" module that should be used + for correctness verification and concurrent testings of rate + limiting. + + If unsure, say N. + config INT_POW_KUNIT_TEST tristate "Integer exponentiation (int_pow) test" if !KUNIT_ALL_TESTS depends on KUNIT
lib/tests/Makefile +1 -0
@@ -45,5 +45,6 @@ obj-$(CONFIG_STRING_KUNIT_TEST) += string_kunit.o obj-$(CONFIG_STRING_HELPERS_KUNIT_TEST) += string_helpers_kunit.o obj-$(CONFIG_USERCOPY_KUNIT_TEST) += usercopy_kunit.o obj-$(CONFIG_UTIL_MACROS_KUNIT) += util_macros_kunit.o +obj-$(CONFIG_RATELIMIT_KUNIT_TEST) += test_ratelimit.o obj-$(CONFIG_TEST_RUNTIME_MODULE) += module/
lib/tests/test_ratelimit.c new file +79 -0
79 lines not on this page · raw message
Re: [PATCH v2 ratelimit 01/14] lib: Add trivial kunit test for ratelimit
Quoted 26 lines · Paul E. McKenney
Quoted 5 lines · Petr Mladek
Hi,
I have been recently involved in a conversion of printf/scanf selftests to KUnit. And I seem that there are some naming conventions. Adding Kees into Cc.
Thank you for the review and feedback! Updated patch is at the end of this email.
------------------------------------------------------------------------
lib: Add trivial kunit test for ratelimit
Add a simple single-threaded smoke test for
lib/ratelimit.cTo run on x86:
make ARCH=x86_64 mrproper
./tools/testing/kunit/kunit.pyrun --archx86_64--kconfig_addCONFIG_RATELIMIT_KUNIT_TEST=y--kconfig_addCONFIG_SMP=ylib_ratelimitThis will fail on old
___ratelimit(), and subsequent patches provide the fixes that are required.[ paulmck: Apply timeout and kunit feedback from Petr Mladek. ]
Quoted 11 lines
Signed-off-by Paul E. McKenney <[email protected]>Reviewed-by Petr Mladek <[email protected]>Cc Andrew Morton <[email protected]>Cc Kuniyuki Iwashima <[email protected]>Cc Mateusz Guzik <[email protected]>Cc Steven Rostedt <[email protected]>Cc John Ogness <[email protected]>Cc Sergey Senozhatsky <[email protected]>Cc Jon Pan-Doh <[email protected]>Cc Bjorn Helgaas <[email protected]>Cc Karolina Stolarek <[email protected]>
JFYI, I have checked this updated version and looks good to me.
Best Regards,
Petr
PATCH v2 2/14 ratelimit: Create functions to handle ratelimit_state internals
In reply to Paul E. McKenneyA number of ratelimit uses do open-coded access to the ratelimit_state structure's ->missed field. This works, but is a bit messy and makes it more annoying to make changes to this field.
Therefore, provide a ratelimit_state_inc_miss() function that increments out the ->missed field ratelimit_state_get_miss() function that reads out the ->missed field of the specified ratelimit_state structure, and a ratelimit_state_reset_miss() function that reads out that field, but that also resets its value to zero. These functions will replace client-code open-coded uses of ->miss.
In addition, a new ratelimit_state_reset_interval() encapsulates what was previously open-coded lock acquisition and resetting.
[ paulmck: Apply kernel test robot feedback. ]
--- include/linux/ratelimit.h | 40 ++++++++++++++++++++++++++++++++++----- lib/ratelimit.c | 8 ++++---- 2 files changed, 39 insertions(+), 9 deletions(-)
include/linux/ratelimit.h +35 -5
@@ -22,16 +22,46 @@ static inline void ratelimit_default_init(struct ratelimit_state *rs) DEFAULT_RATELIMIT_BURST); } +static inline void ratelimit_state_inc_miss (struct ratelimit_state *rs) +{ + rs->missed++; +} + +static inline int ratelimit_state_get_miss (struct ratelimit_state *rs) +{ +return rs->missed; +} + +static inline int ratelimit_state_reset_miss (struct ratelimit_state *rs) +{ +int ret = rs->missed; + + rs->missed =0 ; +return ret; +} + +static inline void ratelimit_state_reset_interval (struct ratelimit_state *rs,int interval_init) +{ +unsigned long flags; + +raw_spin_lock_irqsave (&rs->lock, flags); + rs->interval = interval_init; + rs->begin =0 ; + rs->printed =0 ; +ratelimit_state_reset_miss (rs); +raw_spin_unlock_irqrestore (&rs->lock, flags); +} +static inline void ratelimit_state_exit (struct ratelimit_state *rs) { +int m; +if (!(rs->flags &RATELIMIT_MSG_ON_RELEASE ))return ; -if (rs->missed) { -pr_warn ("%s: %d output lines suppressed due to ratelimiting\n" , - current->comm, rs->missed); - rs->missed =0 ; - } + m =ratelimit_state_reset_miss (rs); +if (m) +pr_warn ("%s: %d output lines suppressed due to ratelimiting\n" , current->comm, m); } static inline void
lib/ratelimit.c +4 -4
@@ -51,12 +51,12 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func) rs->begin = jiffies;if (time_is_before_jiffies (rs->begin + interval)) { -if (rs->missed) { +int m =ratelimit_state_reset_miss (rs); + +if (m) {if (!(rs->flags &RATELIMIT_MSG_ON_RELEASE )) {printk_deferred (KERN_WARNING -"%s: %d callbacks suppressed\n" , - func, rs->missed); - rs->missed =0 ; +"%s: %d callbacks suppressed\n" , func, m); } } rs->begin = jiffies;
Signature · 1 line
2.40.1
PATCH v2 3/14 random: Avoid open-coded use of ratelimit_state structure's ->missed field
In reply to Paul E. McKenneyThe _credit_init_bits() function directly accesses the ratelimit_state structure's ->missed field, which work, but which also makes it more difficult to change this field. Therefore, make use of the ratelimit_state_get_miss() and ratelimit_state_inc_miss() functions instead of directly accessing the ->missed field.
"Jason A. Donenfeld" <[email protected]>
--- drivers/char/random.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-)
drivers/char/random.c +5 -4
@@ -726,6 +726,7 @@ static void __cold _credit_init_bits(size_t bits) static DECLARE_WORK(set_ready, crng_set_ready);unsigned int new, orig, add;unsigned long flags; +int m;if (!bits)return ;
@@ -748,9 +749,9 @@ static void __cold _credit_init_bits(size_t bits)wake_up_interruptible (&crng_init_wait);kill_fasync (&fasync,SIGIO ,POLL_IN );pr_notice ("crng init done\n" ); -if (urandom_warning.missed) -pr_notice ("%d urandom warning(s) missed due to ratelimiting\n" , - urandom_warning.missed); + m =ratelimit_state_get_miss (&urandom_warning); +if (m) +pr_notice ("%d urandom warning(s) missed due to ratelimiting\n" , m); }else if (orig <POOL_EARLY_BITS && new >=POOL_EARLY_BITS ) {spin_lock_irqsave (&base_crng.lock, flags);/* Check if crng_init is CRNG_EMPTY, to avoid race with crng_reseed(). */
@@ -1466,7 +1467,7 @@ static ssize_t urandom_read_iter(struct kiocb *kiocb, struct iov_iter *iter)if (!crng_ready ()) {if (!ratelimit_disable && maxwarn <=0 ) - ++urandom_warning.missed; +ratelimit_state_inc_miss (&urandom_warning);else if (ratelimit_disable ||__ratelimit (&urandom_warning)) { --maxwarn; pr_notice("%s: uninitialized urandom read (%zu bytes read)\n",
Signature · 1 line
2.40.1
PATCH v2 4/14 drm/i915: Avoid open-coded use of ratelimit_state structure's ->missed field
In reply to Paul E. McKenneyThe i915_oa_stream_destroy() function directly accesses the ratelimit_state structure's ->missed field, which work, but which also makes it more difficult to change this field. Therefore, make use of the ratelimit_state_get_miss() function instead of directly accessing the ->missed field.
--- drivers/gpu/drm/i915/i915_perf.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-)
drivers/gpu/drm/i915/i915_perf.c +4 -4
@@ -1666,6 +1666,7 @@ static void i915_oa_stream_destroy(struct i915_perf_stream *stream)struct i915_perf *perf = stream->perf;struct intel_gt *gt = stream->engine->gt;struct i915_perf_group *g = stream->engine->oa_group; +int m;if (WARN_ON (stream != g->exclusive_stream))return ;
@@ -1690,10 +1691,9 @@ static void i915_oa_stream_destroy(struct i915_perf_stream *stream)free_oa_configs (stream);free_noa_wait (stream); -if (perf->spurious_report_rs.missed) { -gt_notice (gt,"%d spurious OA report notices suppressed due to ratelimiting\n" , - perf->spurious_report_rs.missed); - } + m =ratelimit_state_get_miss (&perf->spurious_report_rs); +if (m) +gt_notice (gt,"%d spurious OA report notices suppressed due to ratelimiting\n" , m); }static void gen7_init_oa_buffer (struct i915_perf_stream *stream)
Signature · 1 line
2.40.1
PATCH v2 5/14 drm/amd/pm: Avoid open-coded use of ratelimit_state structure's internals
In reply to Paul E. McKenneyThe amdgpu_set_thermal_throttling_logging() function directly accesses the ratelimit_state structure's ->missed field, which work, but which also makes it more difficult to change this field. Therefore, make use of the ratelimit_state_reset_miss() function instead of directly accessing the ->missed field.
Nevertheless, open-coded use of ->burst and ->interval is still permitted, for example, for runtime sysfs adjustment of these fields.
--- drivers/gpu/drm/amd/pm/amdgpu_pm.c | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-)
drivers/gpu/drm/amd/pm/amdgpu_pm.c +2 -9
@@ -1606,7 +1606,6 @@ static ssize_t amdgpu_set_thermal_throttling_logging(struct device *dev,struct drm_device *ddev =dev_get_drvdata (dev);struct amdgpu_device *adev =drm_to_adev (ddev);long throttling_logging_interval; -unsigned long flags;int ret =0 ; ret =kstrtol (buf,0 , &throttling_logging_interval);
@@ -1617,18 +1616,12 @@ static ssize_t amdgpu_set_thermal_throttling_logging(struct device *dev,return -EINVAL ;if (throttling_logging_interval >0 ) { -raw_spin_lock_irqsave (&adev->throttling_logging_rs.lock, flags);/* * Reset the ratelimit timer internals. * This can effectively restart the timer. */ - adev->throttling_logging_rs.interval = - (throttling_logging_interval -1 ) *HZ ; - adev->throttling_logging_rs.begin =0 ; - adev->throttling_logging_rs.printed =0 ; - adev->throttling_logging_rs.missed =0 ; -raw_spin_unlock_irqrestore (&adev->throttling_logging_rs.lock, flags); - +ratelimit_state_reset_interval (&adev->throttling_logging_rs, + (throttling_logging_interval -1 ) *HZ );atomic_set (&adev->throttling_logging_enabled,1 ); }else {atomic_set (&adev->throttling_logging_enabled,0 );
Signature · 1 line
2.40.1
PATCH v2 6/14 ratelimit: Convert the ->missed field to atomic_t
In reply to Paul E. McKenneyThe ratelimit_state structure's ->missed field is sometimes incremented locklessly, and it would be good to avoid lost counts. This is also needed to count the number of misses due to try-lock failure. Therefore, convert the rratelimit_state structure's ->missed field to atomic_t.
--- include/linux/ratelimit.h | 9 +++------ include/linux/ratelimit_types.h | 2 +- lib/ratelimit.c | 2 +- 3 files changed, 5 insertions(+), 8 deletions(-)
include/linux/ratelimit.h +3 -6
@@ -24,20 +24,17 @@ static inline void ratelimit_default_init(struct ratelimit_state *rs)static inline void ratelimit_state_inc_miss (struct ratelimit_state *rs) { - rs->missed++; +atomic_inc (&rs->missed); }static inline int ratelimit_state_get_miss (struct ratelimit_state *rs) { -return rs->missed; +return atomic_read (&rs->missed); }static inline int ratelimit_state_reset_miss (struct ratelimit_state *rs) { -int ret = rs->missed; - - rs->missed =0 ; -return ret; +return atomic_xchg_relaxed (&rs->missed,0 ); }static inline void ratelimit_state_reset_interval (struct ratelimit_state *rs,int interval_init)
include/linux/ratelimit_types.h +1 -1
@@ -18,7 +18,7 @@ struct ratelimit_state {int interval;int burst;int printed; -int missed; +atomic_t missed;unsigned int flags;unsigned long begin; };
lib/ratelimit.c +1 -1
@@ -66,7 +66,7 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func) rs->printed++; ret =1 ; }else { - rs->missed++; +ratelimit_state_inc_miss (rs); ret =0 ; }raw_spin_unlock_irqrestore (&rs->lock, flags);
Signature · 1 line
2.40.1
PATCH v2 7/14 ratelimit: Count misses due to lock contention
In reply to Paul E. McKenneyThe ___ratelimit() function simply returns zero ("do ratelimiting") if the trylock fails, but does not adjust the ->missed field. This means that the resulting dropped printk()s are dropped silently, which could seriously confuse people trying to do console-log-based debugging. Therefore, increment the ->missed field upon trylock failure.
--- lib/ratelimit.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-)
lib/ratelimit.c +3 -1
@@ -44,8 +44,10 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func)* in addition to the one that will be printed by * the entity that is holding the lock already: */ -if (!raw_spin_trylock_irqsave (&rs->lock, flags)) +if (!raw_spin_trylock_irqsave (&rs->lock, flags)) { +ratelimit_state_inc_miss (rs);return 0 ; + }if (!rs->begin) rs->begin = jiffies;
Signature · 1 line
2.40.1
PATCH v2 8/14 ratelimit: Avoid jiffies=0 special case
In reply to Paul E. McKenneyThe ___ratelimit() function special-cases the jiffies-counter value of zero as "uninitialized". This works well on 64-bit systems, where the jiffies counter is not going to return to zero for more than half a billion years on systems with HZ=1000, but similar 32-bit systems take less than 50 days to wrap the jiffies counter. And although the consequences of wrapping the jiffies counter seem to be limited to minor confusion on the duration of the rate-limiting interval that happens to end at time zero, it is almost no work to avoid this confusion.
Therefore, introduce a RATELIMIT_INITIALIZED bit to the ratelimit_state structure's ->flags field so that a ->begin value of zero is no longer special.
--- include/linux/ratelimit.h | 2 +- include/linux/ratelimit_types.h | 1 + lib/ratelimit.c | 4 +++- 3 files changed, 5 insertions(+), 2 deletions(-)
include/linux/ratelimit.h +1 -1
@@ -43,7 +43,7 @@ static inline void ratelimit_state_reset_interval(struct ratelimit_state *rs, inraw_spin_lock_irqsave (&rs->lock, flags); rs->interval = interval_init; - rs->begin =0 ; + rs->flags &= ~RATELIMIT_INITIALIZED ; rs->printed =0 ;ratelimit_state_reset_miss (rs);raw_spin_unlock_irqrestore (&rs->lock, flags);
include/linux/ratelimit_types.h +1 -0
@@ -11,6 +11,7 @@/* issue num suppressed message on exit */ #define RATELIMIT_MSG_ON_RELEASE BIT(0) +#define RATELIMIT_INITIALIZED BIT(1)struct ratelimit_state {raw_spinlock_t lock;/* protect the state */
lib/ratelimit.c +3 -1
@@ -49,8 +49,10 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func)return 0 ; } -if (!rs->begin) +if (!(rs->flags &RATELIMIT_INITIALIZED )) { rs->begin = jiffies; + rs->flags |=RATELIMIT_INITIALIZED ; + }if (time_is_before_jiffies (rs->begin + interval)) {int m =ratelimit_state_reset_miss (rs);
Signature · 1 line
2.40.1
PATCH v2 9/14 ratelimit: Reduce ___ratelimit() false-positive rate limiting
In reply to Paul E. McKenneyFrom: Petr Mladek <[email protected]>
Retain the locked design, but check rate-limiting even when the lock could not be acquired.
--- include/linux/ratelimit.h | 2 +- include/linux/ratelimit_types.h | 2 +- lib/ratelimit.c | 51 ++++++++++++++++++++++++--------- 3 files changed, 40 insertions(+), 15 deletions(-)
include/linux/ratelimit.h +1 -1
@@ -44,7 +44,7 @@ static inline void ratelimit_state_reset_interval(struct ratelimit_state *rs, inraw_spin_lock_irqsave (&rs->lock, flags); rs->interval = interval_init; rs->flags &= ~RATELIMIT_INITIALIZED ; - rs->printed =0 ; +atomic_set (&rs->rs_n_left, rs->burst);ratelimit_state_reset_miss (rs);raw_spin_unlock_irqrestore (&rs->lock, flags); }
include/linux/ratelimit_types.h +1 -1
@@ -18,7 +18,7 @@ struct ratelimit_state {int interval;int burst; -int printed; +atomic_t rs_n_left;atomic_t missed;unsigned int flags;unsigned long begin;
lib/ratelimit.c +38 -13
77 lines not on this page · raw message
Signature · 1 line
2.40.1
PATCH v2 10/14 ratelimit: Allow zero ->burst to disable ratelimiting
In reply to Paul E. McKenneyIf ->burst is zero, rate-limiting will be applied unconditionally. Therefore, make it low overhead.
--- lib/ratelimit.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
lib/ratelimit.c +1 -1
@@ -35,7 +35,7 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func)unsigned long flags;int ret; -if (!interval) +if (!interval || !burst)return 1 ;/*
Signature · 1 line
2.40.1
PATCH v2 11/14 ratelimit: Force re-initialization when rate-limiting re-enabled
In reply to Paul E. McKenneyCurrently, rate limiting being disabled results in an immediate early return with no state changes. This can result in false-positive drops when re-enabling rate limiting. Therefore, mark the ratelimit_state structure "uninitialized" when rate limiting is disabled.
Additionally, interpret non-positive ->burst to unconditionally force rate limiting. When ->burst is positive, interpret non-positive interval to unconditionally disable rate limiting.
--- lib/ratelimit.c | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-)
lib/ratelimit.c +14 -2
@@ -35,8 +35,20 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func)unsigned long flags;int ret; -if (!interval || !burst) -return 1 ; +/* +* Non-positive burst says always limit, otherwise, non-positive +* interval says never limit. +*/ +if (interval <=0 || burst <=0 ) { + ret = burst >0 ; +if (!(READ_ONCE (rs->flags) &RATELIMIT_INITIALIZED ) || + !raw_spin_trylock_irqsave (&rs->lock, flags)) +return ret; + +/* Force re-initialization once re-enabled. */ + rs->flags &= ~RATELIMIT_INITIALIZED ; + goto unlock_ret; + }/* * If we contend on this state's lock then just check if
Signature · 1 line
2.40.1
Re: [PATCH v2 ratelimit 11/14] ratelimit: Force re-initialization when rate-limiting re-enabled
Quoted 4 lines · Paul E. McKenney
Currently, rate limiting being disabled results in an immediate early return with no state changes. This can result in false-positive drops when re-enabling rate limiting. Therefore, mark the
ratelimit_statestructure "uninitialized" when rate limiting is disabled.
Today's -next is failing to boot a defconfig on a wide range of arm64 ACPI platforms. One ACPI platform (Softiron) is still fine, and I've not noticed any DT platforms having issues. Output grinds to a halt during boot shortly after userspace starts on affected systems:
[ 23.334050] Freeing unused kernel memory: 11328K [ 23.355182] Run /init as init process [ 23.378221] NET: Registered PF_INET6 protocol family [ 23.396506] Segment Routing with IPv6 [ 23.414054] In-situ OAM (IOAM) with IPv6
A bisect converges fairly smoothly on this patch in -next, which doesn't look completely implausible for something that stops console output - I didn't dig into it at all though. I see that Sirkanth (CCed) seems to have reported a similar issue though with a different SHA1 since he noticed on yesterday's -next.
Bisect log:
git bisect start
# status: waiting for both good and bad commits
# bad: [6ac908f24cd7ddae52c496bbc888e97ee7b033ac] Add linux-next specific files for 20250423
git bisect bad 6ac908f24cd7ddae52c496bbc888e97ee7b033ac
# status: waiting for good commit(s), bad commit known
# good: [2a9308cbc987660dd54df7af2a8685d512d467d0] Merge branch 'for-linux-next-fixes' of https://gitlab.freedesktop.org/drm/misc/kernel.git
git bisect good 2a9308cbc987660dd54df7af2a8685d512d467d0
# good: [68123785b5b6b54698b92e534c2d42cbb39701c0] Merge branch 'drm-next' of https://gitlab.freedesktop.org/drm/kernel.git
git bisect good 68123785b5b6b54698b92e534c2d42cbb39701c0
# good: [99554718252d6f01036ac8e0b45bcfc16bf0e0ce] next-20250414/tip
git bisect good 99554718252d6f01036ac8e0b45bcfc16bf0e0ce
# bad: [d017479907b87ffbcfe908c7188f19722d3fdb46] Merge branch 'togreg' of git://git.kernel.org/pub/scm/linux/kernel/git/jic23/iio.git
git bisect bad d017479907b87ffbcfe908c7188f19722d3fdb46
# bad: [379565a0c27c2b9c9e6df8cac66dd458a71a33b5] Merge branch 'driver-core-next' of git://git.kernel.org/pub/scm/linux/kernel/git/driver-core/driver-core.git
git bisect bad 379565a0c27c2b9c9e6df8cac66dd458a71a33b5
# good: [fd02aa45bda6d2f2fedcab70e828867332ef7e1c] Merge branch 'kvm-tdx-initial' into HEAD
git bisect good fd02aa45bda6d2f2fedcab70e828867332ef7e1c
# bad: [a46505d01338de4299d417ebe01b12060ebadce7] next-20250414/workqueues
git bisect bad a46505d01338de4299d417ebe01b12060ebadce7
# good: [5109fd27d5ade19fe5de22c746157c693878dde7] Merge branch 'next' of git://git.kernel.org/pub/scm/linux/kernel/git/rcu/linux.git
git bisect good 5109fd27d5ade19fe5de22c746157c693878dde7
# bad: [ad0c7a85f8b529154c99103d4daa0e86f4c14272] ratelimit: Avoid atomic decrement under lock if already rate-limited
git bisect bad ad0c7a85f8b529154c99103d4daa0e86f4c14272
# good: [945e63d4897045421a64f86cbd8194e0ff06dc41] ratelimit: Count misses due to lock contention
git bisect good 945e63d4897045421a64f86cbd8194e0ff06dc41
# good: [13fa70e052ddc778ecf70db21a07d97c2487da90] ratelimit: Allow zero ->burst to disable ratelimiting
git bisect good 13fa70e052ddc778ecf70db21a07d97c2487da90
# bad: [5d80e7bbfddda9369f301b484d621c45de8edf5f] ratelimit: Don't flush misses counter if RATELIMIT_MSG_ON_RELEASE
git bisect bad 5d80e7bbfddda9369f301b484d621c45de8edf5f
# bad: [c8943bdecfc76711f83241e21da9d4530f872f0d] ratelimit: Force re-initialization when rate-limiting re-enabled
git bisect bad c8943bdecfc76711f83241e21da9d4530f872f0d
# first bad commit: [c8943bdecfc76711f83241e21da9d4530f872f0d] ratelimit: Force re-initialization when rate-limiting re-enabled
Re: [PATCH v2 ratelimit 11/14] ratelimit: Force re-initialization when rate-limiting re-enabled
Quoted 17 lines · Mark Brown
Quoted 4 lines · Paul E. McKenney
Currently, rate limiting being disabled results in an immediate early return with no state changes. This can result in false-positive drops when re-enabling rate limiting. Therefore, mark the
ratelimit_statestructure "uninitialized" when rate limiting is disabled.Today's -next is failing to boot a defconfig on a wide range of arm64 ACPI platforms. One ACPI platform (Softiron) is still fine, and I've not noticed any DT platforms having issues. Output grinds to a halt during boot shortly after userspace starts on affected systems:
[ 23.334050] Freeing unused kernel memory: 11328K [ 23.355182] Run /init as init process [ 23.378221] NET: Registered PF_INET6 protocol family [ 23.396506] Segment Routing with IPv6 [ 23.414054] In-situ OAM (IOAM) with IPv6
Quoted 5 lines
A bisect converges fairly smoothly on this patch in -next, which doesn't look completely implausible for something that stops console output - I didn't dig into it at all though. I see that Sirkanth (CCed) seems to have reported a similar issue though with a different SHA1 since he noticed on yesterday's -next.
Thank you for the confirmation of Bert's and Srikanth's reports!
They reported that the replacing the offending commit with the following patch fixed things up:
------------------------------------------------------------------------
lib/ratelimit.c +1 -1
@@ -35,7 +35,7 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func)unsigned long flags;int ret; -if (!interval || !burst) +if (interval <=0 || burst <=0 )return 1 ;/*
------------------------------------------------------------------------
If that fixes things for you, could you also please try the following, also replacing that same commit?
Thanx, Paul
------------------------------------------------------------------------
lib/ratelimit.c +3 -1
@@ -35,8 +35,10 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func)unsigned long flags;int ret; -if (!interval || !burst) +if (interval <=0 || burst <=0 ) { + ret = burst >0 ;return 1 ; + }/* * If we contend on this state's lock then just check if
Re: [PATCH v2 ratelimit 11/14] ratelimit: Force re-initialization when rate-limiting re-enabled
lib/ratelimit.c int ___ratelimit(struct ratelimit_state *rs, const char *func) · line 39
Quoted 10 lines · Paul E. McKenney
On Wed, Apr 23, 2025 at 04:59:49PM +0100, Mark Brown wrote:
They reported that the replacing the offending commit with the following patch fixed things up:
lib/ratelimit.c +1
-1-if (!interval || !burst) +if (interval <=0 || burst <=0 )return 1 ;
That fixes things, yes.
lib/ratelimit.c int ___ratelimit(struct ratelimit_state *rs, const char *func) · line 41
Quoted 9 lines
If that fixes things for you, could you also please try the following, also replacing that same commit?
lib/ratelimit.c +3
-1-if (!interval || !burst) +if (interval <=0 || burst <=0 ) { + ret = burst >0 ;return 1 ; + }
Are you sure about that - the value set for ret will be ignored, we still return 1 regardless of the value of burst as for the first patch AFAICT? I tried instead:
+ if (interval <= 0 || burst <= 0) {
+ return burst > 0;
+ }
which failed.
Re: [PATCH v2 ratelimit 11/14] ratelimit: Force re-initialization when rate-limiting re-enabled
Quoted 32 lines · Mark Brown
Review
Quoted 10 lines · Paul E. McKenney
On Wed, Apr 23, 2025 at 04:59:49PM +0100, Mark Brown wrote:
They reported that the replacing the offending commit with the following patch fixed things up:
(quoted diff) +1
-1- if (!interval || !burst) + if (interval <= 0 || burst <= 0) return 1;That fixes things, yes.
Review
Quoted 9 lines
If that fixes things for you, could you also please try the following, also replacing that same commit?
(quoted diff) +3
-1- if (!interval || !burst) + if (interval <= 0 || burst <= 0) { + ret = burst > 0; return 1; + }Are you sure about that - the value set for ret will be ignored, we still return 1 regardless of the value of burst as for the first patch AFAICT? I tried instead:
(quoted diff) +3
-0+ if (interval <= 0 || burst <= 0) { + return burst > 0; + }
Quoted 1 line
which failed.
Thank you (and Bert) very much for testing this!
You are of course quite right. But my previous patch above, the one that just returned 1 if either interval or burst was non-positive, was also wrong.
Before that patch, if both burst and interval were negative, rate limiting would be imposed unconditionally. With that patch, if if both burst and interval were negative, rate limiting would never happen, which might come as a nasty surprise to someone setting burst to zero in order to unconditionally suppress output.
To preserve necessary portions of the old semantics, I believe that we need something like this:
interval: <0 =0 >0
burst:
<0 RL !RL RL =0 RL !RL RL >0 !RL !RL ?RL
Here, "RL" says to always do rate-limiting, "!RL" says to never do rate-limiting, and "?RL" says to let the atomic decrement decide.
The idea is that if interval is =0, rate-limiting never happens. But if interval <0, then burst has full control, so that burst <=0 always rate-limits and burst >0 never rate-limits. Finally, if interval >0, burst again has full control, but while burst <=0 always rate-limits, burst >0 lets the atomic decrement decide.
Except that in the "?RL" case, the "if" condition directs execution elsewhere anyway, so I believe that the "interval == 0 || burst > 0" condition in my updated patch (see below) fixes things.
Could you please try the patch shown below? I might actually have it right this time. ;-)
Thanx, Paul
------------------------------------------------------------------------
lib/ratelimit.c +2 -2
@@ -35,8 +35,8 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func)unsigned long flags;int ret; -if (!interval || !burst) -return 1 ; +if (interval <=0 || burst <=0 ) +return interval ==0 || burst >0 ;/* * If we contend on this state's lock then just check if
Re: [PATCH v2 ratelimit 11/14] ratelimit: Force re-initialization when rate-limiting re-enabled
Quoted 2 lines · Paul E. McKenney
Could you please try the patch shown below? I might actually have it right this time. ;-)
This seems to also fix the boot on the affected platforms.
Re: [PATCH v2 ratelimit 11/14] ratelimit: Force re-initialization when rate-limiting re-enabled
Quoted 6 lines · Mark Brown
Quoted 2 lines · Paul E. McKenney
Could you please try the patch shown below? I might actually have it right this time. ;-)
This seems to also fix the boot on the affected platforms.
Thank you very much for bearing with me!
Thanx, Paul
PATCH v2 12/14 ratelimit: Don't flush misses counter if RATELIMIT_MSG_ON_RELEASE
In reply to Paul E. McKenneyRestore the previous semantics where the misses counter is unchanged if the RATELIMIT_MSG_ON_RELEASE flag is set.
--- lib/ratelimit.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-)
lib/ratelimit.c +3 -3
@@ -87,9 +87,9 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func)atomic_set (&rs->rs_n_left, rs->burst); rs->begin = jiffies; - m =ratelimit_state_reset_miss (rs); -if (m) { -if (!(rs->flags &RATELIMIT_MSG_ON_RELEASE )) { +if (!(rs->flags &RATELIMIT_MSG_ON_RELEASE )) { + m =ratelimit_state_reset_miss (rs); +if (m) {printk_deferred (KERN_WARNING "%s: %d callbacks suppressed\n" , func, m); }
Signature · 1 line
2.40.1
PATCH v2 13/14 ratelimit: Avoid atomic decrement if already rate-limited
In reply to Paul E. McKenneyCurrently, if the lock could not be acquired, the code unconditionally does an atomic decrement on ->rs_n_left, even if that atomic operation is guaranteed to return a limit-rate verdict. This incurs needless overhead and also raises the spectre of counter wrap.
Therefore, do the atomic decrement only if there is some chance that rates won't be limited.
--- lib/ratelimit.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-)
lib/ratelimit.c +3 -1
@@ -60,8 +60,10 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func)unsigned int rs_flags =READ_ONCE (rs->flags);if (rs_flags &RATELIMIT_INITIALIZED && burst) { -int n_left; +int n_left =atomic_read (&rs->rs_n_left); +if (n_left <=0 ) +return 0 ; n_left =atomic_dec_return (&rs->rs_n_left);if (n_left >=0 )return 1 ;
Signature · 1 line
2.40.1
PATCH v2 14/14 ratelimit: Avoid atomic decrement under lock if already rate-limited
In reply to Paul E. McKenneyCurrently, if the lock is acquired, the code unconditionally does an atomic decrement on ->rs_n_left, even if that atomic operation is guaranteed to return a limit-rate verdict. A limit-rate verdict will in fact be the common case when something is spewing into a rate limit. This unconditional atomic operation incurs needless overhead and also raises the spectre of counter wrap.
Therefore, do the atomic decrement only if there is some chance that rates won't be limited.
--- lib/ratelimit.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-)
lib/ratelimit.c +8 -5
@@ -98,13 +98,16 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func) } }if (burst) { -int n_left; +int n_left =atomic_read (&rs->rs_n_left);/* The burst might have been taken by a parallel call. */ - n_left =atomic_dec_return (&rs->rs_n_left); -if (n_left >=0 ) { - ret =1 ; - goto unlock_ret; + +if (n_left >0 ) { + n_left =atomic_dec_return (&rs->rs_n_left); +if (n_left >=0 ) { + ret =1 ; + goto unlock_ret; + } } }
Signature · 1 line
2.40.1
Petr Mladek
Reviewed-byQuoted 7 lines · Paul E. McKenney
Hello!
This v2 series replaces open-coded uses of the
ratelimit_statestructure with formal APIs, counts all rate-limit misses, replaces jiffies=0 special case with a flag, provides a___ratelimit()trylock-failure fastpath to (almost) eliminate false-positive misses, and adds a simple test.
Quoted 5 lines
The key point of this series is the reduction of false-positive misses.
The individual patches are as follows:
1. Add trivial kunit test for ratelimit.
I have suggested few cosmetic changes for the above patch.
Quoted 21 lines
2. Create functions to handle
ratelimit_stateinternals.3. Avoid open-coded use of
ratelimit_statestructure's ->missed
field.4. Avoid open-coded use of
ratelimit_statestructure's ->missed
field.5. Avoid open-coded use of
ratelimit_statestructure's internals.6. Convert the ->missed field to
atomic_t.7. Count misses due to lock contention.
8. Avoid jiffies=0 special case.
9. Reduce
___ratelimit()false-positive rate limiting, courtesy of
Petr Mladek.10. Allow zero ->burst to disable ratelimiting.
Quoted 7 lines
11. Force re-initialization when rate-limiting re-enabled.
12. Don't flush misses counter if
RATELIMIT_MSG_ON_RELEASE.13. Avoid atomic decrement if already rate-limited.
14. Avoid atomic decrement under lock if already rate-limited.
The rest looks good. And I think that it is a great improvement. Feel free to use for the entire patchset:
Best Regards,
Petr
Quoted 8 lines · Petr Mladek
Quoted 7 lines · Paul E. McKenney
Hello!
This v2 series replaces open-coded uses of the
ratelimit_statestructure with formal APIs, counts all rate-limit misses, replaces jiffies=0 special case with a flag, provides a___ratelimit()trylock-failure fastpath to (almost) eliminate false-positive misses, and adds a simple test.
Quoted 7 lines
Quoted 5 lines
The key point of this series is the reduction of false-positive misses.
The individual patches are as follows:
1. Add trivial kunit test for ratelimit.
I have suggested few cosmetic changes for the above patch.
Thank you for the KUNIT tutorial, and I will apply these.
Quoted 25 lines
Quoted 25 lines
2. Create functions to handle
ratelimit_stateinternals.3. Avoid open-coded use of
ratelimit_statestructure's ->missed
field.4. Avoid open-coded use of
ratelimit_statestructure's ->missed
field.5. Avoid open-coded use of
ratelimit_statestructure's internals.6. Convert the ->missed field to
atomic_t.7. Count misses due to lock contention.
8. Avoid jiffies=0 special case.
9. Reduce
___ratelimit()false-positive rate limiting, courtesy of
Petr Mladek.10. Allow zero ->burst to disable ratelimiting.
11. Force re-initialization when rate-limiting re-enabled.
12. Don't flush misses counter if
RATELIMIT_MSG_ON_RELEASE.
Quoted 8 lines
Quoted 3 lines
13. Avoid atomic decrement if already rate-limited.
14. Avoid atomic decrement under lock if already rate-limited.
The rest looks good. And I think that it is a great improvement. Feel free to use for the entire patchset:
Reviewed-by Petr Mladek <[email protected]>
Again, thank you, and I will apply these to the series!
Thanx, Paul
Hello!
This v3 series replaces open-coded uses of the ratelimit_state structure with formal APIs, counts all rate-limit misses, replaces jiffies=0 special case with a flag, provides a ___ratelimit() trylock-failure fastpath to (almost) eliminate false-positive misses, simplifies the code, and adds a simple test.
The key point of this series is the reduction of false-positive misses.
The individual patches are as follows:
1. Add trivial kunit test for ratelimit.
2. Create functions to handle ratelimit_state internals.
3. Avoid open-coded use of ratelimit_state structure's ->missed
field.
4. Avoid open-coded use of ratelimit_state structure's ->missed
field.
5. Avoid open-coded use of ratelimit_state structure's internals.
6. Convert the ->missed field to atomic_t.
7. Count misses due to lock contention.
8. Avoid jiffies=0 special case.
9. Reduce ___ratelimit() false-positive rate limiting, courtesy of
Petr Mladek.
10. Allow zero ->burst to disable ratelimiting.
11. Force re-initialization when rate-limiting re-enabled.
12. Don't flush misses counter if RATELIMIT_MSG_ON_RELEASE.
13. Avoid atomic decrement if already rate-limited.
14. Avoid atomic decrement under lock if already rate-limited.
15. Warn if ->interval or ->burst are negative, courtesy of Petr
Mladek.
16. Simplify common-case exit path.
17. Use nolock_ret label to save a couple of lines of code.
18. Use nolock_ret label to collapse lock-failure code.
19. Use nolock_ret restructuring to collapse common case code.
20. Drop redundant accesses to burst.
Changes since v2:
o Apply feedback from Bert Karwatzki, Srikanth Aithal, and Mark
Brown, fixing a hang that happened on some systems.
o Applied Reviewed-by tags and added links.
o Added a prototype patch from Petr Mladek that splats if either
interval or burst are negative.
o Added several commits that simplify the code.
Changes since v1 (RFC):
o Switch from lockless fastpath to carrying out needed updates
upon trylock failure, per Petr Mladek feedback. This greatly simplifies the code and is a much smaller change from the current code. There is a small performance penalty compared to the lockless fastpath, but not enough to matter.
o Never unconditionally acquire the lock, again per Petr Mladek
feedback.
o Better define effects of non-positive burst values (always
ratelimit) and non-positive interval values (never ratelimit when the burst value is positive).
o The changes from Petr's original are supplied as five incremental
patches, but could easily be folded into Petr's original if desired. (Left to my lazy self, they stay as-is.)
Thanx, Paul
------------------------------------------------------------------------
b/drivers/char/random.c | 9 + b/drivers/gpu/drm/amd/pm/amdgpu_pm.c | 11 -- b/drivers/gpu/drm/i915/i915_perf.c | 8 - b/include/linux/ratelimit.h | 40 +++++++- b/include/linux/ratelimit_types.h | 2 b/lib/Kconfig.debug | 11 ++ b/lib/ratelimit.c | 8 - b/lib/tests/Makefile | 1 b/lib/tests/test_ratelimit.c | 79 ++++++++++++++++ include/linux/ratelimit.h | 13 +- include/linux/ratelimit_types.h | 3 lib/ratelimit.c | 165 ++++++++++++++++++++--------------- 12 files changed, 246 insertions(+), 104 deletions(-)
PATCH v3 1/20 lib: Add trivial kunit test for ratelimit
Add a simple single-threaded smoke test for lib/ratelimit.c
To run on x86:
make ARCH=x86_64 mrproper ./tools/testing/kunit/kunit.py run --arch x86_64 --kconfig_add CONFIG_RATELIMIT_KUNIT_TEST=y --kconfig_add CONFIG_SMP=y lib_ratelimit
This will fail on old ___ratelimit(), and subsequent patches provide the fixes that are required.
[ paulmck: Apply timeout and kunit feedback from Petr Mladek. ]
--- lib/Kconfig.debug | 11 ++++++ lib/tests/Makefile | 1 + lib/tests/test_ratelimit.c | 79 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 91 insertions(+) create mode 100644 lib/tests/test_ratelimit.c
lib/Kconfig.debug +11 -0
@@ -3232,6 +3232,17 @@ config TEST_OBJPOOL If unsure, say N. +config RATELIMIT_KUNIT_TEST + tristate "KUnit Test for correctness and stress of ratelimit" if !KUNIT_ALL_TESTS + depends on KUNIT + default KUNIT_ALL_TESTS + help + This builds the "test_ratelimit" module that should be used + for correctness verification and concurrent testings of rate + limiting. + + If unsure, say N. + config INT_POW_KUNIT_TEST tristate "Integer exponentiation (int_pow) test" if !KUNIT_ALL_TESTS depends on KUNIT
lib/tests/Makefile +1 -0
@@ -45,5 +45,6 @@ obj-$(CONFIG_STRING_KUNIT_TEST) += string_kunit.o obj-$(CONFIG_STRING_HELPERS_KUNIT_TEST) += string_helpers_kunit.o obj-$(CONFIG_USERCOPY_KUNIT_TEST) += usercopy_kunit.o obj-$(CONFIG_UTIL_MACROS_KUNIT) += util_macros_kunit.o +obj-$(CONFIG_RATELIMIT_KUNIT_TEST) += test_ratelimit.o obj-$(CONFIG_TEST_RUNTIME_MODULE) += module/
lib/tests/test_ratelimit.c new file +79 -0
79 lines not on this page · raw message
Signature · 1 line
2.40.1
PATCH v3 2/20 ratelimit: Create functions to handle ratelimit_state internals
In reply to Paul E. McKenneyA number of ratelimit uses do open-coded access to the ratelimit_state structure's ->missed field. This works, but is a bit messy and makes it more annoying to make changes to this field.
Therefore, provide a ratelimit_state_inc_miss() function that increments out the ->missed field ratelimit_state_get_miss() function that reads out the ->missed field of the specified ratelimit_state structure, and a ratelimit_state_reset_miss() function that reads out that field, but that also resets its value to zero. These functions will replace client-code open-coded uses of ->miss.
In addition, a new ratelimit_state_reset_interval() encapsulates what was previously open-coded lock acquisition and resetting.
[ paulmck: Apply kernel test robot feedback. ]
--- include/linux/ratelimit.h | 40 ++++++++++++++++++++++++++++++++++----- lib/ratelimit.c | 8 ++++---- 2 files changed, 39 insertions(+), 9 deletions(-)
include/linux/ratelimit.h +35 -5
@@ -22,16 +22,46 @@ static inline void ratelimit_default_init(struct ratelimit_state *rs) DEFAULT_RATELIMIT_BURST); } +static inline void ratelimit_state_inc_miss (struct ratelimit_state *rs) +{ + rs->missed++; +} + +static inline int ratelimit_state_get_miss (struct ratelimit_state *rs) +{ +return rs->missed; +} + +static inline int ratelimit_state_reset_miss (struct ratelimit_state *rs) +{ +int ret = rs->missed; + + rs->missed =0 ; +return ret; +} + +static inline void ratelimit_state_reset_interval (struct ratelimit_state *rs,int interval_init) +{ +unsigned long flags; + +raw_spin_lock_irqsave (&rs->lock, flags); + rs->interval = interval_init; + rs->begin =0 ; + rs->printed =0 ; +ratelimit_state_reset_miss (rs); +raw_spin_unlock_irqrestore (&rs->lock, flags); +} +static inline void ratelimit_state_exit (struct ratelimit_state *rs) { +int m; +if (!(rs->flags &RATELIMIT_MSG_ON_RELEASE ))return ; -if (rs->missed) { -pr_warn ("%s: %d output lines suppressed due to ratelimiting\n" , - current->comm, rs->missed); - rs->missed =0 ; - } + m =ratelimit_state_reset_miss (rs); +if (m) +pr_warn ("%s: %d output lines suppressed due to ratelimiting\n" , current->comm, m); } static inline void
lib/ratelimit.c +4 -4
@@ -51,12 +51,12 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func) rs->begin = jiffies;if (time_is_before_jiffies (rs->begin + interval)) { -if (rs->missed) { +int m =ratelimit_state_reset_miss (rs); + +if (m) {if (!(rs->flags &RATELIMIT_MSG_ON_RELEASE )) {printk_deferred (KERN_WARNING -"%s: %d callbacks suppressed\n" , - func, rs->missed); - rs->missed =0 ; +"%s: %d callbacks suppressed\n" , func, m); } } rs->begin = jiffies;
Signature · 1 line
2.40.1
PATCH v3 3/20 random: Avoid open-coded use of ratelimit_state structure's ->missed field
In reply to Paul E. McKenneyThe _credit_init_bits() function directly accesses the ratelimit_state structure's ->missed field, which work, but which also makes it more difficult to change this field. Therefore, make use of the ratelimit_state_get_miss() and ratelimit_state_inc_miss() functions instead of directly accessing the ->missed field.
"Jason A. Donenfeld" <[email protected]>
--- drivers/char/random.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-)
drivers/char/random.c +5 -4
@@ -726,6 +726,7 @@ static void __cold _credit_init_bits(size_t bits) static DECLARE_WORK(set_ready, crng_set_ready);unsigned int new, orig, add;unsigned long flags; +int m;if (!bits)return ;
@@ -748,9 +749,9 @@ static void __cold _credit_init_bits(size_t bits)wake_up_interruptible (&crng_init_wait);kill_fasync (&fasync,SIGIO ,POLL_IN );pr_notice ("crng init done\n" ); -if (urandom_warning.missed) -pr_notice ("%d urandom warning(s) missed due to ratelimiting\n" , - urandom_warning.missed); + m =ratelimit_state_get_miss (&urandom_warning); +if (m) +pr_notice ("%d urandom warning(s) missed due to ratelimiting\n" , m); }else if (orig <POOL_EARLY_BITS && new >=POOL_EARLY_BITS ) {spin_lock_irqsave (&base_crng.lock, flags);/* Check if crng_init is CRNG_EMPTY, to avoid race with crng_reseed(). */
@@ -1466,7 +1467,7 @@ static ssize_t urandom_read_iter(struct kiocb *kiocb, struct iov_iter *iter)if (!crng_ready ()) {if (!ratelimit_disable && maxwarn <=0 ) - ++urandom_warning.missed; +ratelimit_state_inc_miss (&urandom_warning);else if (ratelimit_disable ||__ratelimit (&urandom_warning)) { --maxwarn; pr_notice("%s: uninitialized urandom read (%zu bytes read)\n",
Signature · 1 line
2.40.1
PATCH v3 4/20 drm/i915: Avoid open-coded use of ratelimit_state structure's ->missed field
In reply to Paul E. McKenneyThe i915_oa_stream_destroy() function directly accesses the ratelimit_state structure's ->missed field, which work, but which also makes it more difficult to change this field. Therefore, make use of the ratelimit_state_get_miss() function instead of directly accessing the ->missed field.
--- drivers/gpu/drm/i915/i915_perf.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-)
drivers/gpu/drm/i915/i915_perf.c +4 -4
@@ -1666,6 +1666,7 @@ static void i915_oa_stream_destroy(struct i915_perf_stream *stream)struct i915_perf *perf = stream->perf;struct intel_gt *gt = stream->engine->gt;struct i915_perf_group *g = stream->engine->oa_group; +int m;if (WARN_ON (stream != g->exclusive_stream))return ;
@@ -1690,10 +1691,9 @@ static void i915_oa_stream_destroy(struct i915_perf_stream *stream)free_oa_configs (stream);free_noa_wait (stream); -if (perf->spurious_report_rs.missed) { -gt_notice (gt,"%d spurious OA report notices suppressed due to ratelimiting\n" , - perf->spurious_report_rs.missed); - } + m =ratelimit_state_get_miss (&perf->spurious_report_rs); +if (m) +gt_notice (gt,"%d spurious OA report notices suppressed due to ratelimiting\n" , m); }static void gen7_init_oa_buffer (struct i915_perf_stream *stream)
Signature · 1 line
2.40.1
Re: [PATCH v3 04/20] drm/i915: Avoid open-coded use of ratelimit_state structure's ->missed field
Quoted 5 lines · Paul E. McKenney
The
i915_oa_stream_destroy()function directly accesses theratelimit_statestructure's ->missed field, which work, but which also makes it more difficult to change this field. Therefore, make use of theratelimit_state_get_miss()function instead of directly accessing the ->missed field.
For merging via whichever tree is convenient for you. Please let us know if you want us to pick it up via drm-intel.
Quoted 43 lines
Signed-off-by Paul E. McKenney <[email protected]>Reviewed-by Petr Mladek <[email protected]>Cc Jani Nikula <[email protected]>Cc Joonas Lahtinen <[email protected]>Cc Rodrigo Vivi <[email protected]>Cc Tvrtko Ursulin <[email protected]>Cc David Airlie <[email protected]>Cc Simona Vetter <[email protected]>Cc <[email protected]>Cc <[email protected]>--- drivers/gpu/drm/i915/i915_perf.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-)drivers/gpu/drm/i915/i915_perf.c +4
-4@@ -1666,6 +1666,7 @@ static void i915_oa_stream_destroy(struct i915_perf_stream *stream)struct i915_perf *perf = stream->perf;struct intel_gt *gt = stream->engine->gt;struct i915_perf_group *g = stream->engine->oa_group; +int m;if (WARN_ON (stream != g->exclusive_stream))return ;@@ -1690,10 +1691,9 @@ static void i915_oa_stream_destroy(struct i915_perf_stream *stream)free_oa_configs (stream);free_noa_wait (stream); -if (perf->spurious_report_rs.missed) { -gt_notice (gt,"%d spurious OA report notices suppressed due to ratelimiting\n" , - perf->spurious_report_rs.missed); - } + m =ratelimit_state_get_miss (&perf->spurious_report_rs); +if (m) +gt_notice (gt,"%d spurious OA report notices suppressed due to ratelimiting\n" , m); }static void gen7_init_oa_buffer (struct i915_perf_stream *stream)
Signature · 1 line
Jani Nikula, Intel
Re: [PATCH v3 04/20] drm/i915: Avoid open-coded use of ratelimit_state structure's ->missed field
Quoted 8 lines · Jani Nikula
Quoted 5 lines · Paul E. McKenney
The
i915_oa_stream_destroy()function directly accesses theratelimit_statestructure's ->missed field, which work, but which also makes it more difficult to change this field. Therefore, make use of theratelimit_state_get_miss()function instead of directly accessing the ->missed field.Acked-by Jani Nikula <[email protected]>
Thank you, Jani! I will apply this on my next rebase later today.
Quoted 2 lines
For merging via whichever tree is convenient for you. Please let us know if you want us to pick it up via drm-intel.
This one depends on a commit earlier in the series, so I will very happily take you up on your kind offer of letting me push it. ;-)
Thanx, Paul
Quoted 45 lines
Quoted 42 lines
Signed-off-by Paul E. McKenney <[email protected]>Reviewed-by Petr Mladek <[email protected]>Cc Jani Nikula <[email protected]>Cc Joonas Lahtinen <[email protected]>Cc Rodrigo Vivi <[email protected]>Cc Tvrtko Ursulin <[email protected]>Cc David Airlie <[email protected]>Cc Simona Vetter <[email protected]>Cc <[email protected]>Cc <[email protected]>--- drivers/gpu/drm/i915/i915_perf.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-)drivers/gpu/drm/i915/i915_perf.c +4
-4@@ -1666,6 +1666,7 @@ static void i915_oa_stream_destroy(struct i915_perf_stream *stream)struct i915_perf *perf = stream->perf;struct intel_gt *gt = stream->engine->gt;struct i915_perf_group *g = stream->engine->oa_group; +int m;if (WARN_ON (stream != g->exclusive_stream))return ;@@ -1690,10 +1691,9 @@ static void i915_oa_stream_destroy(struct i915_perf_stream *stream)free_oa_configs (stream);free_noa_wait (stream); -if (perf->spurious_report_rs.missed) { -gt_notice (gt,"%d spurious OA report notices suppressed due to ratelimiting\n" , - perf->spurious_report_rs.missed); - } + m =ratelimit_state_get_miss (&perf->spurious_report_rs); +if (m) +gt_notice (gt,"%d spurious OA report notices suppressed due to ratelimiting\n" , m); }static void gen7_init_oa_buffer (struct i915_perf_stream *stream)Signature · 1 line
Jani Nikula, Intel
PATCH v3 5/20 drm/amd/pm: Avoid open-coded use of ratelimit_state structure's internals
In reply to Paul E. McKenneyThe amdgpu_set_thermal_throttling_logging() function directly accesses the ratelimit_state structure's ->missed field, which work, but which also makes it more difficult to change this field. Therefore, make use of the ratelimit_state_reset_miss() function instead of directly accessing the ->missed field.
Nevertheless, open-coded use of ->burst and ->interval is still permitted, for example, for runtime sysfs adjustment of these fields.
--- drivers/gpu/drm/amd/pm/amdgpu_pm.c | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-)
drivers/gpu/drm/amd/pm/amdgpu_pm.c +2 -9
@@ -1606,7 +1606,6 @@ static ssize_t amdgpu_set_thermal_throttling_logging(struct device *dev,struct drm_device *ddev =dev_get_drvdata (dev);struct amdgpu_device *adev =drm_to_adev (ddev);long throttling_logging_interval; -unsigned long flags;int ret =0 ; ret =kstrtol (buf,0 , &throttling_logging_interval);
@@ -1617,18 +1616,12 @@ static ssize_t amdgpu_set_thermal_throttling_logging(struct device *dev,return -EINVAL ;if (throttling_logging_interval >0 ) { -raw_spin_lock_irqsave (&adev->throttling_logging_rs.lock, flags);/* * Reset the ratelimit timer internals. * This can effectively restart the timer. */ - adev->throttling_logging_rs.interval = - (throttling_logging_interval -1 ) *HZ ; - adev->throttling_logging_rs.begin =0 ; - adev->throttling_logging_rs.printed =0 ; - adev->throttling_logging_rs.missed =0 ; -raw_spin_unlock_irqrestore (&adev->throttling_logging_rs.lock, flags); - +ratelimit_state_reset_interval (&adev->throttling_logging_rs, + (throttling_logging_interval -1 ) *HZ );atomic_set (&adev->throttling_logging_enabled,1 ); }else {atomic_set (&adev->throttling_logging_enabled,0 );
Signature · 1 line
2.40.1
PATCH v3 6/20 ratelimit: Convert the ->missed field to atomic_t
In reply to Paul E. McKenneyThe ratelimit_state structure's ->missed field is sometimes incremented locklessly, and it would be good to avoid lost counts. This is also needed to count the number of misses due to try-lock failure. Therefore, convert the rratelimit_state structure's ->missed field to atomic_t.
--- include/linux/ratelimit.h | 9 +++------ include/linux/ratelimit_types.h | 2 +- lib/ratelimit.c | 2 +- 3 files changed, 5 insertions(+), 8 deletions(-)
include/linux/ratelimit.h +3 -6
@@ -24,20 +24,17 @@ static inline void ratelimit_default_init(struct ratelimit_state *rs)static inline void ratelimit_state_inc_miss (struct ratelimit_state *rs) { - rs->missed++; +atomic_inc (&rs->missed); }static inline int ratelimit_state_get_miss (struct ratelimit_state *rs) { -return rs->missed; +return atomic_read (&rs->missed); }static inline int ratelimit_state_reset_miss (struct ratelimit_state *rs) { -int ret = rs->missed; - - rs->missed =0 ; -return ret; +return atomic_xchg_relaxed (&rs->missed,0 ); }static inline void ratelimit_state_reset_interval (struct ratelimit_state *rs,int interval_init)
include/linux/ratelimit_types.h +1 -1
@@ -18,7 +18,7 @@ struct ratelimit_state {int interval;int burst;int printed; -int missed; +atomic_t missed;unsigned int flags;unsigned long begin; };
lib/ratelimit.c +1 -1
@@ -66,7 +66,7 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func) rs->printed++; ret =1 ; }else { - rs->missed++; +ratelimit_state_inc_miss (rs); ret =0 ; }raw_spin_unlock_irqrestore (&rs->lock, flags);
Signature · 1 line
2.40.1
PATCH v3 7/20 ratelimit: Count misses due to lock contention
In reply to Paul E. McKenneyThe ___ratelimit() function simply returns zero ("do ratelimiting") if the trylock fails, but does not adjust the ->missed field. This means that the resulting dropped printk()s are dropped silently, which could seriously confuse people trying to do console-log-based debugging. Therefore, increment the ->missed field upon trylock failure.
--- lib/ratelimit.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-)
lib/ratelimit.c +3 -1
@@ -44,8 +44,10 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func)* in addition to the one that will be printed by * the entity that is holding the lock already: */ -if (!raw_spin_trylock_irqsave (&rs->lock, flags)) +if (!raw_spin_trylock_irqsave (&rs->lock, flags)) { +ratelimit_state_inc_miss (rs);return 0 ; + }if (!rs->begin) rs->begin = jiffies;
Signature · 1 line
2.40.1
PATCH v3 8/20 ratelimit: Avoid jiffies=0 special case
In reply to Paul E. McKenneyThe ___ratelimit() function special-cases the jiffies-counter value of zero as "uninitialized". This works well on 64-bit systems, where the jiffies counter is not going to return to zero for more than half a billion years on systems with HZ=1000, but similar 32-bit systems take less than 50 days to wrap the jiffies counter. And although the consequences of wrapping the jiffies counter seem to be limited to minor confusion on the duration of the rate-limiting interval that happens to end at time zero, it is almost no work to avoid this confusion.
Therefore, introduce a RATELIMIT_INITIALIZED bit to the ratelimit_state structure's ->flags field so that a ->begin value of zero is no longer special.
--- include/linux/ratelimit.h | 2 +- include/linux/ratelimit_types.h | 1 + lib/ratelimit.c | 4 +++- 3 files changed, 5 insertions(+), 2 deletions(-)
include/linux/ratelimit.h +1 -1
@@ -43,7 +43,7 @@ static inline void ratelimit_state_reset_interval(struct ratelimit_state *rs, inraw_spin_lock_irqsave (&rs->lock, flags); rs->interval = interval_init; - rs->begin =0 ; + rs->flags &= ~RATELIMIT_INITIALIZED ; rs->printed =0 ;ratelimit_state_reset_miss (rs);raw_spin_unlock_irqrestore (&rs->lock, flags);
include/linux/ratelimit_types.h +1 -0
@@ -11,6 +11,7 @@/* issue num suppressed message on exit */ #define RATELIMIT_MSG_ON_RELEASE BIT(0) +#define RATELIMIT_INITIALIZED BIT(1)struct ratelimit_state {raw_spinlock_t lock;/* protect the state */
lib/ratelimit.c +3 -1
@@ -49,8 +49,10 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func)return 0 ; } -if (!rs->begin) +if (!(rs->flags &RATELIMIT_INITIALIZED )) { rs->begin = jiffies; + rs->flags |=RATELIMIT_INITIALIZED ; + }if (time_is_before_jiffies (rs->begin + interval)) {int m =ratelimit_state_reset_miss (rs);
Signature · 1 line
2.40.1
PATCH v3 9/20 ratelimit: Reduce ___ratelimit() false-positive rate limiting
In reply to Paul E. McKenneyFrom: Petr Mladek <[email protected]>
Retain the locked design, but check rate-limiting even when the lock could not be acquired.
--- include/linux/ratelimit.h | 2 +- include/linux/ratelimit_types.h | 2 +- lib/ratelimit.c | 51 ++++++++++++++++++++++++--------- 3 files changed, 40 insertions(+), 15 deletions(-)
include/linux/ratelimit.h +1 -1
@@ -44,7 +44,7 @@ static inline void ratelimit_state_reset_interval(struct ratelimit_state *rs, inraw_spin_lock_irqsave (&rs->lock, flags); rs->interval = interval_init; rs->flags &= ~RATELIMIT_INITIALIZED ; - rs->printed =0 ; +atomic_set (&rs->rs_n_left, rs->burst);ratelimit_state_reset_miss (rs);raw_spin_unlock_irqrestore (&rs->lock, flags); }
include/linux/ratelimit_types.h +1 -1
@@ -18,7 +18,7 @@ struct ratelimit_state {int interval;int burst; -int printed; +atomic_t rs_n_left;atomic_t missed;unsigned int flags;unsigned long begin;
lib/ratelimit.c +38 -13
77 lines not on this page · raw message
Signature · 1 line
2.40.1
PATCH v3 10/20 ratelimit: Allow zero ->burst to disable ratelimiting
In reply to Paul E. McKenneyIf ->interval is zero, then rate-limiting will be disabled. Alternatively, if interval is greater than zero and ->burst is zero, then rate-limiting will be applied unconditionally.
Therefore, make this classification be lockless.
Note that although negative ->interval and ->burst happen to be treated as if they were zero, this is an accident of the current implementation. The semantics of negative values for these fields is subject to change without notice. Especially given that Bert Karwatzki determined that current calls to ___ratelimit() currently never have negative values for these fields.
This commit replaces an earlier buggy versions.
--- lib/ratelimit.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-)
lib/ratelimit.c +6 -2
@@ -35,8 +35,12 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func)unsigned long flags;int ret; -if (!interval) -return 1 ; +if (interval <=0 || burst <=0 ) { + ret = interval ==0 || burst >0 ; +if (!ret) +ratelimit_state_inc_miss (rs); +return ret; + }/* * If we contend on this state's lock then just check if
Signature · 1 line
2.40.1
Re: [PATCH v3 10/20] ratelimit: Allow zero ->burst to disable ratelimiting
Quoted 6 lines · Paul E. McKenney
If ->interval is zero, then rate-limiting will be disabled. Alternatively, if interval is greater than zero and ->burst is zero, then rate-limiting will be applied unconditionally.
Therefore, make this classification be lockless.
Quoted 8 lines
Note that although negative ->interval and ->burst happen to be treated as if they were zero, this is an accident of the current implementation. The semantics of negative values for these fields is subject to change without notice. Especially given that Bert Karwatzki determined that current calls to
___ratelimit()currently never have negative values for these fields.This commit replaces an earlier buggy versions.
If there was another revision then it would be nice to explicitly describe also the reason why both zero ->interval and ->burst never rate-limits. It is the state when the structure is zeroed. Some existing code relied in this behavior.
If I get it correctly then this is the difference between this and the previous version of this patch. And the previous version caused regressions described by the Links...
Quoted 9 lines
Reported-by Bert Karwatzki <[email protected]>Reported-by "Aithal, Srikanth" <[email protected]>Reported-by Mark Brown <[email protected]>Signed-off-by Paul E. McKenney <[email protected]>Tested-by "Aithal, Srikanth" <[email protected]>
Otherwise, it looks good to me:
Best Regards,
Petr
Re: [PATCH v3 10/20] ratelimit: Allow zero ->burst to disable ratelimiting
Quoted 16 lines · Petr Mladek
Quoted 6 lines · Paul E. McKenney
If ->interval is zero, then rate-limiting will be disabled. Alternatively, if interval is greater than zero and ->burst is zero, then rate-limiting will be applied unconditionally.
Therefore, make this classification be lockless.
Quoted 8 lines
Note that although negative ->interval and ->burst happen to be treated as if they were zero, this is an accident of the current implementation. The semantics of negative values for these fields is subject to change without notice. Especially given that Bert Karwatzki determined that current calls to
___ratelimit()currently never have negative values for these fields.This commit replaces an earlier buggy versions.
Quoted 8 lines
If there was another revision then it would be nice to explicitly describe also the reason why both zero ->interval and ->burst never rate-limits. It is the state when the structure is zeroed. Some existing code relied in this behavior.
If I get it correctly then this is the difference between this and the previous version of this patch. And the previous version caused regressions described by the Links...
Very good, I will update the commit log on my next rebase.
Quoted 10 lines
Quoted 9 lines
Reported-by Bert Karwatzki <[email protected]>Reported-by "Aithal, Srikanth" <[email protected]>Reported-by Mark Brown <[email protected]>Signed-off-by Paul E. McKenney <[email protected]>Tested-by "Aithal, Srikanth" <[email protected]>
Quoted 3 lines
Otherwise, it looks good to me:
Reviewed-by Petr Mladek <[email protected]>
Thank you!!! And I will also apply this on my next rebase.
Thanx, Paul
PATCH v3 11/20 ratelimit: Force re-initialization when rate-limiting re-enabled
In reply to Paul E. McKenneyCurrently, rate limiting being disabled results in an immediate early return with no state changes. This can result in false-positive drops when re-enabling rate limiting. Therefore, mark the ratelimit_state structure "uninitialized" when rate limiting is disabled.
--- lib/ratelimit.c | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-)
lib/ratelimit.c +14 -1
@@ -35,11 +35,24 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func)unsigned long flags;int ret; +/* +* Zero interval says never limit, otherwise, non-positive burst +* says always limit. +*/ if (interval <=0 || burst <=0 ) { ret = interval ==0 || burst >0 ; +if (!(READ_ONCE (rs->flags) &RATELIMIT_INITIALIZED ) || + !raw_spin_trylock_irqsave (&rs->lock, flags)) { +if (!ret) +ratelimit_state_inc_miss (rs); +return ret; + } + +/* Force re-initialization once re-enabled. */ + rs->flags &= ~RATELIMIT_INITIALIZED ;if (!ret)ratelimit_state_inc_miss (rs); -return ret; + goto unlock_ret; }/*
Signature · 1 line
2.40.1
Re: [PATCH v3 11/20] ratelimit: Force re-initialization when rate-limiting re-enabled
Quoted 25 lines · Paul E. McKenney
Currently, rate limiting being disabled results in an immediate early return with no state changes. This can result in false-positive drops when re-enabling rate limiting. Therefore, mark the
ratelimit_statestructure "uninitialized" when rate limiting is disabled.Signed-off-by Paul E. McKenney <[email protected]>Cc Petr Mladek <[email protected]>Cc Andrew Morton <[email protected]>Cc Kuniyuki Iwashima <[email protected]>Cc Mateusz Guzik <[email protected]>Cc Steven Rostedt <[email protected]>Cc John Ogness <[email protected]>Cc Sergey Senozhatsky <[email protected]>--- lib/ratelimit.c | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-)lib/ratelimit.c +0
-0@@ -35,11 +35,24 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func)unsigned long flags;
lib/ratelimit.c int ___ratelimit(struct ratelimit_state *rs, const char *func) · line 45
Quoted 10 lines
lib/ratelimit.c +6
-0int ret; +/* +* Zero interval says never limit, otherwise, non-positive burst +* says always limit. +*/ if (interval <=0 || burst <=0 ) { ret = interval ==0 || burst >0 ; +if (!(READ_ONCE (rs->flags) &RATELIMIT_INITIALIZED ) || + !raw_spin_trylock_irqsave (&rs->lock, flags)) {
I though more about this. And I am not sure if this is safe.
We are here when the structure has not been initialized yet. The spin lock is going to be likely still initialized. In theory, it might happen in parallel:
CPU0 CPU1
___ratelimit()
if (interval <= 0 || burst <= 0)
// true on zero initialized struct
if (!(READ_ONCE(rs->flags) & RATELIMIT_INITIALIZED) ||
// same here
!raw_spin_trylock_irqsave(&rs->lock, flags)) {
// success???
ratelimit_state_init()
raw_spin_lock_init(&rs->lock);
raw_spin_unlock_irqrestore(&rs->lock, flags);BANG: Unlocked rs->lock which has been initialized in the meantime.
Note: The non-initialized structure can be used in ext4 code,
definitely, see
https://lore.kernel.org/r/[email protected]
Well, I think that it happens in the same CPU. So, it should be "safe".
Sigh, I guess that this patch is needed to fix the lib/tests/test_ratelimit.c. It works because the test modifies .burst and .interval directly.
What is a sane behavior/API?
IMHO:
1. Nobody, including ext4 code, should use non-initialized struct ratelimit_state. It is a ticking bomb.
IMHO, it should trigger a warning and the callers should get fixed.
2. Nobody, including the selftest, should modify struct ratelimit_state directly.
This patchset adds ratelimit_state_reset_interval() for this purpose. It clears ~RATELIMIT_INITIALIZED. So this patch won't be needed when the API was used in the selftest.
It was actually great that ___ratelimit() was able to handle non-initialized struct ratelimit_state without taking the bundled lock.
What do you think, please?
lib/ratelimit.c int ___ratelimit(struct ratelimit_state *rs, const char *func) · line 56
Quoted 12 lines
lib/ratelimit.c +8
-1+if (!ret) +ratelimit_state_inc_miss (rs); +return ret; + } + +/* Force re-initialization once re-enabled. */ + rs->flags &= ~RATELIMIT_INITIALIZED ;if (!ret)ratelimit_state_inc_miss (rs); -return ret; + goto unlock_ret; }
Best Regards,
Petr
Re: [PATCH v3 11/20] ratelimit: Force re-initialization when rate-limiting re-enabled
Quoted 74 lines · Petr Mladek
Quoted 25 lines · Paul E. McKenney
Currently, rate limiting being disabled results in an immediate early return with no state changes. This can result in false-positive drops when re-enabling rate limiting. Therefore, mark the
ratelimit_statestructure "uninitialized" when rate limiting is disabled.Signed-off-by Paul E. McKenney <[email protected]>Cc Petr Mladek <[email protected]>Cc Andrew Morton <[email protected]>Cc Kuniyuki Iwashima <[email protected]>Cc Mateusz Guzik <[email protected]>Cc Steven Rostedt <[email protected]>Cc John Ogness <[email protected]>Cc Sergey Senozhatsky <[email protected]>--- lib/ratelimit.c | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-)lib/ratelimit.c +0
-0@@ -35,11 +35,24 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func)unsigned long flags;lib/ratelimit.c int ___ratelimit(struct ratelimit_state *rs, const char *func) · line 45
Quoted 10 lines
lib/ratelimit.c +6
-0int ret; +/* +* Zero interval says never limit, otherwise, non-positive burst +* says always limit. +*/ if (interval <=0 || burst <=0 ) { ret = interval ==0 || burst >0 ; +if (!(READ_ONCE (rs->flags) &RATELIMIT_INITIALIZED ) || + !raw_spin_trylock_irqsave (&rs->lock, flags)) {I though more about this. And I am not sure if this is safe.
We are here when the structure has not been initialized yet. The spin lock is going to be likely still initialized. In theory, it might happen in parallel:
CPU0 CPU1
___ratelimit()
if (interval <= 0 || burst <= 0)// true on zero initialized struct if (!(READ_ONCE(rs->flags) & RATELIMIT_INITIALIZED) || // same here !raw_spin_trylock_irqsave(&rs->lock, flags)) { // success??? ratelimit_state_init() raw_spin_lock_init(&rs->lock); raw_spin_unlock_irqrestore(&rs->lock, flags);BANG: Unlocked
rs->lockwhich has been initialized in the meantime.Note: The non-initialized structure can be used in ext4 code,
definitely, see
https://lore.kernel.org/r/[email protected]Well, I think that it happens in the same CPU. So, it should be "safe".
Sigh, I guess that this patch is needed to fix the
lib/tests/test_ratelimit.c. It works because the test modifies .burst and .interval directly.
Quoted 8 lines
What is a sane behavior/API?
IMHO:
1. Nobody, including ext4 code, should use non-initialized struct
ratelimit_state. It is a ticking bomb.IMHO, it should trigger a warning and the callers should get fixed.
I agree in principle, but in practice it will at best take some time to drive the change into the other systems. So let's make the this code work with the existing client code, get that accepted, and then separately I can do another round of cleanup. (Or you can, if you prefer.)
After that, the WARN_ONCE() can be upgraded to complain if both interval and burst are zero.
Quoted 7 lines
2. Nobody, including the selftest, should modify struct
ratelimit_statedirectly.This patchset adds
ratelimit_state_reset_interval()for this purpose. It clears ~RATELIMIT_INITIALIZED. So this patch won't be needed when the API was used in the selftest.
Quoted 5 lines
It was actually great that
___ratelimit()was able to handle non-initialized structratelimit_statewithout taking the bundled lock.What do you think, please?
Hmmm...
It sounds like I need to avoid acquiring that lock if both interval and burst are zero. A bit inelegant, but it seems to be what needs to happen.
In the short term, how about the patch shown below?
Thanx, Paul
------------------------------------------------------------------------
lib/ratelimit.c +1 -1
@@ -42,7 +42,7 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func)if (interval <=0 || burst <=0 ) {WARN_ONCE (interval <0 || burst <0 ,"Negative interval (%d) or burst (%d): Uninitialized ratelimit_state structure?\n" , interval, burst); ret = interval ==0 || burst >0 ; -if (!(READ_ONCE (rs->flags) &RATELIMIT_INITIALIZED ) || +if (!(READ_ONCE (rs->flags) &RATELIMIT_INITIALIZED ) || (!interval && !burst) || !raw_spin_trylock_irqsave (&rs->lock, flags)) goto nolock_ret;
Re: [PATCH v3 11/20] ratelimit: Force re-initialization when rate-limiting re-enabled
Quoted 84 lines · Paul E. McKenney
Quoted 74 lines · Petr Mladek
Quoted 25 lines · Paul E. McKenney
Currently, rate limiting being disabled results in an immediate early return with no state changes. This can result in false-positive drops when re-enabling rate limiting. Therefore, mark the
ratelimit_statestructure "uninitialized" when rate limiting is disabled.Signed-off-by Paul E. McKenney <[email protected]>Cc Petr Mladek <[email protected]>Cc Andrew Morton <[email protected]>Cc Kuniyuki Iwashima <[email protected]>Cc Mateusz Guzik <[email protected]>Cc Steven Rostedt <[email protected]>Cc John Ogness <[email protected]>Cc Sergey Senozhatsky <[email protected]>--- lib/ratelimit.c | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-)lib/ratelimit.c +0
-0@@ -35,11 +35,24 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func)unsigned long flags;lib/ratelimit.c int ___ratelimit(struct ratelimit_state *rs, const char *func) · line 45
Quoted 10 lines
lib/ratelimit.c +6
-0int ret; +/* +* Zero interval says never limit, otherwise, non-positive burst +* says always limit. +*/ if (interval <=0 || burst <=0 ) { ret = interval ==0 || burst >0 ; +if (!(READ_ONCE (rs->flags) &RATELIMIT_INITIALIZED ) || + !raw_spin_trylock_irqsave (&rs->lock, flags)) {I though more about this. And I am not sure if this is safe.
We are here when the structure has not been initialized yet. The spin lock is going to be likely still initialized. In theory, it might happen in parallel:
CPU0 CPU1
___ratelimit()
if (interval <= 0 || burst <= 0)// true on zero initialized struct if (!(READ_ONCE(rs->flags) & RATELIMIT_INITIALIZED) || // same here !raw_spin_trylock_irqsave(&rs->lock, flags)) { // success??? ratelimit_state_init() raw_spin_lock_init(&rs->lock); raw_spin_unlock_irqrestore(&rs->lock, flags);BANG: Unlocked
rs->lockwhich has been initialized in the meantime.Note: The non-initialized structure can be used in ext4 code,
definitely, see
https://lore.kernel.org/r/[email protected]Well, I think that it happens in the same CPU. So, it should be "safe".
Sigh, I guess that this patch is needed to fix the
lib/tests/test_ratelimit.c. It works because the test modifies .burst and .interval directly.Quoted 8 lines
What is a sane behavior/API?
IMHO:
1. Nobody, including ext4 code, should use non-initialized struct
ratelimit_state. It is a ticking bomb.IMHO, it should trigger a warning and the callers should get fixed.
Quoted 7 lines
I agree in principle, but in practice it will at best take some time to drive the change into the other systems. So let's make the this code work with the existing client code, get that accepted, and then separately I can do another round of cleanup. (Or you can, if you prefer.)
After that, the
WARN_ONCE()can be upgraded to complain if both interval and burst are zero.
Fair enough. This patchset already is huge...
Quoted 29 lines
Quoted 7 lines
2. Nobody, including the selftest, should modify struct
ratelimit_statedirectly.This patchset adds
ratelimit_state_reset_interval()for this purpose. It clears ~RATELIMIT_INITIALIZED. So this patch won't be needed when the API was used in the selftest.Quoted 5 lines
It was actually great that
___ratelimit()was able to handle non-initialized structratelimit_statewithout taking the bundled lock.What do you think, please?
Hmmm...
It sounds like I need to avoid acquiring that lock if both interval and burst are zero. A bit inelegant, but it seems to be what needs to happen.
In the short term, how about the patch shown below?
Thanx, Paul------------------------------------------------------------------------
lib/ratelimit.c +0
-0
lib/ratelimit.c int ___ratelimit(struct ratelimit_state *rs, const char *func) · line 48
Quoted 8 lines
lib/ratelimit.c +1
-1@@ -42,7 +42,7 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func)if (interval <=0 || burst <=0 ) {WARN_ONCE (interval <0 || burst <0 ,"Negative interval (%d) or burst (%d): Uninitialized ratelimit_state structure?\n" , interval, burst); ret = interval ==0 || burst >0 ; -if (!(READ_ONCE (rs->flags) &RATELIMIT_INITIALIZED ) || +if (!(READ_ONCE (rs->flags) &RATELIMIT_INITIALIZED ) || (!interval && !burst) || !raw_spin_trylock_irqsave (&rs->lock, flags)) goto nolock_ret;
It looks good as a short term solution.
Let's see how long it would stay in the code. ;-)
Best Regards,
Petr
PS: I am not sure how much you are motivated on doing a further
clean up. You did great changes. But I guess that it has already went much further than you expected.
I could continue when time permits. But I rather do not promise anything.
Re: [PATCH v3 11/20] ratelimit: Force re-initialization when rate-limiting re-enabled
Quoted 119 lines · Petr Mladek
Quoted 84 lines · Paul E. McKenney
Quoted 74 lines · Petr Mladek
Quoted 25 lines · Paul E. McKenney
Currently, rate limiting being disabled results in an immediate early return with no state changes. This can result in false-positive drops when re-enabling rate limiting. Therefore, mark the
ratelimit_statestructure "uninitialized" when rate limiting is disabled.Signed-off-by Paul E. McKenney <[email protected]>Cc Petr Mladek <[email protected]>Cc Andrew Morton <[email protected]>Cc Kuniyuki Iwashima <[email protected]>Cc Mateusz Guzik <[email protected]>Cc Steven Rostedt <[email protected]>Cc John Ogness <[email protected]>Cc Sergey Senozhatsky <[email protected]>--- lib/ratelimit.c | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-)lib/ratelimit.c +0
-0@@ -35,11 +35,24 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func)unsigned long flags;lib/ratelimit.c int ___ratelimit(struct ratelimit_state *rs, const char *func) · line 45
Quoted 10 lines
lib/ratelimit.c +6
-0int ret; +/* +* Zero interval says never limit, otherwise, non-positive burst +* says always limit. +*/ if (interval <=0 || burst <=0 ) { ret = interval ==0 || burst >0 ; +if (!(READ_ONCE (rs->flags) &RATELIMIT_INITIALIZED ) || + !raw_spin_trylock_irqsave (&rs->lock, flags)) {I though more about this. And I am not sure if this is safe.
We are here when the structure has not been initialized yet. The spin lock is going to be likely still initialized. In theory, it might happen in parallel:
CPU0 CPU1
___ratelimit()
if (interval <= 0 || burst <= 0)// true on zero initialized struct if (!(READ_ONCE(rs->flags) & RATELIMIT_INITIALIZED) || // same here !raw_spin_trylock_irqsave(&rs->lock, flags)) { // success??? ratelimit_state_init() raw_spin_lock_init(&rs->lock); raw_spin_unlock_irqrestore(&rs->lock, flags);BANG: Unlocked
rs->lockwhich has been initialized in the meantime.Note: The non-initialized structure can be used in ext4 code,
definitely, see
https://lore.kernel.org/r/[email protected]Well, I think that it happens in the same CPU. So, it should be "safe".
Sigh, I guess that this patch is needed to fix the
lib/tests/test_ratelimit.c. It works because the test modifies .burst and .interval directly.Quoted 8 lines
What is a sane behavior/API?
IMHO:
1. Nobody, including ext4 code, should use non-initialized struct
ratelimit_state. It is a ticking bomb.IMHO, it should trigger a warning and the callers should get fixed.
Quoted 7 lines
I agree in principle, but in practice it will at best take some time to drive the change into the other systems. So let's make the this code work with the existing client code, get that accepted, and then separately I can do another round of cleanup. (Or you can, if you prefer.)
After that, the
WARN_ONCE()can be upgraded to complain if both interval and burst are zero.Fair enough. This patchset already is huge...
Quoted 24 lines
Quoted 7 lines
2. Nobody, including the selftest, should modify struct
ratelimit_statedirectly.This patchset adds
ratelimit_state_reset_interval()for this purpose. It clears ~RATELIMIT_INITIALIZED. So this patch won't be needed when the API was used in the selftest.Quoted 5 lines
It was actually great that
___ratelimit()was able to handle non-initialized structratelimit_statewithout taking the bundled lock.What do you think, please?
Hmmm...
It sounds like I need to avoid acquiring that lock if both interval and burst are zero. A bit inelegant, but it seems to be what needs to happen.
In the short term, how about the patch shown below?
Thanx, Paul------------------------------------------------------------------------
Quoted 15 lines
lib/ratelimit.c int ___ratelimit(struct ratelimit_state *rs, const char *func) · line 48
Quoted 13 lines
lib/ratelimit.c +1
-1@@ -42,7 +42,7 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func)if (interval <=0 || burst <=0 ) {WARN_ONCE (interval <0 || burst <0 ,"Negative interval (%d) or burst (%d): Uninitialized ratelimit_state structure?\n" , interval, burst); ret = interval ==0 || burst >0 ; -if (!(READ_ONCE (rs->flags) &RATELIMIT_INITIALIZED ) || +if (!(READ_ONCE (rs->flags) &RATELIMIT_INITIALIZED ) || (!interval && !burst) || !raw_spin_trylock_irqsave (&rs->lock, flags)) goto nolock_ret;It looks good as a short term solution.
Thank you!
Quoted 1 line
Let's see how long it would stay in the code. ;-)
Excellent question. ;-)
Quoted 6 lines
Best Regards,
PetrPS: I am not sure how much you are motivated on doing a further
clean up. You did great changes. But I guess that it has already went much further than you expected.
And this will be some fun due to the fact that the external modifications of the ->burst and ->interval fields are often via sysfs or similar. And these tend to take pointers to these fields.
From what I can see, these are currently validated on a field-by-field basis, but we would want to check both fields. This is because it would be OK to set either field to zero, but not both of them.
Except that if both fields are zero, we need the lock to be initialized first. But that won't play nicely with a subsequent initialization of the lock.
So some care may be required on the journey from where we are to a more prinicipled destination. ;-)
Quoted 2 lines
I could continue when time permits. But I rather do not promise anything.
My current thought is to get what I have set up. I would of course be more than happy to let you do the subsequent work. But let's see how things look at that point.
Thanx, Paul
PATCH v3 12/20 ratelimit: Don't flush misses counter if RATELIMIT_MSG_ON_RELEASE
In reply to Paul E. McKenneyRestore the previous semantics where the misses counter is unchanged if the RATELIMIT_MSG_ON_RELEASE flag is set.
--- lib/ratelimit.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-)
lib/ratelimit.c +3 -3
@@ -92,9 +92,9 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func)atomic_set (&rs->rs_n_left, rs->burst); rs->begin = jiffies; - m =ratelimit_state_reset_miss (rs); -if (m) { -if (!(rs->flags &RATELIMIT_MSG_ON_RELEASE )) { +if (!(rs->flags &RATELIMIT_MSG_ON_RELEASE )) { + m =ratelimit_state_reset_miss (rs); +if (m) {printk_deferred (KERN_WARNING "%s: %d callbacks suppressed\n" , func, m); }
Signature · 1 line
2.40.1
PATCH v3 13/20 ratelimit: Avoid atomic decrement if already rate-limited
In reply to Paul E. McKenneyCurrently, if the lock could not be acquired, the code unconditionally does an atomic decrement on ->rs_n_left, even if that atomic operation is guaranteed to return a limit-rate verdict. This incurs needless overhead and also raises the spectre of counter wrap.
Therefore, do the atomic decrement only if there is some chance that rates won't be limited.
--- lib/ratelimit.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-)
lib/ratelimit.c +3 -1
@@ -65,8 +65,10 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func)unsigned int rs_flags =READ_ONCE (rs->flags);if (rs_flags &RATELIMIT_INITIALIZED && burst) { -int n_left; +int n_left =atomic_read (&rs->rs_n_left); +if (n_left <=0 ) +return 0 ; n_left =atomic_dec_return (&rs->rs_n_left);if (n_left >=0 )return 1 ;
Signature · 1 line
2.40.1
PATCH v3 14/20 ratelimit: Avoid atomic decrement under lock if already rate-limited
In reply to Paul E. McKenneyCurrently, if the lock is acquired, the code unconditionally does an atomic decrement on ->rs_n_left, even if that atomic operation is guaranteed to return a limit-rate verdict. A limit-rate verdict will in fact be the common case when something is spewing into a rate limit. This unconditional atomic operation incurs needless overhead and also raises the spectre of counter wrap.
Therefore, do the atomic decrement only if there is some chance that rates won't be limited.
--- lib/ratelimit.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-)
lib/ratelimit.c +8 -5
@@ -103,13 +103,16 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func) } }if (burst) { -int n_left; +int n_left =atomic_read (&rs->rs_n_left);/* The burst might have been taken by a parallel call. */ - n_left =atomic_dec_return (&rs->rs_n_left); -if (n_left >=0 ) { - ret =1 ; - goto unlock_ret; + +if (n_left >0 ) { + n_left =atomic_dec_return (&rs->rs_n_left); +if (n_left >=0 ) { + ret =1 ; + goto unlock_ret; + } } }
Signature · 1 line
2.40.1
PATCH v3 15/20 ratelimit: Warn if ->interval or ->burst are negative
In reply to Paul E. McKenneyFrom: Petr Mladek <[email protected]>
Currently, ___ratelimit() treats a negative ->interval or ->burst as if it was zero, but this is an accident of the current implementation. Therefore, splat in this case, which might have the benefit of detecting use of uninitialized ratelimit_state structures on the one hand or easing addition of new features on the other.
--- lib/ratelimit.c | 1 + 1 file changed, 1 insertion(+)
lib/ratelimit.c +1 -0
@@ -40,6 +40,7 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func)* says always limit. */ if (interval <=0 || burst <=0 ) { +WARN_ONCE (interval <0 || burst <0 ,"Negative interval (%d) or burst (%d): Uninitialized ratelimit_state structure?\n" , interval, burst); ret = interval ==0 || burst >0 ;if (!(READ_ONCE (rs->flags) &RATELIMIT_INITIALIZED ) || !raw_spin_trylock_irqsave (&rs->lock, flags)) {
Signature · 1 line
2.40.1
Re: [PATCH v3 15/20] ratelimit: Warn if ->interval or ->burst are negative
Quoted 27 lines · Paul E. McKenney
From: Petr Mladek <[email protected]>
Currently,
___ratelimit()treats a negative ->interval or ->burst as if it was zero, but this is an accident of the current implementation. Therefore, splat in this case, which might have the benefit of detecting use of uninitializedratelimit_statestructures on the one hand or easing addition of new features on the other.Signed-off-by Petr Mladek <[email protected]>Signed-off-by Paul E. McKenney <[email protected]>Cc Andrew Morton <[email protected]>Cc Kuniyuki Iwashima <[email protected]>Cc Mateusz Guzik <[email protected]>Cc Steven Rostedt <[email protected]>Cc John Ogness <[email protected]>Cc Sergey Senozhatsky <[email protected]>--- lib/ratelimit.c | 1 + 1 file changed, 1 insertion(+)lib/ratelimit.c +0
-0
lib/ratelimit.c int ___ratelimit(struct ratelimit_state *rs, const char *func) · line 43
Quoted 4 lines
lib/ratelimit.c +1
-0@@ -40,6 +40,7 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func)* says always limit. */ if (interval <=0 || burst <=0 ) { +WARN_ONCE (interval <0 || burst <0 ,"Negative interval (%d) or burst (%d): Uninitialized ratelimit_state structure?\n" , interval, burst);
Just for record, I agree with having this patch in this form in this series.
That said, I think that we should warn even about using zero initialized structure in the long term because of a possible use of to-be-initialized spin lock. But it would require fixing existing users and it is beyond scope of this patchset. It is related to the discussion at
https://lore.kernel.org/r/[email protected] .
Best Regards,
Petr
Re: [PATCH v3 15/20] ratelimit: Warn if ->interval or ->burst are negative
Quoted 36 lines · Petr Mladek
Quoted 27 lines · Paul E. McKenney
From: Petr Mladek <[email protected]>
Currently,
___ratelimit()treats a negative ->interval or ->burst as if it was zero, but this is an accident of the current implementation. Therefore, splat in this case, which might have the benefit of detecting use of uninitializedratelimit_statestructures on the one hand or easing addition of new features on the other.Signed-off-by Petr Mladek <[email protected]>Signed-off-by Paul E. McKenney <[email protected]>Cc Andrew Morton <[email protected]>Cc Kuniyuki Iwashima <[email protected]>Cc Mateusz Guzik <[email protected]>Cc Steven Rostedt <[email protected]>Cc John Ogness <[email protected]>Cc Sergey Senozhatsky <[email protected]>--- lib/ratelimit.c | 1 + 1 file changed, 1 insertion(+)lib/ratelimit.c +0
-0lib/ratelimit.c int ___ratelimit(struct ratelimit_state *rs, const char *func) · line 43
Quoted 4 lines
lib/ratelimit.c +1
-0@@ -40,6 +40,7 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func)* says always limit. */ if (interval <=0 || burst <=0 ) { +WARN_ONCE (interval <0 || burst <0 ,"Negative interval (%d) or burst (%d): Uninitialized ratelimit_state structure?\n" , interval, burst);Just for record, I agree with having this patch in this form in this series.
Quoted 6 lines
That said, I think that we should warn even about using zero initialized structure in the long term because of a possible use of to-be-initialized spin lock. But it would require fixing existing users and it is beyond scope of this patchset. It is related to the discussion at
https://lore.kernel.org/r/[email protected] .
Agreed, it would be good to get to a more principled destination.
In contrast, this series is what I can come up with that deals with the current ratelimit clients without too much violence to the current code base, but we can clearly do better longer term.
And thank you for the Reviewed-by on the other patches! I will apply these later today, Pacific Time.
Thanx, Paul
PATCH v3 16/20 ratelimit: Simplify common-case exit path
In reply to Paul E. McKenneyBy making "ret" always be initialized, and moving the final call to ratelimit_state_inc_miss() out from under the lock, we save a goto and a couple lines of code. This also saves a couple of lines of code from the unconditional enable/disable slowpath.
--- lib/ratelimit.c | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-)
lib/ratelimit.c +5 -9
@@ -33,7 +33,7 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func)int interval =READ_ONCE (rs->interval);int burst =READ_ONCE (rs->burst);unsigned long flags; -int ret; +int ret =0 ;/* * Zero interval says never limit, otherwise, non-positive burst
@@ -51,8 +51,6 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func)/* Force re-initialization once re-enabled. */ rs->flags &= ~RATELIMIT_INITIALIZED ; -if (!ret) -ratelimit_state_inc_miss (rs); goto unlock_ret; }
@@ -110,19 +108,17 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func)if (n_left >0 ) { n_left =atomic_dec_return (&rs->rs_n_left); -if (n_left >=0 ) { +if (n_left >=0 ) ret =1 ; - goto unlock_ret; - } } } -ratelimit_state_inc_miss (rs); - ret =0 ; - unlock_ret:raw_spin_unlock_irqrestore (&rs->lock, flags); +if (!ret) +ratelimit_state_inc_miss (rs); +return ret; }EXPORT_SYMBOL (___ratelimit);
Signature · 1 line
2.40.1
Re: [PATCH v3 16/20] ratelimit: Simplify common-case exit path
Quoted 8 lines · Paul E. McKenney
By making "ret" always be initialized, and moving the final call to
ratelimit_state_inc_miss()out from under the lock, we save a goto and a couple lines of code. This also saves a couple of lines of code from the unconditional enable/disable slowpath.
Looks good to me:
Best Regards,
Petr
PATCH v3 17/20 ratelimit: Use nolock_ret label to save a couple of lines of code
In reply to Paul E. McKenneyCreate a nolock_ret label in order to start consolidating the unlocked return paths that separately invoke ratelimit_state_inc_miss() or not.
--- lib/ratelimit.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-)
lib/ratelimit.c +3 -5
@@ -43,11 +43,8 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func)WARN_ONCE (interval <0 || burst <0 ,"Negative interval (%d) or burst (%d): Uninitialized ratelimit_state structure?\n" , interval, burst); ret = interval ==0 || burst >0 ;if (!(READ_ONCE (rs->flags) &RATELIMIT_INITIALIZED ) || - !raw_spin_trylock_irqsave (&rs->lock, flags)) { -if (!ret) -ratelimit_state_inc_miss (rs); -return ret; - } + !raw_spin_trylock_irqsave (&rs->lock, flags)) + goto nolock_ret;/* Force re-initialization once re-enabled. */ rs->flags &= ~RATELIMIT_INITIALIZED ;
@@ -116,6 +113,7 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func) unlock_ret:raw_spin_unlock_irqrestore (&rs->lock, flags); +nolock_ret:if (!ret)ratelimit_state_inc_miss (rs);
Signature · 1 line
2.40.1
Re: [PATCH v3 17/20] ratelimit: Use nolock_ret label to save a couple of lines of code
Quoted 6 lines · Paul E. McKenney
Create a
nolock_retlabel in order to start consolidating the unlocked return paths that separately invokeratelimit_state_inc_miss()or not.
Looks good to me:
Best Regards,
Petr
PATCH v3 18/20 ratelimit: Use nolock_ret label to collapse lock-failure code
In reply to Paul E. McKenneyNow that we have a nolock_ret label that handles ->missed correctly based on the value of ret, we can eliminate a local variable and collapse several "if" statements on the lock-acquisition-failure code path.
--- lib/ratelimit.c | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-)
lib/ratelimit.c +4 -14
@@ -58,20 +58,10 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func)* the current lock owner is just about to reset it. */ if (!raw_spin_trylock_irqsave (&rs->lock, flags)) { -unsigned int rs_flags =READ_ONCE (rs->flags); - -if (rs_flags &RATELIMIT_INITIALIZED && burst) { -int n_left =atomic_read (&rs->rs_n_left); - -if (n_left <=0 ) -return 0 ; - n_left =atomic_dec_return (&rs->rs_n_left); -if (n_left >=0 ) -return 1 ; - } - -ratelimit_state_inc_miss (rs); -return 0 ; +if (READ_ONCE (rs->flags) &RATELIMIT_INITIALIZED && burst && +atomic_read (&rs->rs_n_left) >0 &&atomic_dec_return (&rs->rs_n_left) >=0 ) + ret =1 ; + goto nolock_ret; }if (!(rs->flags &RATELIMIT_INITIALIZED )) {
Signature · 1 line
2.40.1
Re: [PATCH v3 18/20] ratelimit: Use nolock_ret label to collapse lock-failure code
Quoted 7 lines · Paul E. McKenney
Now that we have a
nolock_retlabel that handles ->missed correctly based on the value of ret, we can eliminate a local variable and collapse several "if" statements on the lock-acquisition-failure code path.
Looks good to me:
Best Regards,
Petr
PATCH v3 19/20 ratelimit: Use nolock_ret restructuring to collapse common case code
In reply to Paul E. McKenneyNow that unlock_ret releases the lock, then falls into nolock_ret, which handles ->missed based on the value of ret, the common-case lock-held code can be collapsed into a single "if" statement with a single-statement "then" clause.
Yes, we could go further and just assign the "if" condition to ret, but in the immortal words of MSDOS, "Are you sure?".
--- lib/ratelimit.c | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-)
lib/ratelimit.c +3 -10
@@ -88,17 +88,10 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func) } } } -if (burst) { -int n_left =atomic_read (&rs->rs_n_left); -/* The burst might have been taken by a parallel call. */ - -if (n_left >0 ) { - n_left =atomic_dec_return (&rs->rs_n_left); -if (n_left >=0 ) - ret =1 ; - } - } +/* Note that the burst might be taken by a parallel call. */ +if (burst &&atomic_read (&rs->rs_n_left) >0 &&atomic_dec_return (&rs->rs_n_left) >=0 ) + ret =1 ; unlock_ret:raw_spin_unlock_irqrestore (&rs->lock, flags);
Signature · 1 line
2.40.1
Re: [PATCH v3 19/20] ratelimit: Use nolock_ret restructuring to collapse common case code
Quoted 11 lines · Paul E. McKenney
Now that
unlock_retreleases the lock, then falls intonolock_ret, which handles ->missed based on the value of ret, the common-case lock-held code can be collapsed into a single "if" statement with a single-statement "then" clause.Yes, we could go further and just assign the "if" condition to ret, but in the immortal words of MSDOS, "Are you sure?".
Looks good to me:
Best Regards,
Petr
PATCH v3 20/20 ratelimit: Drop redundant accesses to burst
In reply to Paul E. McKenneyNow that there is the "burst <= 0" fastpath, for all later code, burst must be strictly greater than zero. Therefore, drop the redundant checks of this local variable.
--- lib/ratelimit.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
lib/ratelimit.c +2 -2
@@ -58,7 +58,7 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func)* the current lock owner is just about to reset it. */ if (!raw_spin_trylock_irqsave (&rs->lock, flags)) { -if (READ_ONCE (rs->flags) &RATELIMIT_INITIALIZED && burst && +if (READ_ONCE (rs->flags) &RATELIMIT_INITIALIZED &&atomic_read (&rs->rs_n_left) >0 &&atomic_dec_return (&rs->rs_n_left) >=0 ) ret =1 ; goto nolock_ret;
@@ -90,7 +90,7 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func) }/* Note that the burst might be taken by a parallel call. */ -if (burst &&atomic_read (&rs->rs_n_left) >0 &&atomic_dec_return (&rs->rs_n_left) >=0 ) +if (atomic_read (&rs->rs_n_left) >0 &&atomic_dec_return (&rs->rs_n_left) >=0 ) ret =1 ; unlock_ret:
Signature · 1 line
2.40.1
v4 cover ratelimit: Reduce ratelimit's false-positive misses
In reply to Paul E. McKenneyHello!
This v4 series replaces open-coded uses of the ratelimit_state structure with formal APIs, counts all rate-limit misses, replaces jiffies=0 special case with a flag, provides a ___ratelimit() trylock-failure fastpath to (almost) eliminate false-positive misses, simplifies the code, and adds a simple test.
The key point of this series is the reduction of false-positive misses. More could be done to avoid open-coded access to the ->interval and ->burst fields, and to tighten up checking of user input for these fields, but those are jobs for later patches.
The individual patches are as follows:
1. Add trivial kunit test for ratelimit.
2. Create functions to handle ratelimit_state internals.
3. Avoid open-coded use of ratelimit_state structure's ->missed
field.
4. Avoid open-coded use of ratelimit_state structure's ->missed
field.
5. Avoid open-coded use of ratelimit_state structure's internals.
6. Convert the ->missed field to atomic_t.
7. Count misses due to lock contention.
8. Avoid jiffies=0 special case.
9. Reduce ___ratelimit() false-positive rate limiting, courtesy of
Petr Mladek.
10. Allow zero ->burst to disable ratelimiting.
11. Force re-initialization when rate-limiting re-enabled.
12. Don't flush misses counter if RATELIMIT_MSG_ON_RELEASE.
13. Avoid atomic decrement if already rate-limited.
14. Avoid atomic decrement under lock if already rate-limited.
15. Warn if ->interval or ->burst are negative, courtesy of Petr
Mladek.
16. Simplify common-case exit path.
17. Use nolock_ret label to save a couple of lines of code.
18. Use nolock_ret label to collapse lock-failure code.
19. Use nolock_ret restructuring to collapse common case code.
20. Drop redundant accesses to burst.
Changes since v3:
o Correctly handle zero-initialized ratelimit_state structures,
being careful to avoid acquiring the uninitialized ->lock.
o Remove redundant checks of the "burst" local variable.
o Add Reviewed-by tags.
Changes since v2:
o Apply feedback from Bert Karwatzki, Srikanth Aithal, and Mark
Brown, fixing a hang that happened on some systems.
o Applied Reviewed-by tags and added links.
o Added a prototype patch from Petr Mladek that splats if either
interval or burst are negative.
o Added several commits that simplify the code.
Changes since v1 (RFC):
o Switch from lockless fastpath to carrying out needed updates
upon trylock failure, per Petr Mladek feedback. This greatly simplifies the code and is a much smaller change from the current code. There is a small performance penalty compared to the lockless fastpath, but not enough to matter.
o Never unconditionally acquire the lock, again per Petr Mladek
feedback.
o Better define effects of non-positive burst values (always
ratelimit) and non-positive interval values (never ratelimit when the burst value is positive).
o The changes from Petr's original are supplied as five incremental
patches, but could easily be folded into Petr's original if desired. (Left to my lazy self, they stay as-is.)
Thanx, Paul
------------------------------------------------------------------------
b/drivers/char/random.c | 9 + b/drivers/gpu/drm/amd/pm/amdgpu_pm.c | 11 -- b/drivers/gpu/drm/i915/i915_perf.c | 8 - b/include/linux/ratelimit.h | 40 +++++++- b/include/linux/ratelimit_types.h | 2 b/lib/Kconfig.debug | 11 ++ b/lib/ratelimit.c | 8 - b/lib/tests/Makefile | 1 b/lib/tests/test_ratelimit.c | 79 ++++++++++++++++ include/linux/ratelimit.h | 13 +- include/linux/ratelimit_types.h | 3 lib/ratelimit.c | 165 ++++++++++++++++++++--------------- 12 files changed, 246 insertions(+), 104 deletions(-)
PATCH v4 1/20 lib: Add trivial kunit test for ratelimit
Add a simple single-threaded smoke test for lib/ratelimit.c
To run on x86:
make ARCH=x86_64 mrproper ./tools/testing/kunit/kunit.py run --arch x86_64 --kconfig_add CONFIG_RATELIMIT_KUNIT_TEST=y --kconfig_add CONFIG_SMP=y lib_ratelimit
This will fail on old ___ratelimit(), and subsequent patches provide the fixes that are required.
[ paulmck: Apply timeout and kunit feedback from Petr Mladek. ]
--- lib/Kconfig.debug | 11 ++++++ lib/tests/Makefile | 1 + lib/tests/test_ratelimit.c | 79 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 91 insertions(+) create mode 100644 lib/tests/test_ratelimit.c
lib/Kconfig.debug +11 -0
@@ -3232,6 +3232,17 @@ config TEST_OBJPOOL If unsure, say N. +config RATELIMIT_KUNIT_TEST + tristate "KUnit Test for correctness and stress of ratelimit" if !KUNIT_ALL_TESTS + depends on KUNIT + default KUNIT_ALL_TESTS + help + This builds the "test_ratelimit" module that should be used + for correctness verification and concurrent testings of rate + limiting. + + If unsure, say N. + config INT_POW_KUNIT_TEST tristate "Integer exponentiation (int_pow) test" if !KUNIT_ALL_TESTS depends on KUNIT
lib/tests/Makefile +1 -0
@@ -45,5 +45,6 @@ obj-$(CONFIG_STRING_KUNIT_TEST) += string_kunit.o obj-$(CONFIG_STRING_HELPERS_KUNIT_TEST) += string_helpers_kunit.o obj-$(CONFIG_USERCOPY_KUNIT_TEST) += usercopy_kunit.o obj-$(CONFIG_UTIL_MACROS_KUNIT) += util_macros_kunit.o +obj-$(CONFIG_RATELIMIT_KUNIT_TEST) += test_ratelimit.o obj-$(CONFIG_TEST_RUNTIME_MODULE) += module/
lib/tests/test_ratelimit.c new file +79 -0
79 lines not on this page · raw message
Signature · 1 line
2.40.1
PATCH v4 2/20 ratelimit: Create functions to handle ratelimit_state internals
In reply to Paul E. McKenneyA number of ratelimit use cases do open-coded access to the ratelimit_state structure's ->missed field. This works, but is a bit messy and makes it more annoying to make changes to this field.
Therefore, provide a ratelimit_state_inc_miss() function that increments the ->missed field, a ratelimit_state_get_miss() function that reads out the ->missed field, and a ratelimit_state_reset_miss() function that reads out that field, but that also resets its value to zero. These functions will replace client-code open-coded uses of ->missed.
In addition, a new ratelimit_state_reset_interval() function encapsulates what was previously open-coded lock acquisition and direct field updates.
[ paulmck: Apply kernel test robot feedback. ]
--- include/linux/ratelimit.h | 40 ++++++++++++++++++++++++++++++++++----- lib/ratelimit.c | 8 ++++---- 2 files changed, 39 insertions(+), 9 deletions(-)
include/linux/ratelimit.h +35 -5
@@ -22,16 +22,46 @@ static inline void ratelimit_default_init(struct ratelimit_state *rs) DEFAULT_RATELIMIT_BURST); } +static inline void ratelimit_state_inc_miss (struct ratelimit_state *rs) +{ + rs->missed++; +} + +static inline int ratelimit_state_get_miss (struct ratelimit_state *rs) +{ +return rs->missed; +} + +static inline int ratelimit_state_reset_miss (struct ratelimit_state *rs) +{ +int ret = rs->missed; + + rs->missed =0 ; +return ret; +} + +static inline void ratelimit_state_reset_interval (struct ratelimit_state *rs,int interval_init) +{ +unsigned long flags; + +raw_spin_lock_irqsave (&rs->lock, flags); + rs->interval = interval_init; + rs->begin =0 ; + rs->printed =0 ; +ratelimit_state_reset_miss (rs); +raw_spin_unlock_irqrestore (&rs->lock, flags); +} +static inline void ratelimit_state_exit (struct ratelimit_state *rs) { +int m; +if (!(rs->flags &RATELIMIT_MSG_ON_RELEASE ))return ; -if (rs->missed) { -pr_warn ("%s: %d output lines suppressed due to ratelimiting\n" , - current->comm, rs->missed); - rs->missed =0 ; - } + m =ratelimit_state_reset_miss (rs); +if (m) +pr_warn ("%s: %d output lines suppressed due to ratelimiting\n" , current->comm, m); } static inline void
lib/ratelimit.c +4 -4
@@ -51,12 +51,12 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func) rs->begin = jiffies;if (time_is_before_jiffies (rs->begin + interval)) { -if (rs->missed) { +int m =ratelimit_state_reset_miss (rs); + +if (m) {if (!(rs->flags &RATELIMIT_MSG_ON_RELEASE )) {printk_deferred (KERN_WARNING -"%s: %d callbacks suppressed\n" , - func, rs->missed); - rs->missed =0 ; +"%s: %d callbacks suppressed\n" , func, m); } } rs->begin = jiffies;
Signature · 1 line
2.40.1
PATCH v4 3/20 random: Avoid open-coded use of ratelimit_state structure's ->missed field
In reply to Paul E. McKenneyThe _credit_init_bits() function directly accesses the ratelimit_state structure's ->missed field, which works, but which also makes it more difficult to change this field. Therefore, make use of the ratelimit_state_get_miss() and ratelimit_state_inc_miss() functions instead of directly accessing the ->missed field.
"Jason A. Donenfeld" <[email protected]>
--- drivers/char/random.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-)
drivers/char/random.c +5 -4
@@ -726,6 +726,7 @@ static void __cold _credit_init_bits(size_t bits) static DECLARE_WORK(set_ready, crng_set_ready);unsigned int new, orig, add;unsigned long flags; +int m;if (!bits)return ;
@@ -748,9 +749,9 @@ static void __cold _credit_init_bits(size_t bits)wake_up_interruptible (&crng_init_wait);kill_fasync (&fasync,SIGIO ,POLL_IN );pr_notice ("crng init done\n" ); -if (urandom_warning.missed) -pr_notice ("%d urandom warning(s) missed due to ratelimiting\n" , - urandom_warning.missed); + m =ratelimit_state_get_miss (&urandom_warning); +if (m) +pr_notice ("%d urandom warning(s) missed due to ratelimiting\n" , m); }else if (orig <POOL_EARLY_BITS && new >=POOL_EARLY_BITS ) {spin_lock_irqsave (&base_crng.lock, flags);/* Check if crng_init is CRNG_EMPTY, to avoid race with crng_reseed(). */
@@ -1466,7 +1467,7 @@ static ssize_t urandom_read_iter(struct kiocb *kiocb, struct iov_iter *iter)if (!crng_ready ()) {if (!ratelimit_disable && maxwarn <=0 ) - ++urandom_warning.missed; +ratelimit_state_inc_miss (&urandom_warning);else if (ratelimit_disable ||__ratelimit (&urandom_warning)) { --maxwarn; pr_notice("%s: uninitialized urandom read (%zu bytes read)\n",
Signature · 1 line
2.40.1
PATCH v4 4/20 drm/i915: Avoid open-coded use of ratelimit_state structure's ->missed field
In reply to Paul E. McKenneyThe i915_oa_stream_destroy() function directly accesses the ratelimit_state structure's ->missed field, which works, but which also makes it more difficult to change this field. Therefore, make use of the ratelimit_state_get_miss() function instead of directly accessing the ->missed field.
--- drivers/gpu/drm/i915/i915_perf.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-)
drivers/gpu/drm/i915/i915_perf.c +4 -4
@@ -1666,6 +1666,7 @@ static void i915_oa_stream_destroy(struct i915_perf_stream *stream)struct i915_perf *perf = stream->perf;struct intel_gt *gt = stream->engine->gt;struct i915_perf_group *g = stream->engine->oa_group; +int m;if (WARN_ON (stream != g->exclusive_stream))return ;
@@ -1690,10 +1691,9 @@ static void i915_oa_stream_destroy(struct i915_perf_stream *stream)free_oa_configs (stream);free_noa_wait (stream); -if (perf->spurious_report_rs.missed) { -gt_notice (gt,"%d spurious OA report notices suppressed due to ratelimiting\n" , - perf->spurious_report_rs.missed); - } + m =ratelimit_state_get_miss (&perf->spurious_report_rs); +if (m) +gt_notice (gt,"%d spurious OA report notices suppressed due to ratelimiting\n" , m); }static void gen7_init_oa_buffer (struct i915_perf_stream *stream)
Signature · 1 line
2.40.1
PATCH v4 5/20 drm/amd/pm: Avoid open-coded use of ratelimit_state structure's internals
In reply to Paul E. McKenneyThe amdgpu_set_thermal_throttling_logging() function directly accesses the ratelimit_state structure's ->missed field, which works, but which also makes it more difficult to change this field. Therefore, make use of the ratelimit_state_reset_interval() function instead of directly accessing the ->missed field.
Nevertheless, open-coded use of ->burst and ->interval is still permitted, for example, for runtime sysfs adjustment of these fields.
--- drivers/gpu/drm/amd/pm/amdgpu_pm.c | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-)
drivers/gpu/drm/amd/pm/amdgpu_pm.c +2 -9
@@ -1606,7 +1606,6 @@ static ssize_t amdgpu_set_thermal_throttling_logging(struct device *dev,struct drm_device *ddev =dev_get_drvdata (dev);struct amdgpu_device *adev =drm_to_adev (ddev);long throttling_logging_interval; -unsigned long flags;int ret =0 ; ret =kstrtol (buf,0 , &throttling_logging_interval);
@@ -1617,18 +1616,12 @@ static ssize_t amdgpu_set_thermal_throttling_logging(struct device *dev,return -EINVAL ;if (throttling_logging_interval >0 ) { -raw_spin_lock_irqsave (&adev->throttling_logging_rs.lock, flags);/* * Reset the ratelimit timer internals. * This can effectively restart the timer. */ - adev->throttling_logging_rs.interval = - (throttling_logging_interval -1 ) *HZ ; - adev->throttling_logging_rs.begin =0 ; - adev->throttling_logging_rs.printed =0 ; - adev->throttling_logging_rs.missed =0 ; -raw_spin_unlock_irqrestore (&adev->throttling_logging_rs.lock, flags); - +ratelimit_state_reset_interval (&adev->throttling_logging_rs, + (throttling_logging_interval -1 ) *HZ );atomic_set (&adev->throttling_logging_enabled,1 ); }else {atomic_set (&adev->throttling_logging_enabled,0 );
Signature · 1 line
2.40.1
PATCH v4 6/20 ratelimit: Convert the ->missed field to atomic_t
In reply to Paul E. McKenneyThe ratelimit_state structure's ->missed field is sometimes incremented locklessly, and it would be good to avoid lost counts. This is also needed to count the number of misses due to trylock failure. Therefore, convert the ratelimit_state structure's ->missed field to atomic_t.
--- include/linux/ratelimit.h | 9 +++------ include/linux/ratelimit_types.h | 2 +- lib/ratelimit.c | 2 +- 3 files changed, 5 insertions(+), 8 deletions(-)
include/linux/ratelimit.h +3 -6
@@ -24,20 +24,17 @@ static inline void ratelimit_default_init(struct ratelimit_state *rs)static inline void ratelimit_state_inc_miss (struct ratelimit_state *rs) { - rs->missed++; +atomic_inc (&rs->missed); }static inline int ratelimit_state_get_miss (struct ratelimit_state *rs) { -return rs->missed; +return atomic_read (&rs->missed); }static inline int ratelimit_state_reset_miss (struct ratelimit_state *rs) { -int ret = rs->missed; - - rs->missed =0 ; -return ret; +return atomic_xchg_relaxed (&rs->missed,0 ); }static inline void ratelimit_state_reset_interval (struct ratelimit_state *rs,int interval_init)
include/linux/ratelimit_types.h +1 -1
@@ -18,7 +18,7 @@ struct ratelimit_state {int interval;int burst;int printed; -int missed; +atomic_t missed;unsigned int flags;unsigned long begin; };
lib/ratelimit.c +1 -1
@@ -66,7 +66,7 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func) rs->printed++; ret =1 ; }else { - rs->missed++; +ratelimit_state_inc_miss (rs); ret =0 ; }raw_spin_unlock_irqrestore (&rs->lock, flags);
Signature · 1 line
2.40.1
PATCH v4 7/20 ratelimit: Count misses due to lock contention
In reply to Paul E. McKenneyThe ___ratelimit() function simply returns zero ("do ratelimiting") if the trylock fails, but does not adjust the ->missed field. This means that the resulting dropped printk()s are dropped silently, which could seriously confuse people trying to do console-log-based debugging. Therefore, increment the ->missed field upon trylock failure.
--- lib/ratelimit.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-)
lib/ratelimit.c +3 -1
@@ -44,8 +44,10 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func)* in addition to the one that will be printed by * the entity that is holding the lock already: */ -if (!raw_spin_trylock_irqsave (&rs->lock, flags)) +if (!raw_spin_trylock_irqsave (&rs->lock, flags)) { +ratelimit_state_inc_miss (rs);return 0 ; + }if (!rs->begin) rs->begin = jiffies;
Signature · 1 line
2.40.1
PATCH v4 8/20 ratelimit: Avoid jiffies=0 special case
In reply to Paul E. McKenneyThe ___ratelimit() function special-cases the jiffies-counter value of zero as "uninitialized". This works well on 64-bit systems, where the jiffies counter is not going to return to zero for more than half a billion years on systems with HZ=1000, but similar 32-bit systems take less than 50 days to wrap the jiffies counter. And although the consequences of wrapping the jiffies counter seem to be limited to minor confusion on the duration of the rate-limiting interval that happens to end at time zero, it is almost no work to avoid this confusion.
Therefore, introduce a RATELIMIT_INITIALIZED bit to the ratelimit_state structure's ->flags field so that a ->begin value of zero is no longer special.
--- include/linux/ratelimit.h | 2 +- include/linux/ratelimit_types.h | 1 + lib/ratelimit.c | 4 +++- 3 files changed, 5 insertions(+), 2 deletions(-)
include/linux/ratelimit.h +1 -1
@@ -43,7 +43,7 @@ static inline void ratelimit_state_reset_interval(struct ratelimit_state *rs, inraw_spin_lock_irqsave (&rs->lock, flags); rs->interval = interval_init; - rs->begin =0 ; + rs->flags &= ~RATELIMIT_INITIALIZED ; rs->printed =0 ;ratelimit_state_reset_miss (rs);raw_spin_unlock_irqrestore (&rs->lock, flags);
include/linux/ratelimit_types.h +1 -0
@@ -11,6 +11,7 @@/* issue num suppressed message on exit */ #define RATELIMIT_MSG_ON_RELEASE BIT(0) +#define RATELIMIT_INITIALIZED BIT(1)struct ratelimit_state {raw_spinlock_t lock;/* protect the state */
lib/ratelimit.c +3 -1
@@ -49,8 +49,10 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func)return 0 ; } -if (!rs->begin) +if (!(rs->flags &RATELIMIT_INITIALIZED )) { rs->begin = jiffies; + rs->flags |=RATELIMIT_INITIALIZED ; + }if (time_is_before_jiffies (rs->begin + interval)) {int m =ratelimit_state_reset_miss (rs);
Signature · 1 line
2.40.1
PATCH v4 9/20 ratelimit: Reduce ___ratelimit() false-positive rate limiting
In reply to Paul E. McKenneyFrom: Petr Mladek <[email protected]>
Retain the locked design, but check rate-limiting even when the lock could not be acquired.
--- include/linux/ratelimit.h | 2 +- include/linux/ratelimit_types.h | 2 +- lib/ratelimit.c | 51 ++++++++++++++++++++++++--------- 3 files changed, 40 insertions(+), 15 deletions(-)
include/linux/ratelimit.h +1 -1
@@ -44,7 +44,7 @@ static inline void ratelimit_state_reset_interval(struct ratelimit_state *rs, inraw_spin_lock_irqsave (&rs->lock, flags); rs->interval = interval_init; rs->flags &= ~RATELIMIT_INITIALIZED ; - rs->printed =0 ; +atomic_set (&rs->rs_n_left, rs->burst);ratelimit_state_reset_miss (rs);raw_spin_unlock_irqrestore (&rs->lock, flags); }
include/linux/ratelimit_types.h +1 -1
@@ -18,7 +18,7 @@ struct ratelimit_state {int interval;int burst; -int printed; +atomic_t rs_n_left;atomic_t missed;unsigned int flags;unsigned long begin;
lib/ratelimit.c +38 -13
77 lines not on this page · raw message
Signature · 1 line
2.40.1
PATCH v4 10/20 ratelimit: Allow zero ->burst to disable ratelimiting
In reply to Paul E. McKenneyIf ->interval is zero, then rate-limiting will be disabled. Alternatively, if interval is greater than zero and ->burst is zero, then rate-limiting will be applied unconditionally. The point of this distinction is to handle current users that pass zero-initialized ratelimit_state structures to ___ratelimit(), and in such cases the ->lock field will be uninitialized. Acquiring ->lock in this case is clearly not a strategy to win.
Therefore, make this classification be lockless.
Note that although negative ->interval and ->burst happen to be treated as if they were zero, this is an accident of the current implementation. The semantics of negative values for these fields is subject to change without notice. Especially given that Bert Karwatzki determined that no current calls to ___ratelimit() ever have negative values for these fields.
This commit replaces an earlier buggy versions.
--- lib/ratelimit.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-)
lib/ratelimit.c +6 -2
@@ -35,8 +35,12 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func)unsigned long flags;int ret; -if (!interval) -return 1 ; +if (interval <=0 || burst <=0 ) { + ret = interval ==0 || burst >0 ; +if (!ret) +ratelimit_state_inc_miss (rs); +return ret; + }/* * If we contend on this state's lock then just check if
Signature · 1 line
2.40.1
PATCH v4 11/20 ratelimit: Force re-initialization when rate-limiting re-enabled
In reply to Paul E. McKenneyCurrently, if rate limiting is disabled, ___ratelimit() does an immediate early return with no state changes. This can result in false-positive drops when re-enabling rate limiting. Therefore, mark the ratelimit_state structure "uninitialized" when rate limiting is disabled.
[ paulmck: Apply Petr Mladek feedback. ]
--- lib/ratelimit.c | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-)
lib/ratelimit.c +14 -1
@@ -35,11 +35,24 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func)unsigned long flags;int ret; +/* +* Zero interval says never limit, otherwise, non-positive burst +* says always limit. +*/ if (interval <=0 || burst <=0 ) { ret = interval ==0 || burst >0 ; +if (!(READ_ONCE (rs->flags) &RATELIMIT_INITIALIZED ) || (!interval && !burst) || + !raw_spin_trylock_irqsave (&rs->lock, flags)) { +if (!ret) +ratelimit_state_inc_miss (rs); +return ret; + } + +/* Force re-initialization once re-enabled. */ + rs->flags &= ~RATELIMIT_INITIALIZED ;if (!ret)ratelimit_state_inc_miss (rs); -return ret; + goto unlock_ret; }/*
Signature · 1 line
2.40.1
Re: [PATCH v4 11/20] ratelimit: Force re-initialization when rate-limiting re-enabled
Quoted 7 lines · Paul E. McKenney
Currently, if rate limiting is disabled,
___ratelimit()does an immediate early return with no state changes. This can result in false-positive drops when re-enabling rate limiting. Therefore, mark theratelimit_statestructure "uninitialized" when rate limiting is disabled.[ paulmck: Apply Petr Mladek feedback. ]
Quoted 10 lines
Signed-off-by Paul E. McKenney <[email protected]>Cc Petr Mladek <[email protected]>Cc Andrew Morton <[email protected]>Cc Kuniyuki Iwashima <[email protected]>Cc Mateusz Guzik <[email protected]>Cc Steven Rostedt <[email protected]>Cc John Ogness <[email protected]>Cc Sergey Senozhatsky <[email protected]>
Looks goot to me:
Best Regards,
Petr
PATCH v4 12/20 ratelimit: Don't flush misses counter if RATELIMIT_MSG_ON_RELEASE
In reply to Paul E. McKenneyRestore the previous semantics where the misses counter is unchanged if the RATELIMIT_MSG_ON_RELEASE flag is set.
--- lib/ratelimit.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-)
lib/ratelimit.c +3 -3
@@ -92,9 +92,9 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func)atomic_set (&rs->rs_n_left, rs->burst); rs->begin = jiffies; - m =ratelimit_state_reset_miss (rs); -if (m) { -if (!(rs->flags &RATELIMIT_MSG_ON_RELEASE )) { +if (!(rs->flags &RATELIMIT_MSG_ON_RELEASE )) { + m =ratelimit_state_reset_miss (rs); +if (m) {printk_deferred (KERN_WARNING "%s: %d callbacks suppressed\n" , func, m); }
Signature · 1 line
2.40.1
PATCH v4 13/20 ratelimit: Avoid atomic decrement if already rate-limited
In reply to Paul E. McKenneyCurrently, if the lock could not be acquired, the code unconditionally does an atomic decrement on ->rs_n_left, even if that atomic operation is guaranteed to return a limit-rate verdict. This incurs needless overhead and also raises the spectre of counter wrap.
Therefore, do the atomic decrement only if there is some chance that rates won't be limited.
--- lib/ratelimit.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-)
lib/ratelimit.c +3 -1
@@ -65,8 +65,10 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func)unsigned int rs_flags =READ_ONCE (rs->flags);if (rs_flags &RATELIMIT_INITIALIZED && burst) { -int n_left; +int n_left =atomic_read (&rs->rs_n_left); +if (n_left <=0 ) +return 0 ; n_left =atomic_dec_return (&rs->rs_n_left);if (n_left >=0 )return 1 ;
Signature · 1 line
2.40.1
PATCH v4 14/20 ratelimit: Avoid atomic decrement under lock if already rate-limited
In reply to Paul E. McKenneyCurrently, if the lock is acquired, the code unconditionally does an atomic decrement on ->rs_n_left, even if that atomic operation is guaranteed to return a limit-rate verdict. A limit-rate verdict will in fact be the common case when something is spewing into a rate limit. This unconditional atomic operation incurs needless overhead and also raises the spectre of counter wrap.
Therefore, do the atomic decrement only if there is some chance that rates won't be limited.
--- lib/ratelimit.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-)
lib/ratelimit.c +8 -5
@@ -103,13 +103,16 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func) } }if (burst) { -int n_left; +int n_left =atomic_read (&rs->rs_n_left);/* The burst might have been taken by a parallel call. */ - n_left =atomic_dec_return (&rs->rs_n_left); -if (n_left >=0 ) { - ret =1 ; - goto unlock_ret; + +if (n_left >0 ) { + n_left =atomic_dec_return (&rs->rs_n_left); +if (n_left >=0 ) { + ret =1 ; + goto unlock_ret; + } } }
Signature · 1 line
2.40.1
PATCH v4 15/20 ratelimit: Warn if ->interval or ->burst are negative
In reply to Paul E. McKenneyFrom: Petr Mladek <[email protected]>
Currently, ___ratelimit() treats a negative ->interval or ->burst as if it was zero, but this is an accident of the current implementation. Therefore, splat in this case, which might have the benefit of detecting use of uninitialized ratelimit_state structures on the one hand or easing addition of new features on the other.
--- lib/ratelimit.c | 1 + 1 file changed, 1 insertion(+)
lib/ratelimit.c +1 -0
@@ -40,6 +40,7 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func)* says always limit. */ if (interval <=0 || burst <=0 ) { +WARN_ONCE (interval <0 || burst <0 ,"Negative interval (%d) or burst (%d): Uninitialized ratelimit_state structure?\n" , interval, burst); ret = interval ==0 || burst >0 ;if (!(READ_ONCE (rs->flags) &RATELIMIT_INITIALIZED ) || (!interval && !burst) || !raw_spin_trylock_irqsave (&rs->lock, flags)) {
Signature · 1 line
2.40.1
PATCH v4 16/20 ratelimit: Simplify common-case exit path
In reply to Paul E. McKenneyBy making "ret" always be initialized, and moving the final call to ratelimit_state_inc_miss() out from under the lock, we save a goto and a couple lines of code. This also saves a couple of lines of code from the unconditional enable/disable slowpath.
--- lib/ratelimit.c | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-)
lib/ratelimit.c +5 -9
@@ -33,7 +33,7 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func)int interval =READ_ONCE (rs->interval);int burst =READ_ONCE (rs->burst);unsigned long flags; -int ret; +int ret =0 ;/* * Zero interval says never limit, otherwise, non-positive burst
@@ -51,8 +51,6 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func)/* Force re-initialization once re-enabled. */ rs->flags &= ~RATELIMIT_INITIALIZED ; -if (!ret) -ratelimit_state_inc_miss (rs); goto unlock_ret; }
@@ -110,19 +108,17 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func)if (n_left >0 ) { n_left =atomic_dec_return (&rs->rs_n_left); -if (n_left >=0 ) { +if (n_left >=0 ) ret =1 ; - goto unlock_ret; - } } } -ratelimit_state_inc_miss (rs); - ret =0 ; - unlock_ret:raw_spin_unlock_irqrestore (&rs->lock, flags); +if (!ret) +ratelimit_state_inc_miss (rs); +return ret; }EXPORT_SYMBOL (___ratelimit);
Signature · 1 line
2.40.1
PATCH v4 17/20 ratelimit: Use nolock_ret label to save a couple of lines of code
In reply to Paul E. McKenneyCreate a nolock_ret label in order to start consolidating the unlocked return paths that conditionally invoke ratelimit_state_inc_miss().
--- lib/ratelimit.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-)
lib/ratelimit.c +3 -5
@@ -43,11 +43,8 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func)WARN_ONCE (interval <0 || burst <0 ,"Negative interval (%d) or burst (%d): Uninitialized ratelimit_state structure?\n" , interval, burst); ret = interval ==0 || burst >0 ;if (!(READ_ONCE (rs->flags) &RATELIMIT_INITIALIZED ) || (!interval && !burst) || - !raw_spin_trylock_irqsave (&rs->lock, flags)) { -if (!ret) -ratelimit_state_inc_miss (rs); -return ret; - } + !raw_spin_trylock_irqsave (&rs->lock, flags)) + goto nolock_ret;/* Force re-initialization once re-enabled. */ rs->flags &= ~RATELIMIT_INITIALIZED ;
@@ -116,6 +113,7 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func) unlock_ret:raw_spin_unlock_irqrestore (&rs->lock, flags); +nolock_ret:if (!ret)ratelimit_state_inc_miss (rs);
Signature · 1 line
2.40.1
PATCH v4 18/20 ratelimit: Use nolock_ret label to collapse lock-failure code
In reply to Paul E. McKenneyNow that we have a nolock_ret label that handles ->missed correctly based on the value of ret, we can eliminate a local variable and collapse several "if" statements on the lock-acquisition-failure code path.
--- lib/ratelimit.c | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-)
lib/ratelimit.c +4 -14
@@ -58,20 +58,10 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func)* the current lock owner is just about to reset it. */ if (!raw_spin_trylock_irqsave (&rs->lock, flags)) { -unsigned int rs_flags =READ_ONCE (rs->flags); - -if (rs_flags &RATELIMIT_INITIALIZED && burst) { -int n_left =atomic_read (&rs->rs_n_left); - -if (n_left <=0 ) -return 0 ; - n_left =atomic_dec_return (&rs->rs_n_left); -if (n_left >=0 ) -return 1 ; - } - -ratelimit_state_inc_miss (rs); -return 0 ; +if (READ_ONCE (rs->flags) &RATELIMIT_INITIALIZED && burst && +atomic_read (&rs->rs_n_left) >0 &&atomic_dec_return (&rs->rs_n_left) >=0 ) + ret =1 ; + goto nolock_ret; }if (!(rs->flags &RATELIMIT_INITIALIZED )) {
Signature · 1 line
2.40.1
PATCH v4 19/20 ratelimit: Use nolock_ret restructuring to collapse common case code
In reply to Paul E. McKenneyNow that unlock_ret releases the lock, then falls into nolock_ret, which handles ->missed based on the value of ret, the common-case lock-held code can be collapsed into a single "if" statement with a single-statement "then" clause.
Yes, we could go further and just assign the "if" condition to ret, but in the immortal words of MSDOS, "Are you sure?".
--- lib/ratelimit.c | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-)
lib/ratelimit.c +3 -10
@@ -88,17 +88,10 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func) } } } -if (burst) { -int n_left =atomic_read (&rs->rs_n_left); -/* The burst might have been taken by a parallel call. */ - -if (n_left >0 ) { - n_left =atomic_dec_return (&rs->rs_n_left); -if (n_left >=0 ) - ret =1 ; - } - } +/* Note that the burst might be taken by a parallel call. */ +if (burst &&atomic_read (&rs->rs_n_left) >0 &&atomic_dec_return (&rs->rs_n_left) >=0 ) + ret =1 ; unlock_ret:raw_spin_unlock_irqrestore (&rs->lock, flags);
Signature · 1 line
2.40.1
PATCH v4 20/20 ratelimit: Drop redundant accesses to burst
In reply to Paul E. McKenneyNow that there is the "burst <= 0" fastpath, for all later code, burst must be strictly greater than zero. Therefore, drop the redundant checks of this local variable.
--- lib/ratelimit.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
lib/ratelimit.c +2 -2
@@ -58,7 +58,7 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func)* the current lock owner is just about to reset it. */ if (!raw_spin_trylock_irqsave (&rs->lock, flags)) { -if (READ_ONCE (rs->flags) &RATELIMIT_INITIALIZED && burst && +if (READ_ONCE (rs->flags) &RATELIMIT_INITIALIZED &&atomic_read (&rs->rs_n_left) >0 &&atomic_dec_return (&rs->rs_n_left) >=0 ) ret =1 ; goto nolock_ret;
@@ -90,7 +90,7 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func) }/* Note that the burst might be taken by a parallel call. */ -if (burst &&atomic_read (&rs->rs_n_left) >0 &&atomic_dec_return (&rs->rs_n_left) >=0 ) +if (atomic_read (&rs->rs_n_left) >0 &&atomic_dec_return (&rs->rs_n_left) >=0 ) ret =1 ; unlock_ret:
Signature · 1 line
2.40.1
Re: [PATCH v4 20/20] ratelimit: Drop redundant accesses to burst
Quoted 7 lines · Paul E. McKenney
Now that there is the "burst <= 0" fastpath, for all later code, burst must be strictly greater than zero. Therefore, drop the redundant checks of this local variable.
Looks good to me:
Best Regards,
Petr
Re: [PATCH v4 0/20] ratelimit: Reduce ratelimit's false-positive misses
In reply to Paul E. McKenneyQuoted 8 lines · Paul E. McKenney
Hello!
This v4 series replaces open-coded uses of the
ratelimit_statestructure with formal APIs, counts all rate-limit misses, replaces jiffies=0 special case with a flag, provides a___ratelimit()trylock-failure fastpath to (almost) eliminate false-positive misses, simplifies the code, and adds a simple test.
Quoted 4 lines
The key point of this series is the reduction of false-positive misses. More could be done to avoid open-coded access to the ->interval and ->burst fields, and to tighten up checking of user input for these fields, but those are jobs for later patches.
JFYI, the whole series looks good to me.
Best Regards,
Petr
Re: [PATCH v4 0/20] ratelimit: Reduce ratelimit's false-positive misses
Quoted 9 lines · Petr Mladek
Quoted 8 lines · Paul E. McKenney
Hello!
This v4 series replaces open-coded uses of the
ratelimit_statestructure with formal APIs, counts all rate-limit misses, replaces jiffies=0 special case with a flag, provides a___ratelimit()trylock-failure fastpath to (almost) eliminate false-positive misses, simplifies the code, and adds a simple test.
Quoted 6 lines
Quoted 4 lines
The key point of this series is the reduction of false-positive misses. More could be done to avoid open-coded access to the ->interval and ->burst fields, and to tighten up checking of user input for these fields, but those are jobs for later patches.
JFYI, the whole series looks good to me.
I double-checked, and after I apply these two Reviewed-by's, each patch in the series will have either your Signed-off-by or your Reviewed-by, so thank you for your reviews and feedback!
Thanx, Paul
v5 cover ratelimit: Reduce ratelimit's false-positive misses
In reply to Paul E. McKenneyHello!
This v5 series replaces open-coded uses of the ratelimit_state structure with formal APIs, counts all rate-limit misses, replaces jiffies=0 special case with a flag, provides a ___ratelimit() trylock-failure fastpath to (almost) eliminate false-positive misses, simplifies the code, and adds a simple "smoke" test along with a simple stress test.
The key point of this series is the reduction of false-positive misses. More could be done to avoid open-coded access to the ->interval and ->burst fields, and to tighten up checking of user input for these fields, but those are jobs for later patches.
The individual patches are as follows:
1. Create functions to handle ratelimit_state internals.
2. Avoid open-coded use of ratelimit_state structure's ->missed
field.
3. Avoid open-coded use of ratelimit_state structure's ->missed
field.
4. Avoid open-coded use of ratelimit_state structure's internals.
5. Convert the ->missed field to atomic_t.
6. Count misses due to lock contention.
7. Avoid jiffies=0 special case.
8. Reduce ___ratelimit() false-positive rate limiting, courtesy of
Petr Mladek.
9. Allow zero ->burst to disable ratelimiting.
10. Force re-initialization when rate-limiting re-enabled.
11. Don't flush misses counter if RATELIMIT_MSG_ON_RELEASE.
12. Avoid atomic decrement if already rate-limited.
13. Avoid atomic decrement under lock if already rate-limited.
14. Warn if ->interval or ->burst are negative, courtesy of Petr
Mladek.
15. Simplify common-case exit path.
16. Use nolock_ret label to save a couple of lines of code.
17. Use nolock_ret label to collapse lock-failure code.
18. Use nolock_ret restructuring to collapse common case code.
19. Drop redundant accesses to burst.
20. Add trivial kunit test for ratelimit.
21. Add stress test for ratelimit.
Changes since v4:
o Add a simple stress test.
o Move the tests to the end of the series for bisectability.
o Add Reviewed-by tags.
Changes since v3:
o Correctly handle zero-initialized ratelimit_state structures,
being careful to avoid acquiring the uninitialized ->lock.
o Remove redundant checks of the "burst" local variable.
o Add Reviewed-by tags.
Changes since v2:
o Apply feedback from Bert Karwatzki, Srikanth Aithal, and Mark
Brown, fixing a hang that happened on some systems.
o Applied Reviewed-by tags and added links.
o Added a prototype patch from Petr Mladek that splats if either
interval or burst are negative.
o Added several commits that simplify the code.
Changes since v1 (RFC):
o Switch from lockless fastpath to carrying out needed updates
upon trylock failure, per Petr Mladek feedback. This greatly simplifies the code and is a much smaller change from the current code. There is a small performance penalty compared to the lockless fastpath, but not enough to matter.
o Never unconditionally acquire the lock, again per Petr Mladek
feedback.
o Better define effects of non-positive burst values (always
ratelimit) and non-positive interval values (never ratelimit when the burst value is positive).
o The changes from Petr's original are supplied as five incremental
patches, but could easily be folded into Petr's original if desired. (Left to my lazy self, they stay as-is.)
Thanx, Paul
------------------------------------------------------------------------
b/drivers/char/random.c | 9 + b/drivers/gpu/drm/amd/pm/amdgpu_pm.c | 11 -- b/drivers/gpu/drm/i915/i915_perf.c | 8 - b/include/linux/ratelimit.h | 40 +++++++- b/include/linux/ratelimit_types.h | 2 b/lib/Kconfig.debug | 11 ++ b/lib/ratelimit.c | 8 - b/lib/tests/Makefile | 1 b/lib/tests/test_ratelimit.c | 79 ++++++++++++++++ include/linux/ratelimit.h | 13 +- include/linux/ratelimit_types.h | 3 lib/ratelimit.c | 165 ++++++++++++++++++++--------------- lib/tests/test_ratelimit.c | 69 ++++++++++++++ 13 files changed, 313 insertions(+), 106 deletions(-)
PATCH v5 1/21 ratelimit: Create functions to handle ratelimit_state internals
A number of ratelimit use cases do open-coded access to the ratelimit_state structure's ->missed field. This works, but is a bit messy and makes it more annoying to make changes to this field.
Therefore, provide a ratelimit_state_inc_miss() function that increments the ->missed field, a ratelimit_state_get_miss() function that reads out the ->missed field, and a ratelimit_state_reset_miss() function that reads out that field, but that also resets its value to zero. These functions will replace client-code open-coded uses of ->missed.
In addition, a new ratelimit_state_reset_interval() function encapsulates what was previously open-coded lock acquisition and direct field updates.
[ paulmck: Apply kernel test robot feedback. ]
--- include/linux/ratelimit.h | 40 ++++++++++++++++++++++++++++++++++----- lib/ratelimit.c | 8 ++++---- 2 files changed, 39 insertions(+), 9 deletions(-)
include/linux/ratelimit.h +35 -5
@@ -22,16 +22,46 @@ static inline void ratelimit_default_init(struct ratelimit_state *rs) DEFAULT_RATELIMIT_BURST); } +static inline void ratelimit_state_inc_miss (struct ratelimit_state *rs) +{ + rs->missed++; +} + +static inline int ratelimit_state_get_miss (struct ratelimit_state *rs) +{ +return rs->missed; +} + +static inline int ratelimit_state_reset_miss (struct ratelimit_state *rs) +{ +int ret = rs->missed; + + rs->missed =0 ; +return ret; +} + +static inline void ratelimit_state_reset_interval (struct ratelimit_state *rs,int interval_init) +{ +unsigned long flags; + +raw_spin_lock_irqsave (&rs->lock, flags); + rs->interval = interval_init; + rs->begin =0 ; + rs->printed =0 ; +ratelimit_state_reset_miss (rs); +raw_spin_unlock_irqrestore (&rs->lock, flags); +} +static inline void ratelimit_state_exit (struct ratelimit_state *rs) { +int m; +if (!(rs->flags &RATELIMIT_MSG_ON_RELEASE ))return ; -if (rs->missed) { -pr_warn ("%s: %d output lines suppressed due to ratelimiting\n" , - current->comm, rs->missed); - rs->missed =0 ; - } + m =ratelimit_state_reset_miss (rs); +if (m) +pr_warn ("%s: %d output lines suppressed due to ratelimiting\n" , current->comm, m); } static inline void
lib/ratelimit.c +4 -4
@@ -51,12 +51,12 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func) rs->begin = jiffies;if (time_is_before_jiffies (rs->begin + interval)) { -if (rs->missed) { +int m =ratelimit_state_reset_miss (rs); + +if (m) {if (!(rs->flags &RATELIMIT_MSG_ON_RELEASE )) {printk_deferred (KERN_WARNING -"%s: %d callbacks suppressed\n" , - func, rs->missed); - rs->missed =0 ; +"%s: %d callbacks suppressed\n" , func, m); } } rs->begin = jiffies;
Signature · 1 line
2.40.1
PATCH v5 2/21 random: Avoid open-coded use of ratelimit_state structure's ->missed field
In reply to Paul E. McKenneyThe _credit_init_bits() function directly accesses the ratelimit_state structure's ->missed field, which works, but which also makes it more difficult to change this field. Therefore, make use of the ratelimit_state_get_miss() and ratelimit_state_inc_miss() functions instead of directly accessing the ->missed field.
"Jason A. Donenfeld" <[email protected]>
--- drivers/char/random.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-)
drivers/char/random.c +5 -4
@@ -726,6 +726,7 @@ static void __cold _credit_init_bits(size_t bits) static DECLARE_WORK(set_ready, crng_set_ready);unsigned int new, orig, add;unsigned long flags; +int m;if (!bits)return ;
@@ -748,9 +749,9 @@ static void __cold _credit_init_bits(size_t bits)wake_up_interruptible (&crng_init_wait);kill_fasync (&fasync,SIGIO ,POLL_IN );pr_notice ("crng init done\n" ); -if (urandom_warning.missed) -pr_notice ("%d urandom warning(s) missed due to ratelimiting\n" , - urandom_warning.missed); + m =ratelimit_state_get_miss (&urandom_warning); +if (m) +pr_notice ("%d urandom warning(s) missed due to ratelimiting\n" , m); }else if (orig <POOL_EARLY_BITS && new >=POOL_EARLY_BITS ) {spin_lock_irqsave (&base_crng.lock, flags);/* Check if crng_init is CRNG_EMPTY, to avoid race with crng_reseed(). */
@@ -1466,7 +1467,7 @@ static ssize_t urandom_read_iter(struct kiocb *kiocb, struct iov_iter *iter)if (!crng_ready ()) {if (!ratelimit_disable && maxwarn <=0 ) - ++urandom_warning.missed; +ratelimit_state_inc_miss (&urandom_warning);else if (ratelimit_disable ||__ratelimit (&urandom_warning)) { --maxwarn; pr_notice("%s: uninitialized urandom read (%zu bytes read)\n",
Signature · 1 line
2.40.1
PATCH v5 3/21 drm/i915: Avoid open-coded use of ratelimit_state structure's ->missed field
In reply to Paul E. McKenneyThe i915_oa_stream_destroy() function directly accesses the ratelimit_state structure's ->missed field, which works, but which also makes it more difficult to change this field. Therefore, make use of the ratelimit_state_get_miss() function instead of directly accessing the ->missed field.
--- drivers/gpu/drm/i915/i915_perf.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-)
drivers/gpu/drm/i915/i915_perf.c +4 -4
@@ -1666,6 +1666,7 @@ static void i915_oa_stream_destroy(struct i915_perf_stream *stream)struct i915_perf *perf = stream->perf;struct intel_gt *gt = stream->engine->gt;struct i915_perf_group *g = stream->engine->oa_group; +int m;if (WARN_ON (stream != g->exclusive_stream))return ;
@@ -1690,10 +1691,9 @@ static void i915_oa_stream_destroy(struct i915_perf_stream *stream)free_oa_configs (stream);free_noa_wait (stream); -if (perf->spurious_report_rs.missed) { -gt_notice (gt,"%d spurious OA report notices suppressed due to ratelimiting\n" , - perf->spurious_report_rs.missed); - } + m =ratelimit_state_get_miss (&perf->spurious_report_rs); +if (m) +gt_notice (gt,"%d spurious OA report notices suppressed due to ratelimiting\n" , m); }static void gen7_init_oa_buffer (struct i915_perf_stream *stream)
Signature · 1 line
2.40.1
PATCH v5 4/21 drm/amd/pm: Avoid open-coded use of ratelimit_state structure's internals
In reply to Paul E. McKenneyThe amdgpu_set_thermal_throttling_logging() function directly accesses the ratelimit_state structure's ->missed field, which works, but which also makes it more difficult to change this field. Therefore, make use of the ratelimit_state_reset_interval() function instead of directly accessing the ->missed field.
Nevertheless, open-coded use of ->burst and ->interval is still permitted, for example, for runtime sysfs adjustment of these fields.
--- drivers/gpu/drm/amd/pm/amdgpu_pm.c | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-)
drivers/gpu/drm/amd/pm/amdgpu_pm.c +2 -9
@@ -1606,7 +1606,6 @@ static ssize_t amdgpu_set_thermal_throttling_logging(struct device *dev,struct drm_device *ddev =dev_get_drvdata (dev);struct amdgpu_device *adev =drm_to_adev (ddev);long throttling_logging_interval; -unsigned long flags;int ret =0 ; ret =kstrtol (buf,0 , &throttling_logging_interval);
@@ -1617,18 +1616,12 @@ static ssize_t amdgpu_set_thermal_throttling_logging(struct device *dev,return -EINVAL ;if (throttling_logging_interval >0 ) { -raw_spin_lock_irqsave (&adev->throttling_logging_rs.lock, flags);/* * Reset the ratelimit timer internals. * This can effectively restart the timer. */ - adev->throttling_logging_rs.interval = - (throttling_logging_interval -1 ) *HZ ; - adev->throttling_logging_rs.begin =0 ; - adev->throttling_logging_rs.printed =0 ; - adev->throttling_logging_rs.missed =0 ; -raw_spin_unlock_irqrestore (&adev->throttling_logging_rs.lock, flags); - +ratelimit_state_reset_interval (&adev->throttling_logging_rs, + (throttling_logging_interval -1 ) *HZ );atomic_set (&adev->throttling_logging_enabled,1 ); }else {atomic_set (&adev->throttling_logging_enabled,0 );
Signature · 1 line
2.40.1
PATCH v5 5/21 ratelimit: Convert the ->missed field to atomic_t
In reply to Paul E. McKenneyThe ratelimit_state structure's ->missed field is sometimes incremented locklessly, and it would be good to avoid lost counts. This is also needed to count the number of misses due to trylock failure. Therefore, convert the ratelimit_state structure's ->missed field to atomic_t.
--- include/linux/ratelimit.h | 9 +++------ include/linux/ratelimit_types.h | 2 +- lib/ratelimit.c | 2 +- 3 files changed, 5 insertions(+), 8 deletions(-)
include/linux/ratelimit.h +3 -6
@@ -24,20 +24,17 @@ static inline void ratelimit_default_init(struct ratelimit_state *rs)static inline void ratelimit_state_inc_miss (struct ratelimit_state *rs) { - rs->missed++; +atomic_inc (&rs->missed); }static inline int ratelimit_state_get_miss (struct ratelimit_state *rs) { -return rs->missed; +return atomic_read (&rs->missed); }static inline int ratelimit_state_reset_miss (struct ratelimit_state *rs) { -int ret = rs->missed; - - rs->missed =0 ; -return ret; +return atomic_xchg_relaxed (&rs->missed,0 ); }static inline void ratelimit_state_reset_interval (struct ratelimit_state *rs,int interval_init)
include/linux/ratelimit_types.h +1 -1
@@ -18,7 +18,7 @@ struct ratelimit_state {int interval;int burst;int printed; -int missed; +atomic_t missed;unsigned int flags;unsigned long begin; };
lib/ratelimit.c +1 -1
@@ -66,7 +66,7 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func) rs->printed++; ret =1 ; }else { - rs->missed++; +ratelimit_state_inc_miss (rs); ret =0 ; }raw_spin_unlock_irqrestore (&rs->lock, flags);
Signature · 1 line
2.40.1
PATCH v5 6/21 ratelimit: Count misses due to lock contention
In reply to Paul E. McKenneyThe ___ratelimit() function simply returns zero ("do ratelimiting") if the trylock fails, but does not adjust the ->missed field. This means that the resulting dropped printk()s are dropped silently, which could seriously confuse people trying to do console-log-based debugging. Therefore, increment the ->missed field upon trylock failure.
--- lib/ratelimit.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-)
lib/ratelimit.c +3 -1
@@ -44,8 +44,10 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func)* in addition to the one that will be printed by * the entity that is holding the lock already: */ -if (!raw_spin_trylock_irqsave (&rs->lock, flags)) +if (!raw_spin_trylock_irqsave (&rs->lock, flags)) { +ratelimit_state_inc_miss (rs);return 0 ; + }if (!rs->begin) rs->begin = jiffies;
Signature · 1 line
2.40.1
PATCH v5 7/21 ratelimit: Avoid jiffies=0 special case
In reply to Paul E. McKenneyThe ___ratelimit() function special-cases the jiffies-counter value of zero as "uninitialized". This works well on 64-bit systems, where the jiffies counter is not going to return to zero for more than half a billion years on systems with HZ=1000, but similar 32-bit systems take less than 50 days to wrap the jiffies counter. And although the consequences of wrapping the jiffies counter seem to be limited to minor confusion on the duration of the rate-limiting interval that happens to end at time zero, it is almost no work to avoid this confusion.
Therefore, introduce a RATELIMIT_INITIALIZED bit to the ratelimit_state structure's ->flags field so that a ->begin value of zero is no longer special.
--- include/linux/ratelimit.h | 2 +- include/linux/ratelimit_types.h | 1 + lib/ratelimit.c | 4 +++- 3 files changed, 5 insertions(+), 2 deletions(-)
include/linux/ratelimit.h +1 -1
@@ -43,7 +43,7 @@ static inline void ratelimit_state_reset_interval(struct ratelimit_state *rs, inraw_spin_lock_irqsave (&rs->lock, flags); rs->interval = interval_init; - rs->begin =0 ; + rs->flags &= ~RATELIMIT_INITIALIZED ; rs->printed =0 ;ratelimit_state_reset_miss (rs);raw_spin_unlock_irqrestore (&rs->lock, flags);
include/linux/ratelimit_types.h +1 -0
@@ -11,6 +11,7 @@/* issue num suppressed message on exit */ #define RATELIMIT_MSG_ON_RELEASE BIT(0) +#define RATELIMIT_INITIALIZED BIT(1)struct ratelimit_state {raw_spinlock_t lock;/* protect the state */
lib/ratelimit.c +3 -1
@@ -49,8 +49,10 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func)return 0 ; } -if (!rs->begin) +if (!(rs->flags &RATELIMIT_INITIALIZED )) { rs->begin = jiffies; + rs->flags |=RATELIMIT_INITIALIZED ; + }if (time_is_before_jiffies (rs->begin + interval)) {int m =ratelimit_state_reset_miss (rs);
Signature · 1 line
2.40.1
PATCH v5 8/21 ratelimit: Reduce ___ratelimit() false-positive rate limiting
In reply to Paul E. McKenneyFrom: Petr Mladek <[email protected]>
Retain the locked design, but check rate-limiting even when the lock could not be acquired.
--- include/linux/ratelimit.h | 2 +- include/linux/ratelimit_types.h | 2 +- lib/ratelimit.c | 51 ++++++++++++++++++++++++--------- 3 files changed, 40 insertions(+), 15 deletions(-)
include/linux/ratelimit.h +1 -1
@@ -44,7 +44,7 @@ static inline void ratelimit_state_reset_interval(struct ratelimit_state *rs, inraw_spin_lock_irqsave (&rs->lock, flags); rs->interval = interval_init; rs->flags &= ~RATELIMIT_INITIALIZED ; - rs->printed =0 ; +atomic_set (&rs->rs_n_left, rs->burst);ratelimit_state_reset_miss (rs);raw_spin_unlock_irqrestore (&rs->lock, flags); }
include/linux/ratelimit_types.h +1 -1
@@ -18,7 +18,7 @@ struct ratelimit_state {int interval;int burst; -int printed; +atomic_t rs_n_left;atomic_t missed;unsigned int flags;unsigned long begin;
lib/ratelimit.c +38 -13
77 lines not on this page · raw message
Signature · 1 line
2.40.1
PATCH v5 9/21 ratelimit: Allow zero ->burst to disable ratelimiting
In reply to Paul E. McKenneyIf ->interval is zero, then rate-limiting will be disabled. Alternatively, if interval is greater than zero and ->burst is zero, then rate-limiting will be applied unconditionally. The point of this distinction is to handle current users that pass zero-initialized ratelimit_state structures to ___ratelimit(), and in such cases the ->lock field will be uninitialized. Acquiring ->lock in this case is clearly not a strategy to win.
Therefore, make this classification be lockless.
Note that although negative ->interval and ->burst happen to be treated as if they were zero, this is an accident of the current implementation. The semantics of negative values for these fields is subject to change without notice. Especially given that Bert Karwatzki determined that no current calls to ___ratelimit() ever have negative values for these fields.
This commit replaces an earlier buggy versions.
--- lib/ratelimit.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-)
lib/ratelimit.c +6 -2
@@ -35,8 +35,12 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func)unsigned long flags;int ret; -if (!interval) -return 1 ; +if (interval <=0 || burst <=0 ) { + ret = interval ==0 || burst >0 ; +if (!ret) +ratelimit_state_inc_miss (rs); +return ret; + }/* * If we contend on this state's lock then just check if
Signature · 1 line
2.40.1
PATCH v5 10/21 ratelimit: Force re-initialization when rate-limiting re-enabled
In reply to Paul E. McKenneyCurrently, if rate limiting is disabled, ___ratelimit() does an immediate early return with no state changes. This can result in false-positive drops when re-enabling rate limiting. Therefore, mark the ratelimit_state structure "uninitialized" when rate limiting is disabled.
[ paulmck: Apply Petr Mladek feedback. ]
--- lib/ratelimit.c | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-)
lib/ratelimit.c +14 -1
@@ -35,11 +35,24 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func)unsigned long flags;int ret; +/* +* Zero interval says never limit, otherwise, non-positive burst +* says always limit. +*/ if (interval <=0 || burst <=0 ) { ret = interval ==0 || burst >0 ; +if (!(READ_ONCE (rs->flags) &RATELIMIT_INITIALIZED ) || (!interval && !burst) || + !raw_spin_trylock_irqsave (&rs->lock, flags)) { +if (!ret) +ratelimit_state_inc_miss (rs); +return ret; + } + +/* Force re-initialization once re-enabled. */ + rs->flags &= ~RATELIMIT_INITIALIZED ;if (!ret)ratelimit_state_inc_miss (rs); -return ret; + goto unlock_ret; }/*
Signature · 1 line
2.40.1
PATCH v5 11/21 ratelimit: Don't flush misses counter if RATELIMIT_MSG_ON_RELEASE
In reply to Paul E. McKenneyRestore the previous semantics where the misses counter is unchanged if the RATELIMIT_MSG_ON_RELEASE flag is set.
--- lib/ratelimit.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-)
lib/ratelimit.c +3 -3
@@ -92,9 +92,9 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func)atomic_set (&rs->rs_n_left, rs->burst); rs->begin = jiffies; - m =ratelimit_state_reset_miss (rs); -if (m) { -if (!(rs->flags &RATELIMIT_MSG_ON_RELEASE )) { +if (!(rs->flags &RATELIMIT_MSG_ON_RELEASE )) { + m =ratelimit_state_reset_miss (rs); +if (m) {printk_deferred (KERN_WARNING "%s: %d callbacks suppressed\n" , func, m); }
Signature · 1 line
2.40.1
PATCH v5 12/21 ratelimit: Avoid atomic decrement if already rate-limited
In reply to Paul E. McKenneyCurrently, if the lock could not be acquired, the code unconditionally does an atomic decrement on ->rs_n_left, even if that atomic operation is guaranteed to return a limit-rate verdict. This incurs needless overhead and also raises the spectre of counter wrap.
Therefore, do the atomic decrement only if there is some chance that rates won't be limited.
--- lib/ratelimit.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-)
lib/ratelimit.c +3 -1
@@ -65,8 +65,10 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func)unsigned int rs_flags =READ_ONCE (rs->flags);if (rs_flags &RATELIMIT_INITIALIZED && burst) { -int n_left; +int n_left =atomic_read (&rs->rs_n_left); +if (n_left <=0 ) +return 0 ; n_left =atomic_dec_return (&rs->rs_n_left);if (n_left >=0 )return 1 ;
Signature · 1 line
2.40.1
PATCH v5 13/21 ratelimit: Avoid atomic decrement under lock if already rate-limited
In reply to Paul E. McKenneyCurrently, if the lock is acquired, the code unconditionally does an atomic decrement on ->rs_n_left, even if that atomic operation is guaranteed to return a limit-rate verdict. A limit-rate verdict will in fact be the common case when something is spewing into a rate limit. This unconditional atomic operation incurs needless overhead and also raises the spectre of counter wrap.
Therefore, do the atomic decrement only if there is some chance that rates won't be limited.
--- lib/ratelimit.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-)
lib/ratelimit.c +8 -5
@@ -103,13 +103,16 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func) } }if (burst) { -int n_left; +int n_left =atomic_read (&rs->rs_n_left);/* The burst might have been taken by a parallel call. */ - n_left =atomic_dec_return (&rs->rs_n_left); -if (n_left >=0 ) { - ret =1 ; - goto unlock_ret; + +if (n_left >0 ) { + n_left =atomic_dec_return (&rs->rs_n_left); +if (n_left >=0 ) { + ret =1 ; + goto unlock_ret; + } } }
Signature · 1 line
2.40.1
PATCH v5 14/21 ratelimit: Warn if ->interval or ->burst are negative
In reply to Paul E. McKenneyFrom: Petr Mladek <[email protected]>
Currently, ___ratelimit() treats a negative ->interval or ->burst as if it was zero, but this is an accident of the current implementation. Therefore, splat in this case, which might have the benefit of detecting use of uninitialized ratelimit_state structures on the one hand or easing addition of new features on the other.
--- lib/ratelimit.c | 1 + 1 file changed, 1 insertion(+)
lib/ratelimit.c +1 -0
@@ -40,6 +40,7 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func)* says always limit. */ if (interval <=0 || burst <=0 ) { +WARN_ONCE (interval <0 || burst <0 ,"Negative interval (%d) or burst (%d): Uninitialized ratelimit_state structure?\n" , interval, burst); ret = interval ==0 || burst >0 ;if (!(READ_ONCE (rs->flags) &RATELIMIT_INITIALIZED ) || (!interval && !burst) || !raw_spin_trylock_irqsave (&rs->lock, flags)) {
Signature · 1 line
2.40.1
PATCH v5 15/21 ratelimit: Simplify common-case exit path
In reply to Paul E. McKenneyBy making "ret" always be initialized, and moving the final call to ratelimit_state_inc_miss() out from under the lock, we save a goto and a couple lines of code. This also saves a couple of lines of code from the unconditional enable/disable slowpath.
--- lib/ratelimit.c | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-)
lib/ratelimit.c +5 -9
@@ -33,7 +33,7 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func)int interval =READ_ONCE (rs->interval);int burst =READ_ONCE (rs->burst);unsigned long flags; -int ret; +int ret =0 ;/* * Zero interval says never limit, otherwise, non-positive burst
@@ -51,8 +51,6 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func)/* Force re-initialization once re-enabled. */ rs->flags &= ~RATELIMIT_INITIALIZED ; -if (!ret) -ratelimit_state_inc_miss (rs); goto unlock_ret; }
@@ -110,19 +108,17 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func)if (n_left >0 ) { n_left =atomic_dec_return (&rs->rs_n_left); -if (n_left >=0 ) { +if (n_left >=0 ) ret =1 ; - goto unlock_ret; - } } } -ratelimit_state_inc_miss (rs); - ret =0 ; - unlock_ret:raw_spin_unlock_irqrestore (&rs->lock, flags); +if (!ret) +ratelimit_state_inc_miss (rs); +return ret; }EXPORT_SYMBOL (___ratelimit);
Signature · 1 line
2.40.1
PATCH v5 16/21 ratelimit: Use nolock_ret label to save a couple of lines of code
In reply to Paul E. McKenneyCreate a nolock_ret label in order to start consolidating the unlocked return paths that conditionally invoke ratelimit_state_inc_miss().
--- lib/ratelimit.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-)
lib/ratelimit.c +3 -5
@@ -43,11 +43,8 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func)WARN_ONCE (interval <0 || burst <0 ,"Negative interval (%d) or burst (%d): Uninitialized ratelimit_state structure?\n" , interval, burst); ret = interval ==0 || burst >0 ;if (!(READ_ONCE (rs->flags) &RATELIMIT_INITIALIZED ) || (!interval && !burst) || - !raw_spin_trylock_irqsave (&rs->lock, flags)) { -if (!ret) -ratelimit_state_inc_miss (rs); -return ret; - } + !raw_spin_trylock_irqsave (&rs->lock, flags)) + goto nolock_ret;/* Force re-initialization once re-enabled. */ rs->flags &= ~RATELIMIT_INITIALIZED ;
@@ -116,6 +113,7 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func) unlock_ret:raw_spin_unlock_irqrestore (&rs->lock, flags); +nolock_ret:if (!ret)ratelimit_state_inc_miss (rs);
Signature · 1 line
2.40.1
PATCH v5 17/21 ratelimit: Use nolock_ret label to collapse lock-failure code
In reply to Paul E. McKenneyNow that we have a nolock_ret label that handles ->missed correctly based on the value of ret, we can eliminate a local variable and collapse several "if" statements on the lock-acquisition-failure code path.
--- lib/ratelimit.c | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-)
lib/ratelimit.c +4 -14
@@ -58,20 +58,10 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func)* the current lock owner is just about to reset it. */ if (!raw_spin_trylock_irqsave (&rs->lock, flags)) { -unsigned int rs_flags =READ_ONCE (rs->flags); - -if (rs_flags &RATELIMIT_INITIALIZED && burst) { -int n_left =atomic_read (&rs->rs_n_left); - -if (n_left <=0 ) -return 0 ; - n_left =atomic_dec_return (&rs->rs_n_left); -if (n_left >=0 ) -return 1 ; - } - -ratelimit_state_inc_miss (rs); -return 0 ; +if (READ_ONCE (rs->flags) &RATELIMIT_INITIALIZED && burst && +atomic_read (&rs->rs_n_left) >0 &&atomic_dec_return (&rs->rs_n_left) >=0 ) + ret =1 ; + goto nolock_ret; }if (!(rs->flags &RATELIMIT_INITIALIZED )) {
Signature · 1 line
2.40.1
PATCH v5 18/21 ratelimit: Use nolock_ret restructuring to collapse common case code
In reply to Paul E. McKenneyNow that unlock_ret releases the lock, then falls into nolock_ret, which handles ->missed based on the value of ret, the common-case lock-held code can be collapsed into a single "if" statement with a single-statement "then" clause.
Yes, we could go further and just assign the "if" condition to ret, but in the immortal words of MSDOS, "Are you sure?".
--- lib/ratelimit.c | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-)
lib/ratelimit.c +3 -10
@@ -88,17 +88,10 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func) } } } -if (burst) { -int n_left =atomic_read (&rs->rs_n_left); -/* The burst might have been taken by a parallel call. */ - -if (n_left >0 ) { - n_left =atomic_dec_return (&rs->rs_n_left); -if (n_left >=0 ) - ret =1 ; - } - } +/* Note that the burst might be taken by a parallel call. */ +if (burst &&atomic_read (&rs->rs_n_left) >0 &&atomic_dec_return (&rs->rs_n_left) >=0 ) + ret =1 ; unlock_ret:raw_spin_unlock_irqrestore (&rs->lock, flags);
Signature · 1 line
2.40.1
PATCH v5 19/21 ratelimit: Drop redundant accesses to burst
In reply to Paul E. McKenneyNow that there is the "burst <= 0" fastpath, for all later code, burst must be strictly greater than zero. Therefore, drop the redundant checks of this local variable.
--- lib/ratelimit.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
lib/ratelimit.c +2 -2
@@ -58,7 +58,7 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func)* the current lock owner is just about to reset it. */ if (!raw_spin_trylock_irqsave (&rs->lock, flags)) { -if (READ_ONCE (rs->flags) &RATELIMIT_INITIALIZED && burst && +if (READ_ONCE (rs->flags) &RATELIMIT_INITIALIZED &&atomic_read (&rs->rs_n_left) >0 &&atomic_dec_return (&rs->rs_n_left) >=0 ) ret =1 ; goto nolock_ret;
@@ -90,7 +90,7 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func) }/* Note that the burst might be taken by a parallel call. */ -if (burst &&atomic_read (&rs->rs_n_left) >0 &&atomic_dec_return (&rs->rs_n_left) >=0 ) +if (atomic_read (&rs->rs_n_left) >0 &&atomic_dec_return (&rs->rs_n_left) >=0 ) ret =1 ; unlock_ret:
Signature · 1 line
2.40.1
PATCH v5 20/21 lib: Add trivial kunit test for ratelimit
In reply to Paul E. McKenneyAdd a simple single-threaded smoke test for lib/ratelimit.c
To run on x86:
make ARCH=x86_64 mrproper ./tools/testing/kunit/kunit.py run --arch x86_64 --kconfig_add CONFIG_RATELIMIT_KUNIT_TEST=y --kconfig_add CONFIG_SMP=y lib_ratelimit
This will fail on old ___ratelimit(), and subsequent patches provide the fixes that are required.
[ paulmck: Apply timeout and kunit feedback from Petr Mladek. ]
--- lib/Kconfig.debug | 11 ++++++ lib/tests/Makefile | 1 + lib/tests/test_ratelimit.c | 79 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 91 insertions(+) create mode 100644 lib/tests/test_ratelimit.c
lib/Kconfig.debug +11 -0
@@ -3232,6 +3232,17 @@ config TEST_OBJPOOL If unsure, say N. +config RATELIMIT_KUNIT_TEST + tristate "KUnit Test for correctness and stress of ratelimit" if !KUNIT_ALL_TESTS + depends on KUNIT + default KUNIT_ALL_TESTS + help + This builds the "test_ratelimit" module that should be used + for correctness verification and concurrent testings of rate + limiting. + + If unsure, say N. + config INT_POW_KUNIT_TEST tristate "Integer exponentiation (int_pow) test" if !KUNIT_ALL_TESTS depends on KUNIT
lib/tests/Makefile +1 -0
@@ -45,5 +45,6 @@ obj-$(CONFIG_STRING_KUNIT_TEST) += string_kunit.o obj-$(CONFIG_STRING_HELPERS_KUNIT_TEST) += string_helpers_kunit.o obj-$(CONFIG_USERCOPY_KUNIT_TEST) += usercopy_kunit.o obj-$(CONFIG_UTIL_MACROS_KUNIT) += util_macros_kunit.o +obj-$(CONFIG_RATELIMIT_KUNIT_TEST) += test_ratelimit.o obj-$(CONFIG_TEST_RUNTIME_MODULE) += module/
lib/tests/test_ratelimit.c new file +79 -0
79 lines not on this page · raw message
Signature · 1 line
2.40.1
Re: [PATCH v5 20/21] lib: Add trivial kunit test for ratelimit
Quoted 30 lines · Paul E. McKenney
Add a simple single-threaded smoke test for
lib/ratelimit.cTo run on x86:
make ARCH=x86_64 mrproper ./tools/testing/kunit/kunit.py run --arch x86_64 --kconfig_add CONFIG_RATELIMIT_KUNIT_TEST=y --kconfig_add CONFIG_SMP=y lib_ratelimitThis will fail on old
___ratelimit(), and subsequent patches provide the fixes that are required.lib/tests/test_ratelimit.c new file +17
-0@@ -0,0 +1,79 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include <kunit/test.h> + +#include <linux/ratelimit.h> +#include <linux/module.h> + +/* a simple boot-time regression test */ + +#define TESTRL_INTERVAL (5 * HZ) +static DEFINE_RATELIMIT_STATE(testrl, TESTRL_INTERVAL, 3); + +#define test_ratelimited (test, expected) \ + KUNIT_ASSERT_EQ(test, ___ratelimit(&testrl, "test_ratelimit_smoke"), (expected)) + +static void test_ratelimit_smoke (struct kunit *test) +{
lib/tests/test_ratelimit.c line 27
Quoted 10 lines
lib/tests/test_ratelimit.c new file +10
-0+// Check settings. +KUNIT_ASSERT_GE (test,TESTRL_INTERVAL ,100 ); + +// Test normal operation. +test_ratelimited (test, true); +test_ratelimited (test, true); +test_ratelimited (test, true); +test_ratelimited (test, false); + +schedule_timeout_idle (TESTRL_INTERVAL -40 );
Heh, I have got a new laptop. The battery in the previous one was about to explode. And the test started failing on the next line most of the time.
The following change helped me:
From 005e00ca09b4bd5b4a5f3026f1835e0435ecfbd9 Mon Sep 17 00:00:00 2001
From: Petr Mladek <[email protected]>
Date: Mon, 12 May 2025 16:38:02 +0200
Subject: [PATCH] lib: Make the ratelimit test more reliable
The selftest fails most of the times when running in qemu with a kernel configured with CONFIG_HZ = 250:
Quoted 5 lines
test_ratelimit_smoke: 1 callbacks suppressed
#test_ratelimit_smoke: ASSERTION FAILED atlib/tests/test_ratelimit.c:28Expected ___ratelimit(&testrl, "test_ratelimit_smoke") == (false), but ___ratelimit(&testrl, "test_ratelimit_smoke") == 1 (0x1) (false) == 0 (0x0)
Try to make the test slightly more reliable by calling the problematic ratelimit in the middle of the interval.
--- lib/tests/test_ratelimit.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-)
lib/tests/test_ratelimit.c +4 -4
@@ -24,19 +24,19 @@ static void test_ratelimit_smoke(struct kunit *test)test_ratelimited (test, true);test_ratelimited (test, false); -schedule_timeout_idle (TESTRL_INTERVAL -40 ); +schedule_timeout_idle (TESTRL_INTERVAL /2 );test_ratelimited (test, false); -schedule_timeout_idle (50 ); +schedule_timeout_idle (TESTRL_INTERVAL *3 /4 );test_ratelimited (test, true);schedule_timeout_idle (2 *TESTRL_INTERVAL );test_ratelimited (test, true);test_ratelimited (test, true); -schedule_timeout_idle (TESTRL_INTERVAL -40 ); +schedule_timeout_idle (TESTRL_INTERVAL /2 );test_ratelimited (test, true); -schedule_timeout_idle (50 ); +schedule_timeout_idle (TESTRL_INTERVAL *3 /4 );test_ratelimited (test, true);test_ratelimited (test, true);test_ratelimited (test, true);
Signature · 3 lines
2.49.0 Feel free to squash it into the original patch which added the test.
lib/tests/test_ratelimit.c
Quoted 9 lines
lib/tests/test_ratelimit.c new file +9
-0+test_ratelimited (test, false); + +schedule_timeout_idle (50 ); +test_ratelimited (test, true); + +schedule_timeout_idle (2 *TESTRL_INTERVAL ); +test_ratelimited (test, true); +test_ratelimited (test, true); +
Best Regards,
Petr
Re: [PATCH v5 20/21] lib: Add trivial kunit test for ratelimit
Quoted 46 lines · Petr Mladek
Quoted 30 lines · Paul E. McKenney
Add a simple single-threaded smoke test for
lib/ratelimit.cTo run on x86:
make ARCH=x86_64 mrproper ./tools/testing/kunit/kunit.py run --arch x86_64 --kconfig_add CONFIG_RATELIMIT_KUNIT_TEST=y --kconfig_add CONFIG_SMP=y lib_ratelimitThis will fail on old
___ratelimit(), and subsequent patches provide the fixes that are required.lib/tests/test_ratelimit.c new file +17
-0@@ -0,0 +1,79 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include <kunit/test.h> + +#include <linux/ratelimit.h> +#include <linux/module.h> + +/* a simple boot-time regression test */ + +#define TESTRL_INTERVAL (5 * HZ) +static DEFINE_RATELIMIT_STATE(testrl, TESTRL_INTERVAL, 3); + +#define test_ratelimited (test, expected) \ + KUNIT_ASSERT_EQ(test, ___ratelimit(&testrl, "test_ratelimit_smoke"), (expected)) + +static void test_ratelimit_smoke (struct kunit *test) +{lib/tests/test_ratelimit.c line 27
Quoted 10 lines
lib/tests/test_ratelimit.c new file +10
-0+// Check settings. +KUNIT_ASSERT_GE (test,TESTRL_INTERVAL ,100 ); + +// Test normal operation. +test_ratelimited (test, true); +test_ratelimited (test, true); +test_ratelimited (test, true); +test_ratelimited (test, false); + +schedule_timeout_idle (TESTRL_INTERVAL -40 );Heh, I have got a new laptop. The battery in the previous one was about to explode. And the test started failing on the next line most of the time.
Quoted 1 line
The following change helped me:
Thank you very much! I have queued this, and intend to keep it as its own commit, following my original.
Thanx, Paul
Quoted 67 lines
Quoted 1 line
From
005e00ca09b4bd5b4a5f3026f1835e0435ecfbd9Mon Sep 17 00:00:00 2001From: Petr Mladek <[email protected]>
Date: Mon, 12 May 2025 16:38:02 +0200
Subject: [PATCH] lib: Make the ratelimit test more reliableThe selftest fails most of the times when running in qemu with a kernel configured with
CONFIG_HZ= 250:Quoted 5 lines
test_ratelimit_smoke: 1 callbacks suppressed
#test_ratelimit_smoke: ASSERTION FAILED atlib/tests/test_ratelimit.c:28Expected ___ratelimit(&testrl, "test_ratelimit_smoke") == (false), but ___ratelimit(&testrl, "test_ratelimit_smoke") == 1 (0x1) (false) == 0 (0x0)Try to make the test slightly more reliable by calling the problematic ratelimit in the middle of the interval.
Signed-off-by Petr Mladek <[email protected]>--- lib/tests/test_ratelimit.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-)lib/tests/test_ratelimit.c +4
-4@@ -24,19 +24,19 @@ static void test_ratelimit_smoke(struct kunit *test)test_ratelimited (test, true);test_ratelimited (test, false); -schedule_timeout_idle (TESTRL_INTERVAL -40 ); +schedule_timeout_idle (TESTRL_INTERVAL /2 );test_ratelimited (test, false); -schedule_timeout_idle (50 ); +schedule_timeout_idle (TESTRL_INTERVAL *3 /4 );test_ratelimited (test, true);schedule_timeout_idle (2 *TESTRL_INTERVAL );test_ratelimited (test, true);test_ratelimited (test, true); -schedule_timeout_idle (TESTRL_INTERVAL -40 ); +schedule_timeout_idle (TESTRL_INTERVAL /2 );test_ratelimited (test, true); -schedule_timeout_idle (50 ); +schedule_timeout_idle (TESTRL_INTERVAL *3 /4 );test_ratelimited (test, true);test_ratelimited (test, true);test_ratelimited (test, true);Signature · 3 lines
2.49.0 Feel free to squash it into the original patch which added the test.lib/tests/test_ratelimit.c
Quoted 9 lines
lib/tests/test_ratelimit.c new file +9
-0+test_ratelimited (test, false); + +schedule_timeout_idle (50 ); +test_ratelimited (test, true); + +schedule_timeout_idle (2 *TESTRL_INTERVAL ); +test_ratelimited (test, true); +test_ratelimited (test, true); +Best Regards,
Petr
PATCH v5 21/21 lib: Add stress test for ratelimit
In reply to Paul E. McKenneyAdd a simple stress test for lib/ratelimit.c
To run on x86:
./tools/testing/kunit/kunit.py run --arch x86_64 --kconfig_add CONFIG_RATELIMIT_KUNIT_TEST=y --kconfig_add CONFIG_SMP=y --qemu_args "-smp 4" lib_ratelimit
On a 16-CPU system, the "4" in "-smp 4" can be varied between 1 and 8. Larger numbers have higher probabilities of introducing delays that break the smoke test. In the extreme case, increasing the number to larger than the number of CPUs in the underlying system is an excellent way to get a test failure.
--- lib/tests/test_ratelimit.c | 69 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 67 insertions(+), 2 deletions(-)
lib/tests/test_ratelimit.c +67 -2
@@ -4,6 +4,8 @@#include <linux/ratelimit.h> #include <linux/module.h> +#include <linux/kthread.h> +#include <linux/cpumask.h> /* a simple boot-time regression test */
@@ -63,14 +65,77 @@ static void test_ratelimit_smoke(struct kunit *test)test_ratelimited (test, false); } -static struct kunit_case sort_test_cases[] = { +static struct ratelimit_state stressrl =RATELIMIT_STATE_INIT_FLAGS ("stressrl" ,HZ /10 ,3 , +RATELIMIT_MSG_ON_RELEASE ); + +static int doneflag; +static const int stress_duration =2 *HZ ; + +struct stress_kthread { +unsigned long nattempts; +unsigned long nunlimited; +unsigned long nlimited; +unsigned long nmissed; +struct task_struct *tp; +}; + +static int test_ratelimit_stress_child (void *arg) +{ +struct stress_kthread *sktp = arg; + +set_user_nice (current,MAX_NICE ); +WARN_ON_ONCE (!sktp->tp); + +while (!READ_ONCE (doneflag)) { + sktp->nattempts++; +if (___ratelimit (&stressrl, __func__)) + sktp->nunlimited++; +else + sktp->nlimited++; +cond_resched (); + } + + sktp->nmissed =ratelimit_state_reset_miss (&stressrl); +return 0 ; +} + +static void test_ratelimit_stress (struct kunit *test) +{ +int i; +const int n_stress_kthread =cpumask_weight (cpu_online_mask); +struct stress_kthread skt = {0 }; +struct stress_kthread *sktp =kcalloc (n_stress_kthread,sizeof (*sktp),GFP_KERNEL ); + +KUNIT_EXPECT_NOT_NULL_MSG (test, sktp,"Memory allocation failure" ); +for (i =0 ; i < n_stress_kthread; i++) { + sktp[i].tp =kthread_run (test_ratelimit_stress_child, &sktp[i],"%s/%i" , +"test_ratelimit_stress_child" , i); +KUNIT_EXPECT_NOT_NULL_MSG (test, sktp,"kthread creation failure" ); +pr_alert ("Spawned test_ratelimit_stress_child %d\n" , i); + } +schedule_timeout_idle (stress_duration); +WRITE_ONCE (doneflag,1 ); +for (i =0 ; i < n_stress_kthread; i++) { +kthread_stop (sktp[i].tp); + skt.nattempts += sktp[i].nattempts; + skt.nunlimited += sktp[i].nunlimited; + skt.nlimited += sktp[i].nlimited; + skt.nmissed += sktp[i].nmissed; + } +KUNIT_ASSERT_EQ_MSG (test, skt.nunlimited + skt.nlimited, skt.nattempts, +"Outcomes not equal to attempts" ); +KUNIT_ASSERT_EQ_MSG (test, skt.nlimited, skt.nmissed,"Misses not equal to limits" ); +} + +static struct kunit_case ratelimit_test_cases[] = {KUNIT_CASE_SLOW (test_ratelimit_smoke), +KUNIT_CASE_SLOW (test_ratelimit_stress), {} };static struct kunit_suite ratelimit_test_suite = { .name ="lib_ratelimit" , - .test_cases = sort_test_cases, + .test_cases = ratelimit_test_cases, };kunit_test_suites (&ratelimit_test_suite);
Signature · 1 line
2.40.1
v6 cover ratelimit: Add tests for lib/ratelimit.c
In reply to Paul E. McKenneyHello!
This v6 series adds a simple "smoke" test along with a simple stress test:
1. Add trivial kunit test for ratelimit.
2. Make the ratelimit test more reliable, courtesy of Petr Mladek.
3. Add stress test for ratelimit.
Thanx, Paul
Changes since v5:
o Drop patches that have since been accepted into mainline.
o Add Petr Mladek's patch improving the reliability of the simple
test.
Changes since v4:
o Add a simple stress test.
o Move the tests to the end of the series for bisectability.
o Add Reviewed-by tags.
Changes since v3:
o Correctly handle zero-initialized ratelimit_state structures,
being careful to avoid acquiring the uninitialized ->lock.
o Remove redundant checks of the "burst" local variable.
o Add Reviewed-by tags.
Changes since v2:
o Apply feedback from Bert Karwatzki, Srikanth Aithal, and Mark
Brown, fixing a hang that happened on some systems.
o Applied Reviewed-by tags and added links.
o Added a prototype patch from Petr Mladek that splats if either
interval or burst are negative.
o Added several commits that simplify the code.
Changes since v1 (RFC):
o Switch from lockless fastpath to carrying out needed updates
upon trylock failure, per Petr Mladek feedback. This greatly simplifies the code and is a much smaller change from the current code. There is a small performance penalty compared to the lockless fastpath, but not enough to matter.
o Never unconditionally acquire the lock, again per Petr Mladek
feedback.
o Better define effects of non-positive burst values (always
ratelimit) and non-positive interval values (never ratelimit when the burst value is positive).
o The changes from Petr's original are supplied as five incremental
patches, but could easily be folded into Petr's original if desired. (Left to my lazy self, they stay as-is.)
------------------------------------------------------------------------
b/lib/Kconfig.debug | 11 +++++ b/lib/tests/Makefile | 1 b/lib/tests/test_ratelimit.c | 79 +++++++++++++++++++++++++++++++++++++++++++ lib/tests/test_ratelimit.c | 77 ++++++++++++++++++++++++++++++++++++++--- 4 files changed, 162 insertions(+), 6 deletions(-)
PATCH v6 1/3 lib: Add trivial kunit test for ratelimit
Add a simple single-threaded smoke test for lib/ratelimit.c
To run on x86:
make ARCH=x86_64 mrproper ./tools/testing/kunit/kunit.py run --arch x86_64 --kconfig_add CONFIG_RATELIMIT_KUNIT_TEST=y --kconfig_add CONFIG_SMP=y lib_ratelimit
This will fail on old ___ratelimit(), and subsequent patches provide the fixes that are required.
[ paulmck: Apply timeout and kunit feedback from Petr Mladek. ]
--- lib/Kconfig.debug | 11 ++++++ lib/tests/Makefile | 1 + lib/tests/test_ratelimit.c | 79 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 91 insertions(+) create mode 100644 lib/tests/test_ratelimit.c
lib/Kconfig.debug +11 -0
@@ -3225,6 +3225,17 @@ config TEST_OBJPOOL If unsure, say N. +config RATELIMIT_KUNIT_TEST + tristate "KUnit Test for correctness and stress of ratelimit" if !KUNIT_ALL_TESTS + depends on KUNIT + default KUNIT_ALL_TESTS + help + This builds the "test_ratelimit" module that should be used + for correctness verification and concurrent testings of rate + limiting. + + If unsure, say N. + config INT_POW_KUNIT_TEST tristate "Integer exponentiation (int_pow) test" if !KUNIT_ALL_TESTS depends on KUNIT
lib/tests/Makefile +1 -0
@@ -46,5 +46,6 @@ obj-$(CONFIG_STRING_KUNIT_TEST) += string_kunit.o obj-$(CONFIG_STRING_HELPERS_KUNIT_TEST) += string_helpers_kunit.o obj-$(CONFIG_USERCOPY_KUNIT_TEST) += usercopy_kunit.o obj-$(CONFIG_UTIL_MACROS_KUNIT) += util_macros_kunit.o +obj-$(CONFIG_RATELIMIT_KUNIT_TEST) += test_ratelimit.o obj-$(CONFIG_TEST_RUNTIME_MODULE) += module/
lib/tests/test_ratelimit.c new file +79 -0
79 lines not on this page · raw message
Signature · 1 line
2.40.1
Andrew Morton
maintainerRe: [PATCH v6 1/3] lib: Add trivial kunit test for ratelimit
Quoted 11 lines · On Wed, 9 Jul 2025 11:03:33 -0700 "Paul E. McKenney" <[email protected]>…
Add a simple single-threaded smoke test for
lib/ratelimit.cTo run on x86:
make ARCH=x86_64 mrproper ./tools/testing/kunit/kunit.py run --arch x86_64 --kconfig_add CONFIG_RATELIMIT_KUNIT_TEST=y --kconfig_add CONFIG_SMP=y lib_ratelimitThis will fail on old
___ratelimit(), and subsequent patches provide the fixes that are required.[ paulmck: Apply timeout and kunit feedback from Petr Mladek. ]
The above line makes me suspect that this was paulmck tweaking someone else's patch. If the authorship correct on this one?
Re: [PATCH v6 1/3] lib: Add trivial kunit test for ratelimit
Quoted 9 lines · On Wed, 9 Jul 2025 15:41:52 -0700 Andrew Morton <[email protected]>…
Quoted 7 lines · On Wed, 9 Jul 2025 11:03:33 -0700 "Paul E. McKenney" <[email protected]>…
Add a simple single-threaded smoke test for
lib/ratelimit.cTo run on x86:
make ARCH=x86_64 mrproper ./tools/testing/kunit/kunit.py run --arch x86_64 --kconfig_add CONFIG_RATELIMIT_KUNIT_TEST=y --kconfig_add CONFIG_SMP=y lib_ratelimit
Quoted 7 lines
Quoted 4 lines
This will fail on old
___ratelimit(), and subsequent patches provide the fixes that are required.[ paulmck: Apply timeout and kunit feedback from Petr Mladek. ]
The above line makes me suspect that this was paulmck tweaking someone else's patch. If the authorship correct on this one?
Looks to me that Paul just took some advice from Petr and was just giving credit. Perhaps he could lose the "paulmck:" part?
Perhaps:
?
-- Steve
Re: [PATCH v6 1/3] lib: Add trivial kunit test for ratelimit
Quoted 15 lines · Steven Rostedt
Quoted 12 lines · On Wed, 9 Jul 2025 15:41:52 -0700 Andrew Morton <[email protected]>…
Quoted 10 lines · On Wed, 9 Jul 2025 11:03:33 -0700 "Paul E. McKenney" <[email protected]>…
Add a simple single-threaded smoke test for
lib/ratelimit.cTo run on x86:
make ARCH=x86_64 mrproper ./tools/testing/kunit/kunit.py run --arch x86_64 --kconfig_add CONFIG_RATELIMIT_KUNIT_TEST=y --kconfig_add CONFIG_SMP=y lib_ratelimitThis will fail on old
___ratelimit(), and subsequent patches provide the fixes that are required.
Quoted 7 lines
Quoted 4 lines
Quoted 1 line
[ paulmck: Apply timeout and kunit feedback from Petr Mladek. ]
The above line makes me suspect that this was paulmck tweaking someone else's patch. If the authorship correct on this one?
Looks to me that Paul just took some advice from Petr and was just giving credit. Perhaps he could lose the "paulmck:" part?
You got it!
Quoted 3 lines
Perhaps:
That would work for me.
Thanx, Paul
PATCH v6 2/3 lib: Make the ratelimit test more reliable
In reply to Paul E. McKenneyFrom: Petr Mladek <[email protected]>
The selftest fails most of the times when running in qemu with a kernel configured with CONFIG_HZ = 250:
Quoted 5 lines
test_ratelimit_smoke: 1 callbacks suppressed
#test_ratelimit_smoke: ASSERTION FAILED atlib/tests/test_ratelimit.c:28Expected ___ratelimit(&testrl, "test_ratelimit_smoke") == (false), but ___ratelimit(&testrl, "test_ratelimit_smoke") == 1 (0x1) (false) == 0 (0x0)
Try to make the test slightly more reliable by calling the problematic ratelimit in the middle of the interval.
--- lib/tests/test_ratelimit.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-)
lib/tests/test_ratelimit.c +4 -4
@@ -24,19 +24,19 @@ static void test_ratelimit_smoke(struct kunit *test)test_ratelimited (test, true);test_ratelimited (test, false); -schedule_timeout_idle (TESTRL_INTERVAL -40 ); +schedule_timeout_idle (TESTRL_INTERVAL /2 );test_ratelimited (test, false); -schedule_timeout_idle (50 ); +schedule_timeout_idle (TESTRL_INTERVAL *3 /4 );test_ratelimited (test, true);schedule_timeout_idle (2 *TESTRL_INTERVAL );test_ratelimited (test, true);test_ratelimited (test, true); -schedule_timeout_idle (TESTRL_INTERVAL -40 ); +schedule_timeout_idle (TESTRL_INTERVAL /2 );test_ratelimited (test, true); -schedule_timeout_idle (50 ); +schedule_timeout_idle (TESTRL_INTERVAL *3 /4 );test_ratelimited (test, true);test_ratelimited (test, true);test_ratelimited (test, true);
Signature · 1 line
2.40.1
Andrew Morton
maintainerRe: [PATCH v6 2/3] lib: Make the ratelimit test more reliable
Quoted 9 lines · On Wed, 9 Jul 2025 11:03:34 -0700 "Paul E. McKenney" <[email protected]>…
The selftest fails most of the times when running in qemu with a kernel configured with
CONFIG_HZ= 250:Quoted 5 lines
test_ratelimit_smoke: 1 callbacks suppressed
#test_ratelimit_smoke: ASSERTION FAILED atlib/tests/test_ratelimit.c:28Expected ___ratelimit(&testrl, "test_ratelimit_smoke") == (false), but ___ratelimit(&testrl, "test_ratelimit_smoke") == 1 (0x1) (false) == 0 (0x0)
Quoted 8 lines
Try to make the test slightly more reliable by calling the problematic ratelimit in the middle of the interval.
Signed-off-by Petr Mladek <[email protected]>Signed-off-by Paul E. McKenney <[email protected]>--- lib/tests/test_ratelimit.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-)
Patch 1 adds test_ratelimit.c and patch 2 fixes it.
Unconventional (and undesirable IMO). Would the world end if I folded 2 into 1?
Re: [PATCH v6 2/3] lib: Make the ratelimit test more reliable
Quoted 20 lines · Andrew Morton
Quoted 9 lines · On Wed, 9 Jul 2025 11:03:34 -0700 "Paul E. McKenney" <[email protected]>…
The selftest fails most of the times when running in qemu with a kernel configured with
CONFIG_HZ= 250:Quoted 5 lines
test_ratelimit_smoke: 1 callbacks suppressed
#test_ratelimit_smoke: ASSERTION FAILED atlib/tests/test_ratelimit.c:28Expected ___ratelimit(&testrl, "test_ratelimit_smoke") == (false), but ___ratelimit(&testrl, "test_ratelimit_smoke") == 1 (0x1) (false) == 0 (0x0)Quoted 8 lines
Try to make the test slightly more reliable by calling the problematic ratelimit in the middle of the interval.
Signed-off-by Petr Mladek <[email protected]>Signed-off-by Paul E. McKenney <[email protected]>--- lib/tests/test_ratelimit.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-)
Quoted 4 lines
Patch 1 adds
test_ratelimit.cand patch 2 fixes it.Unconventional (and undesirable IMO). Would the world end if I folded 2 into 1?
Folding them together works for me, as long as Petr is properly credited.
Thanx, Paul
PATCH v6 3/3 lib: Add stress test for ratelimit
In reply to Paul E. McKenneyAdd a simple stress test for lib/ratelimit.c
To run on x86:
./tools/testing/kunit/kunit.py run --arch x86_64 --kconfig_add CONFIG_RATELIMIT_KUNIT_TEST=y --kconfig_add CONFIG_SMP=y --qemu_args "-smp 4" lib_ratelimit
On a 16-CPU system, the "4" in "-smp 4" can be varied between 1 and 8. Larger numbers have higher probabilities of introducing delays that break the smoke test. In the extreme case, increasing the number to larger than the number of CPUs in the underlying system is an excellent way to get a test failure.
--- lib/tests/test_ratelimit.c | 69 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 67 insertions(+), 2 deletions(-)
lib/tests/test_ratelimit.c +67 -2
@@ -4,6 +4,8 @@#include <linux/ratelimit.h> #include <linux/module.h> +#include <linux/kthread.h> +#include <linux/cpumask.h> /* a simple boot-time regression test */
@@ -63,14 +65,77 @@ static void test_ratelimit_smoke(struct kunit *test)test_ratelimited (test, false); } -static struct kunit_case sort_test_cases[] = { +static struct ratelimit_state stressrl =RATELIMIT_STATE_INIT_FLAGS ("stressrl" ,HZ /10 ,3 , +RATELIMIT_MSG_ON_RELEASE ); + +static int doneflag; +static const int stress_duration =2 *HZ ; + +struct stress_kthread { +unsigned long nattempts; +unsigned long nunlimited; +unsigned long nlimited; +unsigned long nmissed; +struct task_struct *tp; +}; + +static int test_ratelimit_stress_child (void *arg) +{ +struct stress_kthread *sktp = arg; + +set_user_nice (current,MAX_NICE ); +WARN_ON_ONCE (!sktp->tp); + +while (!READ_ONCE (doneflag)) { + sktp->nattempts++; +if (___ratelimit (&stressrl, __func__)) + sktp->nunlimited++; +else + sktp->nlimited++; +cond_resched (); + } + + sktp->nmissed =ratelimit_state_reset_miss (&stressrl); +return 0 ; +} + +static void test_ratelimit_stress (struct kunit *test) +{ +int i; +const int n_stress_kthread =cpumask_weight (cpu_online_mask); +struct stress_kthread skt = {0 }; +struct stress_kthread *sktp =kcalloc (n_stress_kthread,sizeof (*sktp),GFP_KERNEL ); + +KUNIT_EXPECT_NOT_NULL_MSG (test, sktp,"Memory allocation failure" ); +for (i =0 ; i < n_stress_kthread; i++) { + sktp[i].tp =kthread_run (test_ratelimit_stress_child, &sktp[i],"%s/%i" , +"test_ratelimit_stress_child" , i); +KUNIT_EXPECT_NOT_NULL_MSG (test, sktp,"kthread creation failure" ); +pr_alert ("Spawned test_ratelimit_stress_child %d\n" , i); + } +schedule_timeout_idle (stress_duration); +WRITE_ONCE (doneflag,1 ); +for (i =0 ; i < n_stress_kthread; i++) { +kthread_stop (sktp[i].tp); + skt.nattempts += sktp[i].nattempts; + skt.nunlimited += sktp[i].nunlimited; + skt.nlimited += sktp[i].nlimited; + skt.nmissed += sktp[i].nmissed; + } +KUNIT_ASSERT_EQ_MSG (test, skt.nunlimited + skt.nlimited, skt.nattempts, +"Outcomes not equal to attempts" ); +KUNIT_ASSERT_EQ_MSG (test, skt.nlimited, skt.nmissed,"Misses not equal to limits" ); +} + +static struct kunit_case ratelimit_test_cases[] = {KUNIT_CASE_SLOW (test_ratelimit_smoke), +KUNIT_CASE_SLOW (test_ratelimit_stress), {} };static struct kunit_suite ratelimit_test_suite = { .name ="lib_ratelimit" , - .test_cases = sort_test_cases, + .test_cases = ratelimit_test_cases, };kunit_test_suites (&ratelimit_test_suite);
Signature · 1 line
2.40.1