Skip to contentSkip to conversation

cyclic: Add ratelimiting schedule() via CYCLIC_RATELIMIT_US

PATCH In review Stefan Roese 2 participants 3 messages Jul 13, 2023 – Jul 14, 2023 Generic include/asm header files

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 cyclic: Add ratelimiting schedule() via CYCLIC_RATELIMIT_US

This patch adds the new Kconfig option CONFIG_CYCLIC_RATELIMIT_US which defines the min allowed time after with a new call into the cyclic infrastructure is allowed. This results in a ratelimiting of the all functions hooked into the cyclic interface. As it's been noticed that on some platforms, that high frequent calls to schedule() (replaced from WATCHDOG_RESET) may lead to a performance degration.

When a high frequent calling into schedule() is detected, a warning is logged only once to help indentify this frantic caller.

Cc Simon Glass <[email protected]>
Cc Christophe Leroy <[email protected]>
---
 common/Kconfig                    | 11 +++++++++++
 common/cyclic.c                   | 27 +++++++++++++++++++++++++--
 include/asm-generic/global_data.h |  1 +
 3 files changed, 37 insertions(+), 2 deletions(-)
common/Kconfig +11 -0
index 42baca20a618..0611cce301a5 100644
@@ -601,6 +601,17 @@ config CYCLIC_MAX_CPU_TIME_US
 	  takes longer than this duration this function will get unregistered
 	  automatically.
 
+config CYCLIC_RATELIMIT_US
+	int "Sets the min time/delay in us after a new call into schedule is done"
+	default 100
+	help
+	  The min allowed time after with a new call into the cyclic
+	  infrastructure is allowed. This results in a ratelimiting of the
+	  all functions hooked into the cyclic interface.
+	  As it's been noticed that on some platforms, that high frequent
+	  calls to schedule() (replaced from WATCHDOG_RESET) may lead to a
+	  performance degration.
+
 endif # CYCLIC
 
 config EVENT
common/cyclic.c +25 -2
index a49bfc88f5c0..c945ae55c965 100644
@@ -12,6 +12,7 @@
 #include <log.h>
 #include <malloc.h>
 #include <time.h>
+#include <linux/bug.h>
 #include <linux/errno.h>
 #include <linux/list.h>
 #include <asm/global_data.h>
@@ -109,8 +110,30 @@ void schedule(void)
 	 * schedule() might get called very early before the cyclic IF is
 	 * ready. Make sure to only call cyclic_run() when it's initalized.
 	 */
-	if (gd)
-		cyclic_run();
+	if (gd) {
+		uint64_t now;
+
+		/*
+		 * Add some ratelimiting to not call into the cyclic IF too
+		 * ofter. As it's been noticed that on some platforms, that
+		 * high frequent calls to schedule() (replaced from
+		 * WATCHDOG_RESET) may lead to a performance degration.
+		 */
+		/* Do not reset the watchdog too often */
+		now = timer_get_us();
+		if (time_after_eq64(now, gd->cyclic_next_call)) {
+			cyclic_run();
+			gd->cyclic_next_call = now + CONFIG_CYCLIC_RATELIMIT_US;
+		} else {
+			/*
+			 * Throw a warning (only once) to help identifying
+			 * frantic callers
+			 */
+			WARN_ONCE(1, "schedule() called very often, now = %lld us next call at %lld us, dt = %lld us\n",
+				  now, gd->cyclic_next_call,
+				  gd->cyclic_next_call - now);
+		}
+	}
 }
 
 int cyclic_unregister_all(void)
include/asm-generic/global_data.h +1 -0
index a1e1b9d64005..af26ae4dfc65 100644
@@ -484,6 +484,7 @@ struct global_data {
 	 * @cyclic_list: list of registered cyclic functions
 	 */
 	struct hlist_head cyclic_list;
+	uint64_t cyclic_next_call;
 #endif
 	/**
 	 * @dmtag_list: List of DM tags
Signature · 1 line
2.41.0
#1 lore raw

Hi Stefan,

Quoted 25 lines · Stefan Roese

This patch adds the new Kconfig option CONFIG_CYCLIC_RATELIMIT_US which defines the min allowed time after with a new call into the cyclic infrastructure is allowed. This results in a ratelimiting of the all functions hooked into the cyclic interface. As it's been noticed that on some platforms, that high frequent calls to schedule() (replaced from WATCHDOG_RESET) may lead to a performance degration.

When a high frequent calling into schedule() is detected, a warning is logged only once to help indentify this frantic caller.

Cc Simon Glass <[email protected]>
Cc Christophe Leroy <[email protected]>
---
 common/Kconfig                    | 11 +++++++++++
 common/cyclic.c                   | 27 +++++++++++++++++++++++++--
 include/asm-generic/global_data.h |  1 +
 3 files changed, 37 insertions(+), 2 deletions(-)
common/Kconfig +0 -0
index 42baca20a618..0611cce301a5 100644

common/Kconfig config CYCLIC_MAX_CPU_TIME_US · line 606

Quoted 6 lines
common/Kconfig +3 -0
@@ -601,6 +601,17 @@ config CYCLIC_MAX_CPU_TIME_US
          takes longer than this duration this function will get unregistered
          automatically.
 
+config CYCLIC_RATELIMIT_US
+       int "Sets the min time/delay in us after a new call into schedule is done"
+       default 100

1000 ?

common/Kconfig config CYCLIC_MAX_CPU_TIME_US · line 609

Quoted 3 lines
common/Kconfig +3 -0
+       help
+         The min allowed time after with a new call into the cyclic
+         infrastructure is allowed. This results in a ratelimiting of the

rate limiting

common/Kconfig config CYCLIC_MAX_CPU_TIME_US · line 613

Quoted 4 lines
common/Kconfig +4 -0
+         all functions hooked into the cyclic interface.
+         As it's been noticed that on some platforms, that high frequent
+         calls to schedule() (replaced from WATCHDOG_RESET) may lead to a
+         performance degration.

degradation

also below

Quoted 17 lines
common/Kconfig +1 -0
+
 endif # CYCLIC
 
 config EVENT
common/cyclic.c +1 -0
index a49bfc88f5c0..c945ae55c965 100644
@@ -12,6 +12,7 @@
 #include <log.h>
 #include <malloc.h>
 #include <time.h>
+#include <linux/bug.h>
 #include <linux/errno.h>
 #include <linux/list.h>
 #include <asm/global_data.h>

common/cyclic.c void schedule(void) · line 114

Quoted 7 lines
common/cyclic.c +2 -2
@@ -109,8 +110,30 @@ void schedule(void)
         * schedule() might get called very early before the cyclic IF is
         * ready. Make sure to only call cyclic_run() when it's initalized.
         */
-       if (gd)
-               cyclic_run();
+       if (gd) {
+               uint64_t now;

Should be u64 in here I think.

common/cyclic.c void schedule(void) · line 122

Quoted 8 lines
common/cyclic.c +8 -0
+
+               /*
+                * Add some ratelimiting to not call into the cyclic IF too
+                * ofter. As it's been noticed that on some platforms, that
+                * high frequent calls to schedule() (replaced from
+                * WATCHDOG_RESET) may lead to a performance degration.
+                */
+               /* Do not reset the watchdog too often */

join with above comment

Quoted 30 lines
common/cyclic.c +14 -0
+               now = timer_get_us();
+               if (time_after_eq64(now, gd->cyclic_next_call)) {
+                       cyclic_run();
+                       gd->cyclic_next_call = now + CONFIG_CYCLIC_RATELIMIT_US;
+               } else {
+                       /*
+                        * Throw a warning (only once) to help identifying
+                        * frantic callers
+                        */
+                       WARN_ONCE(1, "schedule() called very often, now = %lld us next call at %lld us, dt = %lld us\n",
+                                 now, gd->cyclic_next_call,
+                                 gd->cyclic_next_call - now);
+               }
+       }
 }
 
 int cyclic_unregister_all(void)
include/asm-generic/global_data.h +1 -1
index a1e1b9d64005..af26ae4dfc65 100644
@@ -484,6 +484,7 @@ struct global_data {
         * @cyclic_list: list of registered cyclic functions
         */
        struct hlist_head cyclic_list;
+       uint64_t cyclic_next_call;
 #endif
        /**
         * @dmtag_list: List of DM tags
--
Quoted 2 lines

2.41.0

Regards,
Simon

#2 lore raw

Hi Simon,

Quoted 31 lines · Simon Glass

Hi Stefan,

Quoted 28 lines · Stefan Roese

This patch adds the new Kconfig option CONFIG_CYCLIC_RATELIMIT_US which defines the min allowed time after with a new call into the cyclic infrastructure is allowed. This results in a ratelimiting of the all functions hooked into the cyclic interface. As it's been noticed that on some platforms, that high frequent calls to schedule() (replaced from WATCHDOG_RESET) may lead to a performance degration.

When a high frequent calling into schedule() is detected, a warning is logged only once to help indentify this frantic caller.

Cc Simon Glass <[email protected]>
Cc Christophe Leroy <[email protected]>
---
  common/Kconfig                    | 11 +++++++++++
  common/cyclic.c                   | 27 +++++++++++++++++++++++++--
  include/asm-generic/global_data.h |  1 +
  3 files changed, 37 insertions(+), 2 deletions(-)
common/Kconfig +0 -0
index 42baca20a618..0611cce301a5 100644
@@ -601,6 +601,17 @@ config CYCLIC_MAX_CPU_TIME_US
           takes longer than this duration this function will get unregistered
           automatically.
 
Quoted 5 lines

common/Kconfig config CYCLIC_MAX_CPU_TIME_US · line 606

Quoted 3 lines
common/Kconfig +3 -0
+config CYCLIC_RATELIMIT_US
+       int "Sets the min time/delay in us after a new call into schedule is done"
+       default 100

1000 ?

1000 was my first pick as well. Leading to this WARN_ONE warning:

schedule() called very often, now = 1085562us next call at 1086380us

This is why I switched to a less strict rate limitation as default. Other platforms may choose to override this value if this matches their use cases.

Quoted 5 lines

common/Kconfig config CYCLIC_MAX_CPU_TIME_US · line 609

Quoted 3 lines
common/Kconfig +3 -0
+       help
+         The min allowed time after with a new call into the cyclic
+         infrastructure is allowed. This results in a ratelimiting of the

rate limiting

Thanks.

Quoted 8 lines

common/Kconfig config CYCLIC_MAX_CPU_TIME_US · line 613

Quoted 4 lines
common/Kconfig +4 -0
+         all functions hooked into the cyclic interface.
+         As it's been noticed that on some platforms, that high frequent
+         calls to schedule() (replaced from WATCHDOG_RESET) may lead to a
+         performance degration.

degradation

also below

Thanks, will update in v2.

Quoted 18 lines
Quoted 18 lines
common/Kconfig +1 -0
+
  endif # CYCLIC
 
  config EVENT
common/cyclic.c +1 -0
index a49bfc88f5c0..c945ae55c965 100644
@@ -12,6 +12,7 @@
  #include <log.h>
  #include <malloc.h>
  #include <time.h>
+#include <linux/bug.h>
  #include <linux/errno.h>
  #include <linux/list.h>
  #include <asm/global_data.h>
@@ -109,8 +110,30 @@ void schedule(void)
          * schedule() might get called very early before the cyclic IF is
Quoted 8 lines

common/cyclic.c void schedule(void) · line 114

Quoted 6 lines
common/cyclic.c +2 -2
          * ready. Make sure to only call cyclic_run() when it's initalized.
          */
-       if (gd)
-               cyclic_run();
+       if (gd) {
+               uint64_t now;

Should be u64 in here I think.

This file is already using uint64_t instead of u64. This is why I used this type here as well.

Quoted 10 lines

common/cyclic.c void schedule(void) · line 122

Quoted 8 lines
common/cyclic.c +8 -0
+
+               /*
+                * Add some ratelimiting to not call into the cyclic IF too
+                * ofter. As it's been noticed that on some platforms, that
+                * high frequent calls to schedule() (replaced from
+                * WATCHDOG_RESET) may lead to a performance degration.
+                */
+               /* Do not reset the watchdog too often */

join with above comment

Ok, will do.

Thanks,
Stefan

Quoted 27 lines
Quoted 27 lines
common/cyclic.c +14 -0
+               now = timer_get_us();
+               if (time_after_eq64(now, gd->cyclic_next_call)) {
+                       cyclic_run();
+                       gd->cyclic_next_call = now + CONFIG_CYCLIC_RATELIMIT_US;
+               } else {
+                       /*
+                        * Throw a warning (only once) to help identifying
+                        * frantic callers
+                        */
+                       WARN_ONCE(1, "schedule() called very often, now = %lld us next call at %lld us, dt = %lld us\n",
+                                 now, gd->cyclic_next_call,
+                                 gd->cyclic_next_call - now);
+               }
+       }
  }
 
  int cyclic_unregister_all(void)
include/asm-generic/global_data.h +1 -0
index a1e1b9d64005..af26ae4dfc65 100644
@@ -484,6 +484,7 @@ struct global_data {
          * @cyclic_list: list of registered cyclic functions
          */
         struct hlist_head cyclic_list;
+       uint64_t cyclic_next_call;
  #endif
Quoted 8 lines
Quoted 5 lines
include/asm-generic/global_data.h +0 -1
         /**
          * @dmtag_list: List of DM tags
--

2.41.0

Regards,
Simon

Viele Grüße,
Stefan Roese

Signature · 3 lines
DENX Software Engineering GmbH,      Managing Director: Erika Unter
HRB 165235 Munich, Office: Kirchenstr.5, D-82194 Groebenzell, Germany
Phone: (+49)-8142-66989-51 Fax: (+49)-8142-66989-80 Email: [email protected]