memcg: manually uninline __memcg_memory_event
Subsystems (3)
Shakeel Butt
maintainerPATCH 1/1 memcg: manually uninline __memcg_memory_event
The function __memcg_memory_event has been unnecessarily marked inline even when it is not really performance critical. It is usually called to track extreme conditions. Over the time, it has evolved to include more functionality and inlining it is causing more harm.
Before the patch:
$ size mm/memcontrol.o net/ipv4/tcp_input.o net/ipv4/tcp_output.o text data bss dec hex filename 35645 10574 4192 50411 c4eb mm/memcontrol.o 54738 1658 0 56396 dc4c net/ipv4/tcp_input.o 34644 1065 0 35709 8b7d net/ipv4/tcp_output.o
After the patch:
$ size mm/memcontrol.o net/ipv4/tcp_input.o net/ipv4/tcp_output.o text data bss dec hex filename 35137 10446 4192 49775 c26f mm/memcontrol.o 54322 1562 0 55884 da4c net/ipv4/tcp_input.o 34492 1017 0 35509 8ab5 net/ipv4/tcp_output.o
--- include/linux/memcontrol.h | 32 ++------------------------------ mm/memcontrol.c | 31 +++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 30 deletions(-)
include/linux/memcontrol.h +2 -30
@@ -1002,36 +1002,8 @@ static inline void count_memcg_event_mm(struct mm_struct *mm,count_memcg_events_mm (mm, idx,1 ); } -static inline void __memcg_memory_event (struct mem_cgroup *memcg, -enum memcg_memory_event event, -bool allow_spinning) -{ -bool swap_event = event ==MEMCG_SWAP_HIGH || event ==MEMCG_SWAP_MAX || - event ==MEMCG_SWAP_FAIL ; - -/* For now only MEMCG_MAX can happen with !allow_spinning context. */ -VM_WARN_ON_ONCE (!allow_spinning && event !=MEMCG_MAX ); - -atomic_long_inc (&memcg->memory_events_local[event]); -if (!swap_event && allow_spinning) -cgroup_file_notify (&memcg->events_local_file); - -do { -atomic_long_inc (&memcg->memory_events[event]); -if (allow_spinning) { -if (swap_event) -cgroup_file_notify (&memcg->swap_events_file); -else -cgroup_file_notify (&memcg->events_file); - } - -if (!cgroup_subsys_on_dfl (memory_cgrp_subsys)) -break ; -if (cgrp_dfl_root.flags &CGRP_ROOT_MEMORY_LOCAL_EVENTS ) -break ; - }while ((memcg =parent_mem_cgroup (memcg)) && - !mem_cgroup_is_root (memcg)); -} +void __memcg_memory_event (struct mem_cgroup *memcg, +enum memcg_memory_event event,bool allow_spinning);static inline void memcg_memory_event (struct mem_cgroup *memcg,enum memcg_memory_event event)
mm/memcontrol.c +31 -0
@@ -1626,6 +1626,37 @@ unsigned long mem_cgroup_size(struct mem_cgroup *memcg)return page_counter_read (&memcg->memory); } +void __memcg_memory_event (struct mem_cgroup *memcg, +enum memcg_memory_event event,bool allow_spinning) +{ +bool swap_event = event ==MEMCG_SWAP_HIGH || event ==MEMCG_SWAP_MAX || + event ==MEMCG_SWAP_FAIL ; + +/* For now only MEMCG_MAX can happen with !allow_spinning context. */ +VM_WARN_ON_ONCE (!allow_spinning && event !=MEMCG_MAX ); + +atomic_long_inc (&memcg->memory_events_local[event]); +if (!swap_event && allow_spinning) +cgroup_file_notify (&memcg->events_local_file); + +do { +atomic_long_inc (&memcg->memory_events[event]); +if (allow_spinning) { +if (swap_event) +cgroup_file_notify (&memcg->swap_events_file); +else +cgroup_file_notify (&memcg->events_file); + } + +if (!cgroup_subsys_on_dfl (memory_cgrp_subsys)) +break ; +if (cgrp_dfl_root.flags &CGRP_ROOT_MEMORY_LOCAL_EVENTS ) +break ; + }while ((memcg =parent_mem_cgroup (memcg)) && + !mem_cgroup_is_root (memcg)); +} +EXPORT_SYMBOL (__memcg_memory_event); +static bool mem_cgroup_out_of_memory (struct mem_cgroup *memcg,gfp_t gfp_mask,int order) {
Signature · 1 line
2.47.3
Quoted 73 lines · On Tue, 21 Oct 2025 16:44:25 -0700 Shakeel Butt <[email protected]> wrote:
The function
__memcg_memory_eventhas been unnecessarily marked inline even when it is not really performance critical. It is usually called to track extreme conditions. Over the time, it has evolved to include more functionality and inlining it is causing more harm.Before the patch:
$ size mm/memcontrol.o net/ipv4/tcp_input.o net/ipv4/tcp_output.o text data bss dec hex filename 35645 10574 4192 50411 c4eb mm/memcontrol.o 54738 1658 0 56396 dc4c net/ipv4/tcp_input.o 34644 1065 0 35709 8b7d net/ipv4/tcp_output.oAfter the patch:
$ size mm/memcontrol.o net/ipv4/tcp_input.o net/ipv4/tcp_output.o text data bss dec hex filename 35137 10446 4192 49775 c26f mm/memcontrol.o 54322 1562 0 55884 da4c net/ipv4/tcp_input.o 34492 1017 0 35509 8ab5 net/ipv4/tcp_output.oSigned-off-by Shakeel Butt <[email protected]>--- include/linux/memcontrol.h | 32 ++------------------------------ mm/memcontrol.c | 31 +++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 30 deletions(-)include/linux/memcontrol.h +2
-30@@ -1002,36 +1002,8 @@ static inline void count_memcg_event_mm(struct mm_struct *mm,count_memcg_events_mm (mm, idx,1 ); } -static inline void __memcg_memory_event (struct mem_cgroup *memcg, -enum memcg_memory_event event, -bool allow_spinning) -{ -bool swap_event = event ==MEMCG_SWAP_HIGH || event ==MEMCG_SWAP_MAX || - event ==MEMCG_SWAP_FAIL ; - -/* For now only MEMCG_MAX can happen with !allow_spinning context. */ -VM_WARN_ON_ONCE (!allow_spinning && event !=MEMCG_MAX ); - -atomic_long_inc (&memcg->memory_events_local[event]); -if (!swap_event && allow_spinning) -cgroup_file_notify (&memcg->events_local_file); - -do { -atomic_long_inc (&memcg->memory_events[event]); -if (allow_spinning) { -if (swap_event) -cgroup_file_notify (&memcg->swap_events_file); -else -cgroup_file_notify (&memcg->events_file); - } - -if (!cgroup_subsys_on_dfl (memory_cgrp_subsys)) -break ; -if (cgrp_dfl_root.flags &CGRP_ROOT_MEMORY_LOCAL_EVENTS ) -break ; - }while ((memcg =parent_mem_cgroup (memcg)) && - !mem_cgroup_is_root (memcg)); -} +void __memcg_memory_event (struct mem_cgroup *memcg, +enum memcg_memory_event event,bool allow_spinning);static inline void memcg_memory_event (struct mem_cgroup *memcg,enum memcg_memory_event event)mm/memcontrol.c +0
-0
mm/memcontrol.c unsigned long mem_cgroup_size(struct mem_cgroup *memcg) · line 1630
Quoted 5 lines
mm/memcontrol.c +2
-0@@ -1626,6 +1626,37 @@ unsigned long mem_cgroup_size(struct mem_cgroup *memcg)return page_counter_read (&memcg->memory); } +void __memcg_memory_event (struct mem_cgroup *memcg, +enum memcg_memory_event event,bool allow_spinning)
Seems this function is called only from memcontrol.c. Why not making it a static function?
Quoted 18 lines
mm/memcontrol.c +18
-0+{ +bool swap_event = event ==MEMCG_SWAP_HIGH || event ==MEMCG_SWAP_MAX || + event ==MEMCG_SWAP_FAIL ; + +/* For now only MEMCG_MAX can happen with !allow_spinning context. */ +VM_WARN_ON_ONCE (!allow_spinning && event !=MEMCG_MAX ); + +atomic_long_inc (&memcg->memory_events_local[event]); +if (!swap_event && allow_spinning) +cgroup_file_notify (&memcg->events_local_file); + + do { + atomic_long_inc(&memcg->memory_events[event]); + if (allow_spinning) { + if (swap_event) + cgroup_file_notify(&memcg->swap_events_file); + else + cgroup_file_notify(&memcg->events_file);
mm/memcontrol.c unsigned long mem_cgroup_size(struct mem_cgroup *memcg) · line 1658
Quoted 10 lines
mm/memcontrol.c +10
-0+ } + +if (!cgroup_subsys_on_dfl (memory_cgrp_subsys)) +break ; +if (cgrp_dfl_root.flags &CGRP_ROOT_MEMORY_LOCAL_EVENTS ) +break ; + }while ((memcg =parent_mem_cgroup (memcg)) && + !mem_cgroup_is_root (memcg)); +} +EXPORT_SYMBOL (__memcg_memory_event);
Also, seems there is no reason to export this symbol?
Thanks,
SJ
[...]
Shakeel Butt
maintainerQuoted 78 lines · SeongJae Park
Quoted 76 lines · On Tue, 21 Oct 2025 16:44:25 -0700 Shakeel Butt <[email protected]> wrote:
The function
__memcg_memory_eventhas been unnecessarily marked inline even when it is not really performance critical. It is usually called to track extreme conditions. Over the time, it has evolved to include more functionality and inlining it is causing more harm.Before the patch:
$ size mm/memcontrol.o net/ipv4/tcp_input.o net/ipv4/tcp_output.o text data bss dec hex filename 35645 10574 4192 50411 c4eb mm/memcontrol.o 54738 1658 0 56396 dc4c net/ipv4/tcp_input.o 34644 1065 0 35709 8b7d net/ipv4/tcp_output.oAfter the patch:
$ size mm/memcontrol.o net/ipv4/tcp_input.o net/ipv4/tcp_output.o text data bss dec hex filename 35137 10446 4192 49775 c26f mm/memcontrol.o 54322 1562 0 55884 da4c net/ipv4/tcp_input.o 34492 1017 0 35509 8ab5 net/ipv4/tcp_output.oSigned-off-by Shakeel Butt <[email protected]>--- include/linux/memcontrol.h | 32 ++------------------------------ mm/memcontrol.c | 31 +++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 30 deletions(-)include/linux/memcontrol.h +2
-30@@ -1002,36 +1002,8 @@ static inline void count_memcg_event_mm(struct mm_struct *mm,count_memcg_events_mm (mm, idx,1 ); } -static inline void __memcg_memory_event (struct mem_cgroup *memcg, -enum memcg_memory_event event, -bool allow_spinning) -{ -bool swap_event = event ==MEMCG_SWAP_HIGH || event ==MEMCG_SWAP_MAX || - event ==MEMCG_SWAP_FAIL ; - -/* For now only MEMCG_MAX can happen with !allow_spinning context. */ -VM_WARN_ON_ONCE (!allow_spinning && event !=MEMCG_MAX ); - -atomic_long_inc (&memcg->memory_events_local[event]); -if (!swap_event && allow_spinning) -cgroup_file_notify (&memcg->events_local_file); - -do { -atomic_long_inc (&memcg->memory_events[event]); -if (allow_spinning) { -if (swap_event) -cgroup_file_notify (&memcg->swap_events_file); -else -cgroup_file_notify (&memcg->events_file); - } - -if (!cgroup_subsys_on_dfl (memory_cgrp_subsys)) -break ; -if (cgrp_dfl_root.flags &CGRP_ROOT_MEMORY_LOCAL_EVENTS ) -break ; - }while ((memcg =parent_mem_cgroup (memcg)) && - !mem_cgroup_is_root (memcg)); -} +void __memcg_memory_event (struct mem_cgroup *memcg, +enum memcg_memory_event event,bool allow_spinning);static inline void memcg_memory_event (struct mem_cgroup *memcg,enum memcg_memory_event event)mm/memcontrol.c +0
-0@@ -1626,6 +1626,37 @@ unsigned long mem_cgroup_size(struct mem_cgroup *memcg)return page_counter_read (&memcg->memory); }
Quoted 5 lines
mm/memcontrol.c unsigned long mem_cgroup_size(struct mem_cgroup *memcg) · line 1630
Quoted 2 lines
mm/memcontrol.c +2
-0+void __memcg_memory_event (struct mem_cgroup *memcg, +enum memcg_memory_event event,bool allow_spinning)Seems this function is called only from
memcontrol.c. Why not making it a static function?
There is a recent code where this is called (indirectly) from networking stack for MEMCG_SOCK_THROTTLED.
Quoted 23 lines
Quoted 23 lines
mm/memcontrol.c +22
-0+{ +bool swap_event = event ==MEMCG_SWAP_HIGH || event ==MEMCG_SWAP_MAX || + event ==MEMCG_SWAP_FAIL ; + +/* For now only MEMCG_MAX can happen with !allow_spinning context. */ +VM_WARN_ON_ONCE (!allow_spinning && event !=MEMCG_MAX ); + +atomic_long_inc (&memcg->memory_events_local[event]); +if (!swap_event && allow_spinning) +cgroup_file_notify (&memcg->events_local_file); + + do { + atomic_long_inc(&memcg->memory_events[event]); + if (allow_spinning) { + if (swap_event) + cgroup_file_notify(&memcg->swap_events_file); + else + cgroup_file_notify(&memcg->events_file); + } + + if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) + break;
Quoted 8 lines
mm/memcontrol.c unsigned long mem_cgroup_size(struct mem_cgroup *memcg) · line 1658
Quoted 6 lines
mm/memcontrol.c +6
-0+if (cgrp_dfl_root.flags &CGRP_ROOT_MEMORY_LOCAL_EVENTS ) +break ; + }while ((memcg =parent_mem_cgroup (memcg)) && + !mem_cgroup_is_root (memcg)); +} +EXPORT_SYMBOL (__memcg_memory_event);Also, seems there is no reason to export this symbol?
The networking code needs this export.
Thanks for taking a look.
SeongJae Park
Acked-byQuoted 79 lines · On Tue, 21 Oct 2025 18:28:02 -0700 Shakeel Butt <[email protected]> wrote:
Quoted 78 lines · SeongJae Park
Quoted 76 lines · On Tue, 21 Oct 2025 16:44:25 -0700 Shakeel Butt <[email protected]> wrote:
The function
__memcg_memory_eventhas been unnecessarily marked inline even when it is not really performance critical. It is usually called to track extreme conditions. Over the time, it has evolved to include more functionality and inlining it is causing more harm.Before the patch:
$ size mm/memcontrol.o net/ipv4/tcp_input.o net/ipv4/tcp_output.o text data bss dec hex filename 35645 10574 4192 50411 c4eb mm/memcontrol.o 54738 1658 0 56396 dc4c net/ipv4/tcp_input.o 34644 1065 0 35709 8b7d net/ipv4/tcp_output.oAfter the patch:
$ size mm/memcontrol.o net/ipv4/tcp_input.o net/ipv4/tcp_output.o text data bss dec hex filename 35137 10446 4192 49775 c26f mm/memcontrol.o 54322 1562 0 55884 da4c net/ipv4/tcp_input.o 34492 1017 0 35509 8ab5 net/ipv4/tcp_output.oSigned-off-by Shakeel Butt <[email protected]>--- include/linux/memcontrol.h | 32 ++------------------------------ mm/memcontrol.c | 31 +++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 30 deletions(-)include/linux/memcontrol.h +2
-30@@ -1002,36 +1002,8 @@ static inline void count_memcg_event_mm(struct mm_struct *mm,count_memcg_events_mm (mm, idx,1 ); } -static inline void __memcg_memory_event (struct mem_cgroup *memcg, -enum memcg_memory_event event, -bool allow_spinning) -{ -bool swap_event = event ==MEMCG_SWAP_HIGH || event ==MEMCG_SWAP_MAX || - event ==MEMCG_SWAP_FAIL ; - -/* For now only MEMCG_MAX can happen with !allow_spinning context. */ -VM_WARN_ON_ONCE (!allow_spinning && event !=MEMCG_MAX ); - -atomic_long_inc (&memcg->memory_events_local[event]); -if (!swap_event && allow_spinning) -cgroup_file_notify (&memcg->events_local_file); - -do { -atomic_long_inc (&memcg->memory_events[event]); -if (allow_spinning) { -if (swap_event) -cgroup_file_notify (&memcg->swap_events_file); -else -cgroup_file_notify (&memcg->events_file); - } - -if (!cgroup_subsys_on_dfl (memory_cgrp_subsys)) -break ; -if (cgrp_dfl_root.flags &CGRP_ROOT_MEMORY_LOCAL_EVENTS ) -break ; - }while ((memcg =parent_mem_cgroup (memcg)) && - !mem_cgroup_is_root (memcg)); -} +void __memcg_memory_event (struct mem_cgroup *memcg, +enum memcg_memory_event event,bool allow_spinning);static inline void memcg_memory_event (struct mem_cgroup *memcg,enum memcg_memory_event event)mm/memcontrol.c +0
-0@@ -1626,6 +1626,37 @@ unsigned long mem_cgroup_size(struct mem_cgroup *memcg)return page_counter_read (&memcg->memory); }
Quoted 8 lines
Quoted 5 lines
mm/memcontrol.c unsigned long mem_cgroup_size(struct mem_cgroup *memcg) · line 1630
Quoted 2 lines
mm/memcontrol.c +2
-0+void __memcg_memory_event (struct mem_cgroup *memcg, +enum memcg_memory_event event,bool allow_spinning)Seems this function is called only from
memcontrol.c. Why not making it a static function?There is a recent code where this is called (indirectly) from networking stack for
MEMCG_SOCK_THROTTLED.
Thank you for enlightening me. Apparently the code is from
https://lore.kernel.org/[email protected].
Quoted 33 lines
Quoted 24 lines
Quoted 24 lines
mm/memcontrol.c +22
-0+{ +bool swap_event = event ==MEMCG_SWAP_HIGH || event ==MEMCG_SWAP_MAX || + event ==MEMCG_SWAP_FAIL ; + +/* For now only MEMCG_MAX can happen with !allow_spinning context. */ +VM_WARN_ON_ONCE (!allow_spinning && event !=MEMCG_MAX ); + +atomic_long_inc (&memcg->memory_events_local[event]); +if (!swap_event && allow_spinning) +cgroup_file_notify (&memcg->events_local_file); + + do { + atomic_long_inc(&memcg->memory_events[event]); + if (allow_spinning) { + if (swap_event) + cgroup_file_notify(&memcg->swap_events_file); + else + cgroup_file_notify(&memcg->events_file); + } + + if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) + break;Quoted 8 lines
mm/memcontrol.c unsigned long mem_cgroup_size(struct mem_cgroup *memcg) · line 1658
Quoted 6 lines
mm/memcontrol.c +6
-0+if (cgrp_dfl_root.flags &CGRP_ROOT_MEMORY_LOCAL_EVENTS ) +break ; + }while ((memcg =parent_mem_cgroup (memcg)) && + !mem_cgroup_is_root (memcg)); +} +EXPORT_SYMBOL (__memcg_memory_event);Also, seems there is no reason to export this symbol?
Quoted 3 lines
The networking code needs this export.
Thanks for taking a look.
Again, thank you for clarifying.
Thanks,
SJ
[...]
Quoted 15 lines · Shakeel Butt
Quoted 14 lines · SeongJae Park
Quoted 12 lines · On Tue, 21 Oct 2025 16:44:25 -0700 Shakeel Butt <[email protected]> wrote:
The function
__memcg_memory_eventhas been unnecessarily marked inline even when it is not really performance critical. It is usually called to track extreme conditions. Over the time, it has evolved to include more functionality and inlining it is causing more harm.Before the patch:
$ size mm/memcontrol.o net/ipv4/tcp_input.o net/ipv4/tcp_output.o text data bss dec hex filename 35645 10574 4192 50411 c4eb mm/memcontrol.o 54738 1658 0 56396 dc4c net/ipv4/tcp_input.o 34644 1065 0 35709 8b7d net/ipv4/tcp_output.o
Quoted 8 lines
Quoted 8 lines
Quoted 8 lines
After the patch:
$ size mm/memcontrol.o net/ipv4/tcp_input.o net/ipv4/tcp_output.o text data bss dec hex filename 35137 10446 4192 49775 c26f mm/memcontrol.o 54322 1562 0 55884 da4c net/ipv4/tcp_input.o 34492 1017 0 35509 8ab5 net/ipv4/tcp_output.oSigned-off-by Shakeel Butt <[email protected]>
Thanks!
Quoted 12 lines · Shakeel Butt
The function
__memcg_memory_eventhas been unnecessarily marked inline even when it is not really performance critical. It is usually called to track extreme conditions. Over the time, it has evolved to include more functionality and inlining it is causing more harm.Before the patch:
$ size mm/memcontrol.o net/ipv4/tcp_input.o net/ipv4/tcp_output.o text data bss dec hex filename 35645 10574 4192 50411 c4eb mm/memcontrol.o 54738 1658 0 56396 dc4c net/ipv4/tcp_input.o 34644 1065 0 35709 8b7d net/ipv4/tcp_output.o
Quoted 8 lines
After the patch:
$ size mm/memcontrol.o net/ipv4/tcp_input.o net/ipv4/tcp_output.o text data bss dec hex filename 35137 10446 4192 49775 c26f mm/memcontrol.o 54322 1562 0 55884 da4c net/ipv4/tcp_input.o 34492 1017 0 35509 8ab5 net/ipv4/tcp_output.oSigned-off-by Shakeel Butt <[email protected]>
As the only user is in tree should we make that EXPORT_SYMBOL_GPL instead?
Signature · 2 lines
Michal Hocko SUSE Labs
Shakeel Butt
maintainerQuoted 20 lines · Michal Hocko
Quoted 19 lines · Shakeel Butt
The function
__memcg_memory_eventhas been unnecessarily marked inline even when it is not really performance critical. It is usually called to track extreme conditions. Over the time, it has evolved to include more functionality and inlining it is causing more harm.Before the patch:
$ size mm/memcontrol.o net/ipv4/tcp_input.o net/ipv4/tcp_output.o text data bss dec hex filename 35645 10574 4192 50411 c4eb mm/memcontrol.o 54738 1658 0 56396 dc4c net/ipv4/tcp_input.o 34644 1065 0 35709 8b7d net/ipv4/tcp_output.oAfter the patch:
$ size mm/memcontrol.o net/ipv4/tcp_input.o net/ipv4/tcp_output.o text data bss dec hex filename 35137 10446 4192 49775 c26f mm/memcontrol.o 54322 1562 0 55884 da4c net/ipv4/tcp_input.o 34492 1017 0 35509 8ab5 net/ipv4/tcp_output.o
Quoted 3 lines
Quoted 1 line
Signed-off-by Shakeel Butt <[email protected]>Acked-by Michal Hocko <[email protected]>
Thanks.
Quoted 3 lines
As the only user is in tree should we make that
EXPORT_SYMBOL_GPLinstead?
I just followed the file convention i.e. all other exports in memcontrol.c are like that. I would keep this as is unless you have strong opinion otherwise.
Quoted 1 line · Shakeel Butt
On Thu, Oct 23, 2025 at 03:18:14PM +0200, Michal Hocko wrote:
[...]
Quoted 6 lines
Quoted 2 lines
As the only user is in tree should we make that
EXPORT_SYMBOL_GPLinstead?I just followed the file convention i.e. all other exports in
memcontrol.care like that. I would keep this as is unless you have strong opinion otherwise.
OK
Signature · 2 lines
Michal Hocko SUSE Labs
Quoted 2 lines · Michal Hocko
As the only user is in tree should we make that
EXPORT_SYMBOL_GPLinstead?
Yes.