mm, memcg: add mem_cgroup_disabled checks in vmpressure and swap-related functions
Subsystems (5)
Suren Baghdasaryan
reviewerPATCH v3 1/3 mm, memcg: add mem_cgroup_disabled checks in vmpressure and swap-related functions
Add mem_cgroup_disabled check in vmpressure, mem_cgroup_uncharge_swap and cgroup_throttle_swaprate functions. This minimizes the memcg overhead in the pagefault and exit_mmap paths when memcgs are disabled using cgroup_disable=memory command-line option.
This change results in ~2.1% overhead reduction when running PFT test comparing {CONFIG_MEMCG=n, CONFIG_MEMCG_SWAP=n} against {CONFIG_MEMCG=y, CONFIG_MEMCG_SWAP=y, cgroup_disable=memory} configuration on an 8-core ARM64 Android device.
--- mm/memcontrol.c | 3 +++ mm/swapfile.c | 3 +++ mm/vmpressure.c | 7 ++++++- 3 files changed, 12 insertions(+), 1 deletion(-)
mm/memcontrol.c +3 -0
@@ -7305,6 +7305,9 @@ void mem_cgroup_uncharge_swap(swp_entry_t entry, unsigned int nr_pages)struct mem_cgroup *memcg;unsigned short id; +if (mem_cgroup_disabled ()) +return ; + id =swap_cgroup_record (entry,0 , nr_pages);rcu_read_lock (); memcg =mem_cgroup_from_id (id);
mm/swapfile.c +3 -0
@@ -3778,6 +3778,9 @@ void cgroup_throttle_swaprate(struct page *page, gfp_t gfp_mask)struct swap_info_struct *si, *next;int nid =page_to_nid (page); +if (mem_cgroup_disabled ()) +return ; +if (!(gfp_mask & __GFP_IO))return ;
mm/vmpressure.c +6 -1
@@ -240,7 +240,12 @@ static void vmpressure_work_fn(struct work_struct *work)void vmpressure (gfp_t gfp,struct mem_cgroup *memcg,bool tree,unsigned long scanned,unsigned long reclaimed) { -struct vmpressure *vmpr =memcg_to_vmpressure (memcg); +struct vmpressure *vmpr; + +if (mem_cgroup_disabled ()) +return ; + + vmpr =memcg_to_vmpressure (memcg);/* * Here we only want to account pressure that userland is able to
Signature · 1 line
2.32.0.93.g670b81a890-goog
Suren Baghdasaryan
reviewerPATCH v3 2/3 mm, memcg: inline mem_cgroup_{charge/uncharge} to improve disabled memcg config
Inline mem_cgroup_{charge/uncharge} and mem_cgroup_uncharge_list functions functions to perform mem_cgroup_disabled static key check inline before calling the main body of the function. This minimizes the memcg overhead in the pagefault and exit_mmap paths when memcgs are disabled using cgroup_disable=memory command-line option.
This change results in ~0.4% overhead reduction when running PFT test comparing {CONFIG_MEMCG=n} against {CONFIG_MEMCG=y, cgroup_disable=memory} configurationon on an 8-core ARM64 Android device.
--- include/linux/memcontrol.h | 28 +++++++++++++++++++++++++--- mm/memcontrol.c | 29 ++++++++++------------------- 2 files changed, 35 insertions(+), 22 deletions(-)
include/linux/memcontrol.h +25 -3
@@ -693,13 +693,35 @@ static inline bool mem_cgroup_below_min(struct mem_cgroup *memcg)page_counter_read (&memcg->memory); } -int mem_cgroup_charge (struct page *page,struct mm_struct *mm,gfp_t gfp_mask); +int __mem_cgroup_charge (struct page *page,struct mm_struct *mm, +gfp_t gfp_mask); +static inline int mem_cgroup_charge (struct page *page,struct mm_struct *mm, +gfp_t gfp_mask) +{ +if (mem_cgroup_disabled ()) +return 0 ; +return __mem_cgroup_charge (page, mm, gfp_mask); +} +int mem_cgroup_swapin_charge_page (struct page *page,struct mm_struct *mm,gfp_t gfp,swp_entry_t entry);void mem_cgroup_swapin_uncharge_swap (swp_entry_t entry); -void mem_cgroup_uncharge (struct page *page); -void mem_cgroup_uncharge_list (struct list_head *page_list); +void __mem_cgroup_uncharge (struct page *page); +static inline void mem_cgroup_uncharge (struct page *page) +{ +if (mem_cgroup_disabled ()) +return ; +__mem_cgroup_uncharge (page); +} + +void __mem_cgroup_uncharge_list (struct list_head *page_list); +static inline void mem_cgroup_uncharge_list (struct list_head *page_list) +{ +if (mem_cgroup_disabled ()) +return ; +__mem_cgroup_uncharge_list (page_list); +}void mem_cgroup_migrate (struct page *oldpage,struct page *newpage);
mm/memcontrol.c +10 -19
@@ -6701,8 +6701,7 @@ void mem_cgroup_calculate_protection(struct mem_cgroup *root, atomic_long_read(&parent->memory.children_low_usage))); } -static int __mem_cgroup_charge (struct page *page,struct mem_cgroup *memcg, -gfp_t gfp) +static int charge_memcg (struct page *page,struct mem_cgroup *memcg,gfp_t gfp) {unsigned int nr_pages =thp_nr_pages (page);int ret;
@@ -6723,7 +6722,7 @@ static int __mem_cgroup_charge(struct page *page, struct mem_cgroup *memcg, }/** -* mem_cgroup_charge - charge a newly allocated page to a cgroup +* __mem_cgroup_charge - charge a newly allocated page to a cgroup * @page: page to charge * @mm: mm context of the victim * @gfp_mask: reclaim mode
@@ -6736,16 +6735,14 @@ static int __mem_cgroup_charge(struct page *page, struct mem_cgroup *memcg,* * Returns 0 on success. Otherwise, an error code is returned. */ -int mem_cgroup_charge (struct page *page,struct mm_struct *mm,gfp_t gfp_mask) +int __mem_cgroup_charge (struct page *page,struct mm_struct *mm, +gfp_t gfp_mask) {struct mem_cgroup *memcg;int ret; -if (mem_cgroup_disabled ()) -return 0 ; - memcg =get_mem_cgroup_from_mm (mm); - ret =__mem_cgroup_charge (page, memcg, gfp_mask); + ret =charge_memcg (page, memcg, gfp_mask);css_put (&memcg->css);return ret;
@@ -6780,7 +6777,7 @@ int mem_cgroup_swapin_charge_page(struct page *page, struct mm_struct *mm, memcg =get_mem_cgroup_from_mm (mm);rcu_read_unlock (); - ret =__mem_cgroup_charge (page, memcg, gfp); + ret =charge_memcg (page, memcg, gfp);css_put (&memcg->css);return ret;
@@ -6916,18 +6913,15 @@ static void uncharge_page(struct page *page, struct uncharge_gather *ug) }/** -* mem_cgroup_uncharge - uncharge a page +* __mem_cgroup_uncharge - uncharge a page * @page: page to uncharge * * Uncharge a page previously charged with mem_cgroup_charge(). */ -void mem_cgroup_uncharge (struct page *page) +void __mem_cgroup_uncharge (struct page *page) {struct uncharge_gather ug; -if (mem_cgroup_disabled ()) -return ; -/* Don't touch page->lru of any random page, pre-check: */ if (!page_memcg (page))return ;
@@ -6938,20 +6932,17 @@ void mem_cgroup_uncharge(struct page *page) }/** -* mem_cgroup_uncharge_list - uncharge a list of page +* __mem_cgroup_uncharge_list - uncharge a list of page * @page_list: list of pages to uncharge * * Uncharge a list of pages previously charged with * mem_cgroup_charge(). */ -void mem_cgroup_uncharge_list (struct list_head *page_list) +void __mem_cgroup_uncharge_list (struct list_head *page_list) {struct uncharge_gather ug;struct page *page; -if (mem_cgroup_disabled ()) -return ; -uncharge_gather_clear (&ug);list_for_each_entry (page, page_list, lru)uncharge_page (page, &ug);
Signature · 1 line
2.32.0.93.g670b81a890-goog
Re: [External] [PATCH v3 2/3] mm, memcg: inline mem_cgroup_{charge/uncharge} to improve disabled memcg config
Quoted 10 lines · Suren Baghdasaryan
Inline mem_cgroup_{charge/uncharge} and
mem_cgroup_uncharge_listfunctions functions to performmem_cgroup_disabledstatic key check inline before calling the main body of the function. This minimizes the memcg overhead in the pagefault andexit_mmappaths when memcgs are disabled using cgroup_disable=memory command-line option.
This change results in ~0.4% overhead reduction when running PFT test comparing {CONFIG_MEMCG=n} against {CONFIG_MEMCG=y, cgroup_disable=memory} configurationon on an 8-core ARM64 Android device.
Quoted 2 lines
Signed-off-by Suren Baghdasaryan <[email protected]>Reviewed-by Shakeel Butt <[email protected]>
But some nits below.
Quoted 101 lines
--- include/linux/memcontrol.h | 28 +++++++++++++++++++++++++--- mm/memcontrol.c | 29 ++++++++++------------------- 2 files changed, 35 insertions(+), 22 deletions(-)include/linux/memcontrol.h +25
-3@@ -693,13 +693,35 @@ static inline bool mem_cgroup_below_min(struct mem_cgroup *memcg)page_counter_read (&memcg->memory); } -int mem_cgroup_charge (struct page *page,struct mm_struct *mm,gfp_t gfp_mask); +int __mem_cgroup_charge (struct page *page,struct mm_struct *mm, +gfp_t gfp_mask); +static inline int mem_cgroup_charge (struct page *page,struct mm_struct *mm, +gfp_t gfp_mask) +{ +if (mem_cgroup_disabled ()) +return 0 ; +return __mem_cgroup_charge (page, mm, gfp_mask); +} +int mem_cgroup_swapin_charge_page (struct page *page,struct mm_struct *mm,gfp_t gfp,swp_entry_t entry);void mem_cgroup_swapin_uncharge_swap (swp_entry_t entry); -void mem_cgroup_uncharge (struct page *page); -void mem_cgroup_uncharge_list (struct list_head *page_list); +void __mem_cgroup_uncharge (struct page *page); +static inline void mem_cgroup_uncharge (struct page *page) +{ +if (mem_cgroup_disabled ()) +return ; +__mem_cgroup_uncharge (page); +} + +void __mem_cgroup_uncharge_list (struct list_head *page_list); +static inline void mem_cgroup_uncharge_list (struct list_head *page_list) +{ +if (mem_cgroup_disabled ()) +return ; +__mem_cgroup_uncharge_list (page_list); +}void mem_cgroup_migrate (struct page *oldpage,struct page *newpage);mm/memcontrol.c +6
-9@@ -6701,8 +6701,7 @@ void mem_cgroup_calculate_protection(struct mem_cgroup *root, atomic_long_read(&parent->memory.children_low_usage))); } -static int __mem_cgroup_charge (struct page *page,struct mem_cgroup *memcg, -gfp_t gfp) +static int charge_memcg (struct page *page,struct mem_cgroup *memcg,gfp_t gfp) {unsigned int nr_pages =thp_nr_pages (page);int ret;@@ -6723,7 +6722,7 @@ static int __mem_cgroup_charge(struct page *page, struct mem_cgroup *memcg, }/** -* mem_cgroup_charge - charge a newly allocated page to a cgroup +* __mem_cgroup_charge - charge a newly allocated page to a cgroup * @page: page to charge * @mm: mm context of the victim * @gfp_mask: reclaim mode @@ -6736,16 +6735,14 @@ static int __mem_cgroup_charge(struct page *page, struct mem_cgroup *memcg,* * Returns 0 on success. Otherwise, an error code is returned. */ -int mem_cgroup_charge (struct page *page,struct mm_struct *mm,gfp_t gfp_mask) +int __mem_cgroup_charge (struct page *page,struct mm_struct *mm, +gfp_t gfp_mask) {struct mem_cgroup *memcg;int ret; -if (mem_cgroup_disabled ()) -return 0 ; - memcg =get_mem_cgroup_from_mm (mm); - ret =__mem_cgroup_charge (page, memcg, gfp_mask); + ret =charge_memcg (page, memcg, gfp_mask);css_put (&memcg->css);return ret;@@ -6780,7 +6777,7 @@ int mem_cgroup_swapin_charge_page(struct page *page, struct mm_struct *mm, memcg =get_mem_cgroup_from_mm (mm);rcu_read_unlock (); - ret =__mem_cgroup_charge (page, memcg, gfp); + ret =charge_memcg (page, memcg, gfp);css_put (&memcg->css);return ret;
mm/memcontrol.c static void uncharge_page(struct page *page, struct uncharge_gather *ug) · line 6919
Quoted 8 lines
mm/memcontrol.c +1
-1@@ -6916,18 +6913,15 @@ static void uncharge_page(struct page *page, struct uncharge_gather *ug) }/** -* mem_cgroup_uncharge - uncharge a page +* __mem_cgroup_uncharge - uncharge a page * @page: page to uncharge * * Uncharge a page previously charged with mem_cgroup_charge().
The comment here also needs to be updated.
mem_cgroup_uncharge() -> __mem_cgroup_uncharge()
Quoted 13 lines
mm/memcontrol.c +1
-4*/ -void mem_cgroup_uncharge (struct page *page) +void __mem_cgroup_uncharge (struct page *page) {struct uncharge_gather ug; -if (mem_cgroup_disabled ()) -return ; -/* Don't touch page->lru of any random page, pre-check: */ if (!page_memcg (page))return ;
mm/memcontrol.c void mem_cgroup_uncharge(struct page *page) · line 6939
Quoted 9 lines
mm/memcontrol.c +1
-1@@ -6938,20 +6932,17 @@ void mem_cgroup_uncharge(struct page *page) }/** -* mem_cgroup_uncharge_list - uncharge a list of page +* __mem_cgroup_uncharge_list - uncharge a list of page * @page_list: list of pages to uncharge * * Uncharge a list of pages previously charged with * mem_cgroup_charge().
Should be __mem_cgroup_charge().
Thanks.
Quoted 16 lines
mm/memcontrol.c +1
-5*/ -void mem_cgroup_uncharge_list (struct list_head *page_list) +void __mem_cgroup_uncharge_list (struct list_head *page_list) {struct uncharge_gather ug;struct page *page; -if (mem_cgroup_disabled ()) -return ; -uncharge_gather_clear (&ug);list_for_each_entry (page, page_list, lru)uncharge_page (page, &ug); --2.32.0.93.g670b81a890-goog
Suren Baghdasaryan
reviewerRe: [External] [PATCH v3 2/3] mm, memcg: inline mem_cgroup_{charge/uncharge} to improve disabled memcg config
Quoted 148 lines · Muchun Song
Quoted 10 lines · Suren Baghdasaryan
Inline mem_cgroup_{charge/uncharge} and
mem_cgroup_uncharge_listfunctions functions to performmem_cgroup_disabledstatic key check inline before calling the main body of the function. This minimizes the memcg overhead in the pagefault andexit_mmappaths when memcgs are disabled using cgroup_disable=memory command-line option.
This change results in ~0.4% overhead reduction when running PFT test comparing {CONFIG_MEMCG=n} against {CONFIG_MEMCG=y, cgroup_disable=memory} configurationon on an 8-core ARM64 Android device.Quoted 2 lines
Signed-off-by Suren Baghdasaryan <[email protected]>Reviewed-by Shakeel Butt <[email protected]>Reviewed-by Muchun Song <[email protected]>But some nits below.
Quoted 101 lines
--- include/linux/memcontrol.h | 28 +++++++++++++++++++++++++--- mm/memcontrol.c | 29 ++++++++++------------------- 2 files changed, 35 insertions(+), 22 deletions(-)include/linux/memcontrol.h +25
-3@@ -693,13 +693,35 @@ static inline bool mem_cgroup_below_min(struct mem_cgroup *memcg)page_counter_read (&memcg->memory); } -int mem_cgroup_charge (struct page *page,struct mm_struct *mm,gfp_t gfp_mask); +int __mem_cgroup_charge (struct page *page,struct mm_struct *mm, +gfp_t gfp_mask); +static inline int mem_cgroup_charge (struct page *page,struct mm_struct *mm, +gfp_t gfp_mask) +{ +if (mem_cgroup_disabled ()) +return 0 ; +return __mem_cgroup_charge (page, mm, gfp_mask); +} +int mem_cgroup_swapin_charge_page (struct page *page,struct mm_struct *mm,gfp_t gfp,swp_entry_t entry);void mem_cgroup_swapin_uncharge_swap (swp_entry_t entry); -void mem_cgroup_uncharge (struct page *page); -void mem_cgroup_uncharge_list (struct list_head *page_list); +void __mem_cgroup_uncharge (struct page *page); +static inline void mem_cgroup_uncharge (struct page *page) +{ +if (mem_cgroup_disabled ()) +return ; +__mem_cgroup_uncharge (page); +} + +void __mem_cgroup_uncharge_list (struct list_head *page_list); +static inline void mem_cgroup_uncharge_list (struct list_head *page_list) +{ +if (mem_cgroup_disabled ()) +return ; +__mem_cgroup_uncharge_list (page_list); +}void mem_cgroup_migrate (struct page *oldpage,struct page *newpage);mm/memcontrol.c +6
-9@@ -6701,8 +6701,7 @@ void mem_cgroup_calculate_protection(struct mem_cgroup *root, atomic_long_read(&parent->memory.children_low_usage))); } -static int __mem_cgroup_charge (struct page *page,struct mem_cgroup *memcg, -gfp_t gfp) +static int charge_memcg (struct page *page,struct mem_cgroup *memcg,gfp_t gfp) {unsigned int nr_pages =thp_nr_pages (page);int ret;@@ -6723,7 +6722,7 @@ static int __mem_cgroup_charge(struct page *page, struct mem_cgroup *memcg, }/** -* mem_cgroup_charge - charge a newly allocated page to a cgroup +* __mem_cgroup_charge - charge a newly allocated page to a cgroup * @page: page to charge * @mm: mm context of the victim * @gfp_mask: reclaim mode @@ -6736,16 +6735,14 @@ static int __mem_cgroup_charge(struct page *page, struct mem_cgroup *memcg,* * Returns 0 on success. Otherwise, an error code is returned. */ -int mem_cgroup_charge (struct page *page,struct mm_struct *mm,gfp_t gfp_mask) +int __mem_cgroup_charge (struct page *page,struct mm_struct *mm, +gfp_t gfp_mask) {struct mem_cgroup *memcg;int ret; -if (mem_cgroup_disabled ()) -return 0 ; - memcg =get_mem_cgroup_from_mm (mm); - ret =__mem_cgroup_charge (page, memcg, gfp_mask); + ret =charge_memcg (page, memcg, gfp_mask);css_put (&memcg->css);return ret;@@ -6780,7 +6777,7 @@ int mem_cgroup_swapin_charge_page(struct page *page, struct mm_struct *mm, memcg =get_mem_cgroup_from_mm (mm);rcu_read_unlock (); - ret =__mem_cgroup_charge (page, memcg, gfp); + ret =charge_memcg (page, memcg, gfp);css_put (&memcg->css);return ret;mm/memcontrol.c static void uncharge_page(struct page *page, struct uncharge_gather *ug) · line 6919
Quoted 8 lines
mm/memcontrol.c +1
-1@@ -6916,18 +6913,15 @@ static void uncharge_page(struct page *page, struct uncharge_gather *ug) }/** -* mem_cgroup_uncharge - uncharge a page +* __mem_cgroup_uncharge - uncharge a page * @page: page to uncharge * * Uncharge a page previously charged with mem_cgroup_charge(). The comment here also needs to be updated.
mem_cgroup_uncharge()->__mem_cgroup_uncharge()Quoted 15 lines
mm/memcontrol.c +1
-4*/ -void mem_cgroup_uncharge (struct page *page) +void __mem_cgroup_uncharge (struct page *page) {struct uncharge_gather ug; -if (mem_cgroup_disabled ()) -return ; -/* Don't touch page->lru of any random page, pre-check: */ if (!page_memcg (page))return ;@@ -6938,20 +6932,17 @@ void mem_cgroup_uncharge(struct page *page) }
Quoted 9 lines
mm/memcontrol.c void mem_cgroup_uncharge(struct page *page) · line 6939
Quoted 7 lines
mm/memcontrol.c +1
-1/** -* mem_cgroup_uncharge_list - uncharge a list of page +* __mem_cgroup_uncharge_list - uncharge a list of page * @page_list: list of pages to uncharge * * Uncharge a list of pages previously charged with * mem_cgroup_charge(). Should be
__mem_cgroup_charge().
Fixed and posted in v4 at:
https://lore.kernel.org/patchwork/project/lkml/list/?series=507943
Thanks!
Quoted 19 lines
Thanks.
Quoted 16 lines
mm/memcontrol.c +1
-5*/ -void mem_cgroup_uncharge_list (struct list_head *page_list) +void __mem_cgroup_uncharge_list (struct list_head *page_list) {struct uncharge_gather ug;struct page *page; -if (mem_cgroup_disabled ()) -return ; -uncharge_gather_clear (&ug);list_for_each_entry (page, page_list, lru)uncharge_page (page, &ug); --2.32.0.93.g670b81a890-goog
Re: [PATCH v3 2/3] mm, memcg: inline mem_cgroup_{charge/uncharge} to improve disabled memcg config
In reply to Suren BaghdasaryanQuoted 11 lines · Suren Baghdasaryan
Inline mem_cgroup_{charge/uncharge} and
mem_cgroup_uncharge_listfunctions functions to performmem_cgroup_disabledstatic key check inline before calling the main body of the function. This minimizes the memcg overhead in the pagefault andexit_mmappaths when memcgs are disabled using cgroup_disable=memory command-line option.
This change results in ~0.4% overhead reduction when running PFT test comparing {CONFIG_MEMCG=n} against {CONFIG_MEMCG=y, cgroup_disable=memory} configurationon on an 8-core ARM64 Android device.Signed-off-by Suren Baghdasaryan <[email protected]>Reviewed-by Shakeel Butt <[email protected]>
With doc updated as suggested by Muchun Song
Thanks!
Quoted 146 lines
--- include/linux/memcontrol.h | 28 +++++++++++++++++++++++++--- mm/memcontrol.c | 29 ++++++++++------------------- 2 files changed, 35 insertions(+), 22 deletions(-)include/linux/memcontrol.h +25
-3@@ -693,13 +693,35 @@ static inline bool mem_cgroup_below_min(struct mem_cgroup *memcg)page_counter_read (&memcg->memory); } -int mem_cgroup_charge (struct page *page,struct mm_struct *mm,gfp_t gfp_mask); +int __mem_cgroup_charge (struct page *page,struct mm_struct *mm, +gfp_t gfp_mask); +static inline int mem_cgroup_charge (struct page *page,struct mm_struct *mm, +gfp_t gfp_mask) +{ +if (mem_cgroup_disabled ()) +return 0 ; +return __mem_cgroup_charge (page, mm, gfp_mask); +} +int mem_cgroup_swapin_charge_page (struct page *page,struct mm_struct *mm,gfp_t gfp,swp_entry_t entry);void mem_cgroup_swapin_uncharge_swap (swp_entry_t entry); -void mem_cgroup_uncharge (struct page *page); -void mem_cgroup_uncharge_list (struct list_head *page_list); +void __mem_cgroup_uncharge (struct page *page); +static inline void mem_cgroup_uncharge (struct page *page) +{ +if (mem_cgroup_disabled ()) +return ; +__mem_cgroup_uncharge (page); +} + +void __mem_cgroup_uncharge_list (struct list_head *page_list); +static inline void mem_cgroup_uncharge_list (struct list_head *page_list) +{ +if (mem_cgroup_disabled ()) +return ; +__mem_cgroup_uncharge_list (page_list); +}void mem_cgroup_migrate (struct page *oldpage,struct page *newpage);mm/memcontrol.c +10
-19@@ -6701,8 +6701,7 @@ void mem_cgroup_calculate_protection(struct mem_cgroup *root, atomic_long_read(&parent->memory.children_low_usage))); } -static int __mem_cgroup_charge (struct page *page,struct mem_cgroup *memcg, -gfp_t gfp) +static int charge_memcg (struct page *page,struct mem_cgroup *memcg,gfp_t gfp) {unsigned int nr_pages =thp_nr_pages (page);int ret;@@ -6723,7 +6722,7 @@ static int __mem_cgroup_charge(struct page *page, struct mem_cgroup *memcg, }/** -* mem_cgroup_charge - charge a newly allocated page to a cgroup +* __mem_cgroup_charge - charge a newly allocated page to a cgroup * @page: page to charge * @mm: mm context of the victim * @gfp_mask: reclaim mode @@ -6736,16 +6735,14 @@ static int __mem_cgroup_charge(struct page *page, struct mem_cgroup *memcg,* * Returns 0 on success. Otherwise, an error code is returned. */ -int mem_cgroup_charge (struct page *page,struct mm_struct *mm,gfp_t gfp_mask) +int __mem_cgroup_charge (struct page *page,struct mm_struct *mm, +gfp_t gfp_mask) {struct mem_cgroup *memcg;int ret; -if (mem_cgroup_disabled ()) -return 0 ; - memcg =get_mem_cgroup_from_mm (mm); - ret =__mem_cgroup_charge (page, memcg, gfp_mask); + ret =charge_memcg (page, memcg, gfp_mask);css_put (&memcg->css);return ret;@@ -6780,7 +6777,7 @@ int mem_cgroup_swapin_charge_page(struct page *page, struct mm_struct *mm, memcg =get_mem_cgroup_from_mm (mm);rcu_read_unlock (); - ret =__mem_cgroup_charge (page, memcg, gfp); + ret =charge_memcg (page, memcg, gfp);css_put (&memcg->css);return ret;@@ -6916,18 +6913,15 @@ static void uncharge_page(struct page *page, struct uncharge_gather *ug) }/** -* mem_cgroup_uncharge - uncharge a page +* __mem_cgroup_uncharge - uncharge a page * @page: page to uncharge * * Uncharge a page previously charged with mem_cgroup_charge(). */ -void mem_cgroup_uncharge (struct page *page) +void __mem_cgroup_uncharge (struct page *page) {struct uncharge_gather ug; -if (mem_cgroup_disabled ()) -return ; -/* Don't touch page->lru of any random page, pre-check: */ if (!page_memcg (page))return ;@@ -6938,20 +6932,17 @@ void mem_cgroup_uncharge(struct page *page) }/** -* mem_cgroup_uncharge_list - uncharge a list of page +* __mem_cgroup_uncharge_list - uncharge a list of page * @page_list: list of pages to uncharge * * Uncharge a list of pages previously charged with * mem_cgroup_charge(). */ -void mem_cgroup_uncharge_list (struct list_head *page_list) +void __mem_cgroup_uncharge_list (struct list_head *page_list) {struct uncharge_gather ug;struct page *page; -if (mem_cgroup_disabled ()) -return ; -uncharge_gather_clear (&ug);list_for_each_entry (page, page_list, lru)uncharge_page (page, &ug);Signature · 1 line
2.32.0.93.g670b81a890-goog
Signature · 2 lines
Michal Hocko SUSE Labs
Suren Baghdasaryan
reviewerRe: [PATCH v3 2/3] mm, memcg: inline mem_cgroup_{charge/uncharge} to improve disabled memcg config
Quoted 11 lines · Michal Hocko
Quoted 9 lines · Suren Baghdasaryan
Inline mem_cgroup_{charge/uncharge} and
mem_cgroup_uncharge_listfunctions functions to performmem_cgroup_disabledstatic key check inline before calling the main body of the function. This minimizes the memcg overhead in the pagefault andexit_mmappaths when memcgs are disabled using cgroup_disable=memory command-line option.
This change results in ~0.4% overhead reduction when running PFT test comparing {CONFIG_MEMCG=n} against {CONFIG_MEMCG=y, cgroup_disable=memory} configurationon on an 8-core ARM64 Android device.
Quoted 5 lines
Quoted 2 lines
Signed-off-by Suren Baghdasaryan <[email protected]>Reviewed-by Shakeel Butt <[email protected]>With doc updated as suggested by Muchun Song
Thanks! Will fix the comment and post v4 later today.
Quoted 153 lines
Thanks!
Quoted 145 lines
--- include/linux/memcontrol.h | 28 +++++++++++++++++++++++++--- mm/memcontrol.c | 29 ++++++++++------------------- 2 files changed, 35 insertions(+), 22 deletions(-)include/linux/memcontrol.h +25
-3@@ -693,13 +693,35 @@ static inline bool mem_cgroup_below_min(struct mem_cgroup *memcg)page_counter_read (&memcg->memory); } -int mem_cgroup_charge (struct page *page,struct mm_struct *mm,gfp_t gfp_mask); +int __mem_cgroup_charge (struct page *page,struct mm_struct *mm, +gfp_t gfp_mask); +static inline int mem_cgroup_charge (struct page *page,struct mm_struct *mm, +gfp_t gfp_mask) +{ +if (mem_cgroup_disabled ()) +return 0 ; +return __mem_cgroup_charge (page, mm, gfp_mask); +} +int mem_cgroup_swapin_charge_page (struct page *page,struct mm_struct *mm,gfp_t gfp,swp_entry_t entry);void mem_cgroup_swapin_uncharge_swap (swp_entry_t entry); -void mem_cgroup_uncharge (struct page *page); -void mem_cgroup_uncharge_list (struct list_head *page_list); +void __mem_cgroup_uncharge (struct page *page); +static inline void mem_cgroup_uncharge (struct page *page) +{ +if (mem_cgroup_disabled ()) +return ; +__mem_cgroup_uncharge (page); +} + +void __mem_cgroup_uncharge_list (struct list_head *page_list); +static inline void mem_cgroup_uncharge_list (struct list_head *page_list) +{ +if (mem_cgroup_disabled ()) +return ; +__mem_cgroup_uncharge_list (page_list); +}void mem_cgroup_migrate (struct page *oldpage,struct page *newpage);mm/memcontrol.c +10
-20@@ -6701,8 +6701,7 @@ void mem_cgroup_calculate_protection(struct mem_cgroup *root, atomic_long_read(&parent->memory.children_low_usage))); } -static int __mem_cgroup_charge (struct page *page,struct mem_cgroup *memcg, -gfp_t gfp) +static int charge_memcg (struct page *page,struct mem_cgroup *memcg,gfp_t gfp) {unsigned int nr_pages =thp_nr_pages (page);int ret;@@ -6723,7 +6722,7 @@ static int __mem_cgroup_charge(struct page *page, struct mem_cgroup *memcg, }/** -* mem_cgroup_charge - charge a newly allocated page to a cgroup +* __mem_cgroup_charge - charge a newly allocated page to a cgroup * @page: page to charge * @mm: mm context of the victim * @gfp_mask: reclaim mode @@ -6736,16 +6735,14 @@ static int __mem_cgroup_charge(struct page *page, struct mem_cgroup *memcg,* * Returns 0 on success. Otherwise, an error code is returned. */ -int mem_cgroup_charge (struct page *page,struct mm_struct *mm,gfp_t gfp_mask) +int __mem_cgroup_charge (struct page *page,struct mm_struct *mm, +gfp_t gfp_mask) {struct mem_cgroup *memcg;int ret; -if (mem_cgroup_disabled ()) -return 0 ; - memcg =get_mem_cgroup_from_mm (mm); - ret =__mem_cgroup_charge (page, memcg, gfp_mask); + ret =charge_memcg (page, memcg, gfp_mask);css_put (&memcg->css);return ret;@@ -6780,7 +6777,7 @@ int mem_cgroup_swapin_charge_page(struct page *page, struct mm_struct *mm, memcg =get_mem_cgroup_from_mm (mm);rcu_read_unlock (); - ret =__mem_cgroup_charge (page, memcg, gfp); + ret =charge_memcg (page, memcg, gfp);css_put (&memcg->css);return ret;@@ -6916,18 +6913,15 @@ static void uncharge_page(struct page *page, struct uncharge_gather *ug) }/** -* mem_cgroup_uncharge - uncharge a page +* __mem_cgroup_uncharge - uncharge a page * @page: page to uncharge * * Uncharge a page previously charged with mem_cgroup_charge(). */ -void mem_cgroup_uncharge (struct page *page) +void __mem_cgroup_uncharge (struct page *page) {struct uncharge_gather ug; -if (mem_cgroup_disabled ()) -return ; -/* Don't touch page->lru of any random page, pre-check: */ if (!page_memcg (page))return ;@@ -6938,20 +6932,17 @@ void mem_cgroup_uncharge(struct page *page) }/** -* mem_cgroup_uncharge_list - uncharge a list of page +* __mem_cgroup_uncharge_list - uncharge a list of page * @page_list: list of pages to uncharge * * Uncharge a list of pages previously charged with * mem_cgroup_charge(). */ -void mem_cgroup_uncharge_list (struct list_head *page_list) +void __mem_cgroup_uncharge_list (struct list_head *page_list) {struct uncharge_gather ug;struct page *page; -if (mem_cgroup_disabled ()) -return ; -uncharge_gather_clear (&ug);list_for_each_entry (page, page_list, lru)uncharge_page (page, &ug); --Quoted 1 line
2.32.0.93.g670b81a890-goog
--
Michal Hocko
SUSE Labs
Re: [PATCH v3 2/3] mm, memcg: inline mem_cgroup_{charge/uncharge} to improve disabled memcg config
In reply to Suren Baghdasaryanmm/memcontrol.c static int __mem_cgroup_charge(struct page *page, struct mem_cgroup *memcg, · line 6728
Quoted 9 lines · Suren Baghdasaryan
mm/memcontrol.c +1
-1@@ -6723,7 +6722,7 @@ static int __mem_cgroup_charge(struct page *page, struct mem_cgroup *memcg, }/** -* mem_cgroup_charge - charge a newly allocated page to a cgroup +* __mem_cgroup_charge - charge a newly allocated page to a cgroup * @page: page to charge * @mm: mm context of the victim * @gfp_mask: reclaim mode
This patch conflicts with the folio work, so I'm just rebasing the folio patches on top of this, and I think this part of the patch is a mistake. We don't want to document the __mem_cgroup_charge() function. That's an implementation detail. This patch should instead have moved the kernel-doc to memcontrol.h and continued to document mem_cgroup_charge().
Suren Baghdasaryan
reviewerRe: [PATCH v3 2/3] mm, memcg: inline mem_cgroup_{charge/uncharge} to improve disabled memcg config
Quoted 5 lines · Matthew Wilcox
Quoted 3 lines · Suren Baghdasaryan
(quoted diff) +0
-0@@ -6723,7 +6722,7 @@ static int __mem_cgroup_charge(struct page *page, struct mem_cgroup *memcg, }
Quoted 12 lines
mm/memcontrol.c static int __mem_cgroup_charge(struct page *page, struct mem_cgroup *memcg, · line 6728
Quoted 6 lines
mm/memcontrol.c +1
-1/** -* mem_cgroup_charge - charge a newly allocated page to a cgroup +* __mem_cgroup_charge - charge a newly allocated page to a cgroup * @page: page to charge * @mm: mm context of the victim * @gfp_mask: reclaim mode This patch conflicts with the folio work, so I'm just rebasing the folio patches on top of this, and I think this part of the patch is a mistake. We don't want to document the
__mem_cgroup_charge()function. That's an implementation detail. This patch should instead have moved the kernel-doc tomemcontrol.hand continued to documentmem_cgroup_charge().
Ack.
There was a v4 version of this patch:
https://lore.kernel.org/patchwork/patch/1458907 which was picked up by Andrew already. If others agree that documentation should be moved into the header file then I'll gladly post another version. Or I can post a separate patch moving the documentation only. Whatever works best. Andrew, Michal, Johannes, WDYT?
Re: [PATCH v3 2/3] mm, memcg: inline mem_cgroup_{charge/uncharge} to improve disabled memcg config
Quoted 19 lines · Suren Baghdasaryan
Quoted 5 lines · Matthew Wilcox
Quoted 3 lines · Suren Baghdasaryan
(quoted diff) +0
-0@@ -6723,7 +6722,7 @@ static int __mem_cgroup_charge(struct page *page, struct mem_cgroup *memcg, }Quoted 12 lines
mm/memcontrol.c static int __mem_cgroup_charge(struct page *page, struct mem_cgroup *memcg, · line 6728
Quoted 6 lines
mm/memcontrol.c +1
-1/** -* mem_cgroup_charge - charge a newly allocated page to a cgroup +* __mem_cgroup_charge - charge a newly allocated page to a cgroup * @page: page to charge * @mm: mm context of the victim * @gfp_mask: reclaim mode This patch conflicts with the folio work, so I'm just rebasing the folio patches on top of this, and I think this part of the patch is a mistake. We don't want to document the
__mem_cgroup_charge()function. That's an implementation detail. This patch should instead have moved the kernel-doc tomemcontrol.hand continued to documentmem_cgroup_charge().
Quoted 7 lines
Ack.
There was a v4 version of this patch:
https://lore.kernel.org/patchwork/patch/1458907 which was picked up by Andrew already. If others agree that documentation should be moved into the header file then I'll gladly post another version. Or I can post a separate patch moving the documentation only. Whatever works best. Andrew, Michal, Johannes, WDYT?
At this point, I've moved the documentation as part of the folio patch. I'd rather not redo that patch again ...
Suren Baghdasaryan
reviewerRe: [PATCH v3 2/3] mm, memcg: inline mem_cgroup_{charge/uncharge} to improve disabled memcg config
Quoted 29 lines · Matthew Wilcox
Quoted 19 lines · Suren Baghdasaryan
Quoted 5 lines · Matthew Wilcox
Quoted 3 lines · Suren Baghdasaryan
(quoted diff) +0
-0@@ -6723,7 +6722,7 @@ static int __mem_cgroup_charge(struct page *page, struct mem_cgroup *memcg, }Quoted 12 lines
mm/memcontrol.c static int __mem_cgroup_charge(struct page *page, struct mem_cgroup *memcg, · line 6728
Quoted 6 lines
mm/memcontrol.c +1
-1/** -* mem_cgroup_charge - charge a newly allocated page to a cgroup +* __mem_cgroup_charge - charge a newly allocated page to a cgroup * @page: page to charge * @mm: mm context of the victim * @gfp_mask: reclaim mode This patch conflicts with the folio work, so I'm just rebasing the folio patches on top of this, and I think this part of the patch is a mistake. We don't want to document the
__mem_cgroup_charge()function. That's an implementation detail. This patch should instead have moved the kernel-doc tomemcontrol.hand continued to documentmem_cgroup_charge().Quoted 7 lines
Ack.
There was a v4 version of this patch:
https://lore.kernel.org/patchwork/patch/1458907 which was picked up by Andrew already. If others agree that documentation should be moved into the header file then I'll gladly post another version. Or I can post a separate patch moving the documentation only. Whatever works best. Andrew, Michal, Johannes, WDYT?
Quoted 2 lines
At this point, I've moved the documentation as part of the folio patch. I'd rather not redo that patch again ...
Ok. If you need me to redo anything please let me know.
Suren Baghdasaryan
reviewerPATCH v3 3/3 mm, memcg: inline swap-related functions to improve disabled memcg config
In reply to Suren BaghdasaryanInline mem_cgroup_try_charge_swap, mem_cgroup_uncharge_swap and cgroup_throttle_swaprate functions to perform mem_cgroup_disabled static key check inline before calling the main body of the function. This minimizes the memcg overhead in the pagefault and exit_mmap paths when memcgs are disabled using cgroup_disable=memory command-line option. This change results in ~1% overhead reduction when running PFT test comparing {CONFIG_MEMCG=n} against {CONFIG_MEMCG=y, cgroup_disable=memory} configuration on an 8-core ARM64 Android device.
--- include/linux/swap.h | 26 +++++++++++++++++++++++--- mm/memcontrol.c | 14 ++++---------- mm/swapfile.c | 5 +---- 3 files changed, 28 insertions(+), 17 deletions(-)
include/linux/swap.h +23 -3
@@ -721,7 +721,13 @@ static inline int mem_cgroup_swappiness(struct mem_cgroup *mem)#endif #if defined(CONFIG_SWAP ) && defined(CONFIG_MEMCG ) && defined(CONFIG_BLK_CGROUP ) -extern void cgroup_throttle_swaprate (struct page *page,gfp_t gfp_mask); +extern void __cgroup_throttle_swaprate (struct page *page,gfp_t gfp_mask); +static inline void cgroup_throttle_swaprate (struct page *page,gfp_t gfp_mask) +{ +if (mem_cgroup_disabled ()) +return ; +__cgroup_throttle_swaprate (page, gfp_mask); +}#else static inline void cgroup_throttle_swaprate (struct page *page,gfp_t gfp_mask) {
@@ -730,8 +736,22 @@ static inline void cgroup_throttle_swaprate(struct page *page, gfp_t gfp_mask)#ifdef CONFIG_MEMCG_SWAP extern void mem_cgroup_swapout (struct page *page,swp_entry_t entry); -extern int mem_cgroup_try_charge_swap (struct page *page,swp_entry_t entry); -extern void mem_cgroup_uncharge_swap (swp_entry_t entry,unsigned int nr_pages); +extern int __mem_cgroup_try_charge_swap (struct page *page,swp_entry_t entry); +static inline int mem_cgroup_try_charge_swap (struct page *page,swp_entry_t entry) +{ +if (mem_cgroup_disabled ()) +return 0 ; +return __mem_cgroup_try_charge_swap (page, entry); +} + +extern void __mem_cgroup_uncharge_swap (swp_entry_t entry,unsigned int nr_pages); +static inline void mem_cgroup_uncharge_swap (swp_entry_t entry,unsigned int nr_pages) +{ +if (mem_cgroup_disabled ()) +return ; +__mem_cgroup_uncharge_swap (entry, nr_pages); +} +extern long mem_cgroup_get_nr_swap_pages (struct mem_cgroup *memcg);extern bool mem_cgroup_swap_full (struct page *page);#else
mm/memcontrol.c +4 -10
@@ -7234,7 +7234,7 @@ void mem_cgroup_swapout(struct page *page, swp_entry_t entry) }/** -* mem_cgroup_try_charge_swap - try charging swap space for a page +* __mem_cgroup_try_charge_swap - try charging swap space for a page * @page: page being added to swap * @entry: swap entry to charge *
@@ -7242,16 +7242,13 @@ void mem_cgroup_swapout(struct page *page, swp_entry_t entry)* * Returns 0 on success, -ENOMEM on failure. */ -int mem_cgroup_try_charge_swap (struct page *page,swp_entry_t entry) +int __mem_cgroup_try_charge_swap (struct page *page,swp_entry_t entry) {unsigned int nr_pages =thp_nr_pages (page);struct page_counter *counter;struct mem_cgroup *memcg;unsigned short oldid; -if (mem_cgroup_disabled ()) -return 0 ; -if (!cgroup_subsys_on_dfl (memory_cgrp_subsys))return 0 ;
@@ -7287,18 +7284,15 @@ int mem_cgroup_try_charge_swap(struct page *page, swp_entry_t entry) }/** -* mem_cgroup_uncharge_swap - uncharge swap space +* __mem_cgroup_uncharge_swap - uncharge swap space * @entry: swap entry to uncharge * @nr_pages: the amount of swap space to uncharge */ -void mem_cgroup_uncharge_swap (swp_entry_t entry,unsigned int nr_pages) +void __mem_cgroup_uncharge_swap (swp_entry_t entry,unsigned int nr_pages) {struct mem_cgroup *memcg;unsigned short id; -if (mem_cgroup_disabled ()) -return ; - id =swap_cgroup_record (entry,0 , nr_pages);rcu_read_lock (); memcg =mem_cgroup_from_id (id);
mm/swapfile.c +1 -4
@@ -3773,14 +3773,11 @@ static void free_swap_count_continuations(struct swap_info_struct *si) }#if defined(CONFIG_MEMCG ) && defined(CONFIG_BLK_CGROUP ) -void cgroup_throttle_swaprate (struct page *page,gfp_t gfp_mask) +void __cgroup_throttle_swaprate (struct page *page,gfp_t gfp_mask) {struct swap_info_struct *si, *next;int nid =page_to_nid (page); -if (mem_cgroup_disabled ()) -return ; -if (!(gfp_mask & __GFP_IO))return ;
Signature · 1 line
2.32.0.93.g670b81a890-goog
Re: [External] [PATCH v3 3/3] mm, memcg: inline swap-related functions to improve disabled memcg config
Quoted 10 lines · Suren Baghdasaryan
Inline
mem_cgroup_try_charge_swap,mem_cgroup_uncharge_swapandcgroup_throttle_swapratefunctions to performmem_cgroup_disabledstatic key check inline before calling the main body of the function. This minimizes the memcg overhead in the pagefault andexit_mmappaths when memcgs are disabled using cgroup_disable=memory command-line option. This change results in ~1% overhead reduction when running PFT test comparing {CONFIG_MEMCG=n} against {CONFIG_MEMCG=y, cgroup_disable=memory} configuration on an 8-core ARM64 Android device.
Quoted 3 lines
Signed-off-by Suren Baghdasaryan <[email protected]>Reviewed-by Shakeel Butt <[email protected]>Acked-by Johannes Weiner <[email protected]>
LGTM.
Thanks.
Re: [PATCH v3 3/3] mm, memcg: inline swap-related functions to improve disabled memcg config
In reply to Suren BaghdasaryanQuoted 9 lines · Suren Baghdasaryan
Inline
mem_cgroup_try_charge_swap,mem_cgroup_uncharge_swapandcgroup_throttle_swapratefunctions to performmem_cgroup_disabledstatic key check inline before calling the main body of the function. This minimizes the memcg overhead in the pagefault andexit_mmappaths when memcgs are disabled using cgroup_disable=memory command-line option. This change results in ~1% overhead reduction when running PFT test comparing {CONFIG_MEMCG=n} against {CONFIG_MEMCG=y, cgroup_disable=memory} configuration on an 8-core ARM64 Android device.
Quoted 3 lines
Signed-off-by Suren Baghdasaryan <[email protected]>Reviewed-by Shakeel Butt <[email protected]>Acked-by Johannes Weiner <[email protected]>
I find it a bit surprising to see such a big difference over a function call in a slow path like swap in/out.
Anyway the change makes sense.
Thanks!
Quoted 124 lines
--- include/linux/swap.h | 26 +++++++++++++++++++++++--- mm/memcontrol.c | 14 ++++---------- mm/swapfile.c | 5 +---- 3 files changed, 28 insertions(+), 17 deletions(-)include/linux/swap.h +23
-3@@ -721,7 +721,13 @@ static inline int mem_cgroup_swappiness(struct mem_cgroup *mem)#endif #if defined(CONFIG_SWAP ) && defined(CONFIG_MEMCG ) && defined(CONFIG_BLK_CGROUP ) -extern void cgroup_throttle_swaprate (struct page *page,gfp_t gfp_mask); +extern void __cgroup_throttle_swaprate (struct page *page,gfp_t gfp_mask); +static inline void cgroup_throttle_swaprate (struct page *page,gfp_t gfp_mask) +{ +if (mem_cgroup_disabled ()) +return ; +__cgroup_throttle_swaprate (page, gfp_mask); +}#else static inline void cgroup_throttle_swaprate (struct page *page,gfp_t gfp_mask) {@@ -730,8 +736,22 @@ static inline void cgroup_throttle_swaprate(struct page *page, gfp_t gfp_mask)#ifdef CONFIG_MEMCG_SWAP extern void mem_cgroup_swapout (struct page *page,swp_entry_t entry); -extern int mem_cgroup_try_charge_swap (struct page *page,swp_entry_t entry); -extern void mem_cgroup_uncharge_swap (swp_entry_t entry,unsigned int nr_pages); +extern int __mem_cgroup_try_charge_swap (struct page *page,swp_entry_t entry); +static inline int mem_cgroup_try_charge_swap (struct page *page,swp_entry_t entry) +{ +if (mem_cgroup_disabled ()) +return 0 ; +return __mem_cgroup_try_charge_swap (page, entry); +} + +extern void __mem_cgroup_uncharge_swap (swp_entry_t entry,unsigned int nr_pages); +static inline void mem_cgroup_uncharge_swap (swp_entry_t entry,unsigned int nr_pages) +{ +if (mem_cgroup_disabled ()) +return ; +__mem_cgroup_uncharge_swap (entry, nr_pages); +} +extern long mem_cgroup_get_nr_swap_pages (struct mem_cgroup *memcg);extern bool mem_cgroup_swap_full (struct page *page);#else mm/memcontrol.c +4
-10@@ -7234,7 +7234,7 @@ void mem_cgroup_swapout(struct page *page, swp_entry_t entry) }/** -* mem_cgroup_try_charge_swap - try charging swap space for a page +* __mem_cgroup_try_charge_swap - try charging swap space for a page * @page: page being added to swap * @entry: swap entry to charge * @@ -7242,16 +7242,13 @@ void mem_cgroup_swapout(struct page *page, swp_entry_t entry)* * Returns 0 on success, -ENOMEM on failure. */ -int mem_cgroup_try_charge_swap (struct page *page,swp_entry_t entry) +int __mem_cgroup_try_charge_swap (struct page *page,swp_entry_t entry) {unsigned int nr_pages =thp_nr_pages (page);struct page_counter *counter;struct mem_cgroup *memcg;unsigned short oldid; -if (mem_cgroup_disabled ()) -return 0 ; -if (!cgroup_subsys_on_dfl (memory_cgrp_subsys))return 0 ;@@ -7287,18 +7284,15 @@ int mem_cgroup_try_charge_swap(struct page *page, swp_entry_t entry) }/** -* mem_cgroup_uncharge_swap - uncharge swap space +* __mem_cgroup_uncharge_swap - uncharge swap space * @entry: swap entry to uncharge * @nr_pages: the amount of swap space to uncharge */ -void mem_cgroup_uncharge_swap (swp_entry_t entry,unsigned int nr_pages) +void __mem_cgroup_uncharge_swap (swp_entry_t entry,unsigned int nr_pages) {struct mem_cgroup *memcg;unsigned short id; -if (mem_cgroup_disabled ()) -return ; - id =swap_cgroup_record (entry,0 , nr_pages);rcu_read_lock (); memcg =mem_cgroup_from_id (id);mm/swapfile.c +1
-4@@ -3773,14 +3773,11 @@ static void free_swap_count_continuations(struct swap_info_struct *si) }#if defined(CONFIG_MEMCG ) && defined(CONFIG_BLK_CGROUP ) -void cgroup_throttle_swaprate (struct page *page,gfp_t gfp_mask) +void __cgroup_throttle_swaprate (struct page *page,gfp_t gfp_mask) {struct swap_info_struct *si, *next;int nid =page_to_nid (page); -if (mem_cgroup_disabled ()) -return ; -if (!(gfp_mask & __GFP_IO))return ;Signature · 1 line
2.32.0.93.g670b81a890-goog
Signature · 2 lines
Michal Hocko SUSE Labs
Suren Baghdasaryan
reviewerRe: [PATCH v3 3/3] mm, memcg: inline swap-related functions to improve disabled memcg config
Quoted 11 lines · Michal Hocko
Quoted 9 lines · Suren Baghdasaryan
Inline
mem_cgroup_try_charge_swap,mem_cgroup_uncharge_swapandcgroup_throttle_swapratefunctions to performmem_cgroup_disabledstatic key check inline before calling the main body of the function. This minimizes the memcg overhead in the pagefault andexit_mmappaths when memcgs are disabled using cgroup_disable=memory command-line option. This change results in ~1% overhead reduction when running PFT test comparing {CONFIG_MEMCG=n} against {CONFIG_MEMCG=y, cgroup_disable=memory} configuration on an 8-core ARM64 Android device.
Quoted 6 lines
Quoted 3 lines
Signed-off-by Suren Baghdasaryan <[email protected]>Reviewed-by Shakeel Butt <[email protected]>Acked-by Johannes Weiner <[email protected]>I find it a bit surprising to see such a big difference over a function call in a slow path like swap in/out.
Might be due to the nature of the test. It is designed to generate an avalanche of anonymous pagefaults and that might be the reason swap functions are hit this hard.
Quoted 135 lines
Anyway the change makes sense.
Acked-by Michal Hocko <[email protected]>Thanks!
Quoted 123 lines
--- include/linux/swap.h | 26 +++++++++++++++++++++++--- mm/memcontrol.c | 14 ++++---------- mm/swapfile.c | 5 +---- 3 files changed, 28 insertions(+), 17 deletions(-)include/linux/swap.h +23
-3@@ -721,7 +721,13 @@ static inline int mem_cgroup_swappiness(struct mem_cgroup *mem)#endif #if defined(CONFIG_SWAP ) && defined(CONFIG_MEMCG ) && defined(CONFIG_BLK_CGROUP ) -extern void cgroup_throttle_swaprate (struct page *page,gfp_t gfp_mask); +extern void __cgroup_throttle_swaprate (struct page *page,gfp_t gfp_mask); +static inline void cgroup_throttle_swaprate (struct page *page,gfp_t gfp_mask) +{ +if (mem_cgroup_disabled ()) +return ; +__cgroup_throttle_swaprate (page, gfp_mask); +}#else static inline void cgroup_throttle_swaprate (struct page *page,gfp_t gfp_mask) {@@ -730,8 +736,22 @@ static inline void cgroup_throttle_swaprate(struct page *page, gfp_t gfp_mask)#ifdef CONFIG_MEMCG_SWAP extern void mem_cgroup_swapout (struct page *page,swp_entry_t entry); -extern int mem_cgroup_try_charge_swap (struct page *page,swp_entry_t entry); -extern void mem_cgroup_uncharge_swap (swp_entry_t entry,unsigned int nr_pages); +extern int __mem_cgroup_try_charge_swap (struct page *page,swp_entry_t entry); +static inline int mem_cgroup_try_charge_swap (struct page *page,swp_entry_t entry) +{ +if (mem_cgroup_disabled ()) +return 0 ; +return __mem_cgroup_try_charge_swap (page, entry); +} + +extern void __mem_cgroup_uncharge_swap (swp_entry_t entry,unsigned int nr_pages); +static inline void mem_cgroup_uncharge_swap (swp_entry_t entry,unsigned int nr_pages) +{ +if (mem_cgroup_disabled ()) +return ; +__mem_cgroup_uncharge_swap (entry, nr_pages); +} +extern long mem_cgroup_get_nr_swap_pages (struct mem_cgroup *memcg);extern bool mem_cgroup_swap_full (struct page *page);#else mm/memcontrol.c +4
-10@@ -7234,7 +7234,7 @@ void mem_cgroup_swapout(struct page *page, swp_entry_t entry) }/** -* mem_cgroup_try_charge_swap - try charging swap space for a page +* __mem_cgroup_try_charge_swap - try charging swap space for a page * @page: page being added to swap * @entry: swap entry to charge * @@ -7242,16 +7242,13 @@ void mem_cgroup_swapout(struct page *page, swp_entry_t entry)* * Returns 0 on success, -ENOMEM on failure. */ -int mem_cgroup_try_charge_swap (struct page *page,swp_entry_t entry) +int __mem_cgroup_try_charge_swap (struct page *page,swp_entry_t entry) {unsigned int nr_pages =thp_nr_pages (page);struct page_counter *counter;struct mem_cgroup *memcg;unsigned short oldid; -if (mem_cgroup_disabled ()) -return 0 ; -if (!cgroup_subsys_on_dfl (memory_cgrp_subsys))return 0 ;@@ -7287,18 +7284,15 @@ int mem_cgroup_try_charge_swap(struct page *page, swp_entry_t entry) }/** -* mem_cgroup_uncharge_swap - uncharge swap space +* __mem_cgroup_uncharge_swap - uncharge swap space * @entry: swap entry to uncharge * @nr_pages: the amount of swap space to uncharge */ -void mem_cgroup_uncharge_swap (swp_entry_t entry,unsigned int nr_pages) +void __mem_cgroup_uncharge_swap (swp_entry_t entry,unsigned int nr_pages) {struct mem_cgroup *memcg;unsigned short id; -if (mem_cgroup_disabled ()) -return ; - id =swap_cgroup_record (entry,0 , nr_pages);rcu_read_lock (); memcg =mem_cgroup_from_id (id);mm/swapfile.c +1
-5@@ -3773,14 +3773,11 @@ static void free_swap_count_continuations(struct swap_info_struct *si) }#if defined(CONFIG_MEMCG ) && defined(CONFIG_BLK_CGROUP ) -void cgroup_throttle_swaprate (struct page *page,gfp_t gfp_mask) +void __cgroup_throttle_swaprate (struct page *page,gfp_t gfp_mask) {struct swap_info_struct *si, *next;int nid =page_to_nid (page); -if (mem_cgroup_disabled ()) -return ; -if (!(gfp_mask & __GFP_IO))return ; --Quoted 1 line
2.32.0.93.g670b81a890-goog
--
Michal Hocko
SUSE Labs
Quoted 37 lines · Suren Baghdasaryan
Add
mem_cgroup_disabledcheck in vmpressure,mem_cgroup_uncharge_swapandcgroup_throttle_swapratefunctions. This minimizes the memcg overhead in the pagefault andexit_mmappaths when memcgs are disabled using cgroup_disable=memory command-line option.
This change results in ~2.1% overhead reduction when running PFT test comparing {CONFIG_MEMCG=n,CONFIG_MEMCG_SWAP=n} against {CONFIG_MEMCG=y,CONFIG_MEMCG_SWAP=y, cgroup_disable=memory} configuration on an 8-core ARM64 Android device.Signed-off-by Suren Baghdasaryan <[email protected]>Reviewed-by Shakeel Butt <[email protected]>Acked-by Johannes Weiner <[email protected]>--- mm/memcontrol.c | 3 +++ mm/swapfile.c | 3 +++ mm/vmpressure.c | 7 ++++++- 3 files changed, 12 insertions(+), 1 deletion(-)mm/memcontrol.c +3
-0@@ -7305,6 +7305,9 @@ void mem_cgroup_uncharge_swap(swp_entry_t entry, unsigned int nr_pages)struct mem_cgroup *memcg;unsigned short id; +if (mem_cgroup_disabled ()) +return ; + id =swap_cgroup_record (entry,0 , nr_pages);rcu_read_lock (); memcg =mem_cgroup_from_id (id);mm/swapfile.c +0
-0
mm/swapfile.c void cgroup_throttle_swaprate(struct page *page, gfp_t gfp_mask) · line 3783
Quoted 6 lines
mm/swapfile.c +3
-0@@ -3778,6 +3778,9 @@ void cgroup_throttle_swaprate(struct page *page, gfp_t gfp_mask)struct swap_info_struct *si, *next;int nid =page_to_nid (page); +if (mem_cgroup_disabled ()) +return ; +
Many thanks for your patch. But I'am somewhat confused about this change. IMO, cgroup_throttle_swaprate() is only related to blk_cgroup and it seems it's irrelevant to mem_cgroup. Could you please have a explanation for me?
Thanks!
Quoted 22 lines
mm/swapfile.c +0
-0if (!(gfp_mask & __GFP_IO))return ;mm/vmpressure.c +6
-1@@ -240,7 +240,12 @@ static void vmpressure_work_fn(struct work_struct *work)void vmpressure (gfp_t gfp,struct mem_cgroup *memcg,bool tree,unsigned long scanned,unsigned long reclaimed) { -struct vmpressure *vmpr =memcg_to_vmpressure (memcg); +struct vmpressure *vmpr; + +if (mem_cgroup_disabled ()) +return ; + + vmpr =memcg_to_vmpressure (memcg);/* * Here we only want to account pressure that userland is able to
Suren Baghdasaryan
reviewerQuoted 42 lines · Miaohe Lin
Quoted 40 lines · Suren Baghdasaryan
Add
mem_cgroup_disabledcheck in vmpressure,mem_cgroup_uncharge_swapandcgroup_throttle_swapratefunctions. This minimizes the memcg overhead in the pagefault andexit_mmappaths when memcgs are disabled using cgroup_disable=memory command-line option.
This change results in ~2.1% overhead reduction when running PFT test comparing {CONFIG_MEMCG=n,CONFIG_MEMCG_SWAP=n} against {CONFIG_MEMCG=y,CONFIG_MEMCG_SWAP=y, cgroup_disable=memory} configuration on an 8-core ARM64 Android device.Signed-off-by Suren Baghdasaryan <[email protected]>Reviewed-by Shakeel Butt <[email protected]>Acked-by Johannes Weiner <[email protected]>--- mm/memcontrol.c | 3 +++ mm/swapfile.c | 3 +++ mm/vmpressure.c | 7 ++++++- 3 files changed, 12 insertions(+), 1 deletion(-)mm/memcontrol.c +3
-0@@ -7305,6 +7305,9 @@ void mem_cgroup_uncharge_swap(swp_entry_t entry, unsigned int nr_pages)struct mem_cgroup *memcg;unsigned short id; +if (mem_cgroup_disabled ()) +return ; + id =swap_cgroup_record (entry,0 , nr_pages);rcu_read_lock (); memcg =mem_cgroup_from_id (id);mm/swapfile.c +0
-0@@ -3778,6 +3778,9 @@ void cgroup_throttle_swaprate(struct page *page, gfp_t gfp_mask)struct swap_info_struct *si, *next;int nid =page_to_nid (page);
Quoted 7 lines
mm/swapfile.c void cgroup_throttle_swaprate(struct page *page, gfp_t gfp_mask) · line 3783
Quoted 3 lines
mm/swapfile.c +3
-0+if (mem_cgroup_disabled ()) +return ; +Many thanks for your patch. But I'am somewhat confused about this change. IMO,
cgroup_throttle_swaprate()is only related toblk_cgroupand it seems it's irrelevant tomem_cgroup. Could you please have a explanation for me?
cgroup_throttle_swaprate() is a NoOp when CONFIG_MEMCG=n (see:
https://elixir.bootlin.com/linux/latest/source/include/linux/swap.h#L699),
therefore I assume we can safely skip it when memcgs are disabled via "cgroup_disable=memory". From perf results I also see no hits on this function when CONFIG_MEMCG=n.
However, looking into the code, I'm not sure why it should depend on CONFIG_MEMCG. But it's Friday night and I might be missing some details here...
Quoted 26 lines
Thanks!
Quoted 22 lines
mm/swapfile.c +0
-0if (!(gfp_mask & __GFP_IO))return ;mm/vmpressure.c +6
-1@@ -240,7 +240,12 @@ static void vmpressure_work_fn(struct work_struct *work)void vmpressure (gfp_t gfp,struct mem_cgroup *memcg,bool tree,unsigned long scanned,unsigned long reclaimed) { -struct vmpressure *vmpr =memcg_to_vmpressure (memcg); +struct vmpressure *vmpr; + +if (mem_cgroup_disabled ()) +return ; + + vmpr =memcg_to_vmpressure (memcg);/* * Here we only want to account pressure that userland is able to
Quoted 51 lines · Suren Baghdasaryan
Quoted 42 lines · Miaohe Lin
Quoted 40 lines · Suren Baghdasaryan
Add
mem_cgroup_disabledcheck in vmpressure,mem_cgroup_uncharge_swapandcgroup_throttle_swapratefunctions. This minimizes the memcg overhead in the pagefault andexit_mmappaths when memcgs are disabled using cgroup_disable=memory command-line option.
This change results in ~2.1% overhead reduction when running PFT test comparing {CONFIG_MEMCG=n,CONFIG_MEMCG_SWAP=n} against {CONFIG_MEMCG=y,CONFIG_MEMCG_SWAP=y, cgroup_disable=memory} configuration on an 8-core ARM64 Android device.Signed-off-by Suren Baghdasaryan <[email protected]>Reviewed-by Shakeel Butt <[email protected]>Acked-by Johannes Weiner <[email protected]>--- mm/memcontrol.c | 3 +++ mm/swapfile.c | 3 +++ mm/vmpressure.c | 7 ++++++- 3 files changed, 12 insertions(+), 1 deletion(-)mm/memcontrol.c +3
-0@@ -7305,6 +7305,9 @@ void mem_cgroup_uncharge_swap(swp_entry_t entry, unsigned int nr_pages)struct mem_cgroup *memcg;unsigned short id; +if (mem_cgroup_disabled ()) +return ; + id =swap_cgroup_record (entry,0 , nr_pages);rcu_read_lock (); memcg =mem_cgroup_from_id (id);mm/swapfile.c +0
-0@@ -3778,6 +3778,9 @@ void cgroup_throttle_swaprate(struct page *page, gfp_t gfp_mask)struct swap_info_struct *si, *next;int nid =page_to_nid (page);Quoted 7 lines
mm/swapfile.c void cgroup_throttle_swaprate(struct page *page, gfp_t gfp_mask) · line 3783
Quoted 3 lines
mm/swapfile.c +3
-0+if (mem_cgroup_disabled ()) +return ; +Many thanks for your patch. But I'am somewhat confused about this change. IMO,
cgroup_throttle_swaprate()is only related toblk_cgroupand it seems it's irrelevant tomem_cgroup. Could you please have a explanation for me?
Quoted 2 lines
cgroup_throttle_swaprate()is a NoOp whenCONFIG_MEMCG=n(see:
https://elixir.bootlin.com/linux/latest/source/include/linux/swap.h#L699),
I browsed the git history related to cgroup_throttle_swaprate() and found this:
"""
mm: memcontrol: move out cgroup swaprate throttling
The cgroup swaprate throttling is about matching new anon allocations to the rate of available IO when that is being throttled. It's the io controller hooking into the VM, rather than a memory controller thing.
"""
It seems cgroup_throttle_swaprate() is working with memory allocations. So mem_cgroup matters this way. But I'am not sure...
Quoted 6 lines
therefore I assume we can safely skip it when memcgs are disabled via "cgroup_disable=memory". From perf results I also see no hits on this function when
CONFIG_MEMCG=n.
However, looking into the code, I'm not sure why it should depend onCONFIG_MEMCG. But it's Friday night and I might be missing some details here...
Many thanks for your replay at Friday night. :)
Quoted 29 lines
Quoted 23 lines
Thanks!
Quoted 19 lines
mm/swapfile.c +0
-0if (!(gfp_mask & __GFP_IO))return ;mm/vmpressure.c +6
-1@@ -240,7 +240,12 @@ static void vmpressure_work_fn(struct work_struct *work)void vmpressure (gfp_t gfp,struct mem_cgroup *memcg,bool tree,unsigned long scanned,unsigned long reclaimed) { -struct vmpressure *vmpr =memcg_to_vmpressure (memcg); +struct vmpressure *vmpr; + +if (mem_cgroup_disabled ()) +return ; + + vmpr =memcg_to_vmpressure (memcg);Quoted 4 lines
mm/vmpressure.c static void vmpressure_work_fn(struct work_struct *work) · line 251
Quoted 3 lines
mm/vmpressure.c +0
-0/* * Here we only want to account pressure that userland is able to .
Quoted 10 lines · Suren Baghdasaryan
Add
mem_cgroup_disabledcheck in vmpressure,mem_cgroup_uncharge_swapandcgroup_throttle_swapratefunctions. This minimizes the memcg overhead in the pagefault andexit_mmappaths when memcgs are disabled using cgroup_disable=memory command-line option.
This change results in ~2.1% overhead reduction when running PFT test comparing {CONFIG_MEMCG=n,CONFIG_MEMCG_SWAP=n} against {CONFIG_MEMCG=y,CONFIG_MEMCG_SWAP=y, cgroup_disable=memory} configuration on an 8-core ARM64 Android device.
Quoted 3 lines
Signed-off-by Suren Baghdasaryan <[email protected]>Reviewed-by Shakeel Butt <[email protected]>Acked-by Johannes Weiner <[email protected]>
The changes are straightforward. LGTM.
Quoted 5 lines · Suren Baghdasaryan
Add
mem_cgroup_disabledcheck in vmpressure,mem_cgroup_uncharge_swapandcgroup_throttle_swapratefunctions. This minimizes the memcg overhead in the pagefault andexit_mmappaths when memcgs are disabled using cgroup_disable=memory command-line option.
This change results in ~2.1% overhead reduction when running PFT test
What is PFT test?
Quoted 7 lines
comparing {
CONFIG_MEMCG=n,CONFIG_MEMCG_SWAP=n} against {CONFIG_MEMCG=y,CONFIG_MEMCG_SWAP=y, cgroup_disable=memory} configuration on an 8-core ARM64 Android device.Signed-off-by Suren Baghdasaryan <[email protected]>Reviewed-by Shakeel Butt <[email protected]>Acked-by Johannes Weiner <[email protected]>
Thanks!
Quoted 54 lines
--- mm/memcontrol.c | 3 +++ mm/swapfile.c | 3 +++ mm/vmpressure.c | 7 ++++++- 3 files changed, 12 insertions(+), 1 deletion(-)mm/memcontrol.c +3
-0@@ -7305,6 +7305,9 @@ void mem_cgroup_uncharge_swap(swp_entry_t entry, unsigned int nr_pages)struct mem_cgroup *memcg;unsigned short id; +if (mem_cgroup_disabled ()) +return ; + id =swap_cgroup_record (entry,0 , nr_pages);rcu_read_lock (); memcg =mem_cgroup_from_id (id);mm/swapfile.c +3
-0@@ -3778,6 +3778,9 @@ void cgroup_throttle_swaprate(struct page *page, gfp_t gfp_mask)struct swap_info_struct *si, *next;int nid =page_to_nid (page); +if (mem_cgroup_disabled ()) +return ; +if (!(gfp_mask & __GFP_IO))return ;mm/vmpressure.c +6
-1@@ -240,7 +240,12 @@ static void vmpressure_work_fn(struct work_struct *work)void vmpressure (gfp_t gfp,struct mem_cgroup *memcg,bool tree,unsigned long scanned,unsigned long reclaimed) { -struct vmpressure *vmpr =memcg_to_vmpressure (memcg); +struct vmpressure *vmpr; + +if (mem_cgroup_disabled ()) +return ; + + vmpr =memcg_to_vmpressure (memcg);/* * Here we only want to account pressure that userland is able to Signature · 1 line
2.32.0.93.g670b81a890-goog
Signature · 2 lines
Michal Hocko SUSE Labs
Suren Baghdasaryan
reviewerQuoted 9 lines · Michal Hocko
Quoted 5 lines · Suren Baghdasaryan
Add
mem_cgroup_disabledcheck in vmpressure,mem_cgroup_uncharge_swapandcgroup_throttle_swapratefunctions. This minimizes the memcg overhead in the pagefault andexit_mmappaths when memcgs are disabled using cgroup_disable=memory command-line option.
This change results in ~2.1% overhead reduction when running PFT testWhat is PFT test?
Christoph Lamenter’s pagefault tool
(https://lkml.org/lkml/2006/8/29/294). I'll add the link in the description for clarity.
Quoted 10 lines
Quoted 8 lines
comparing {
CONFIG_MEMCG=n,CONFIG_MEMCG_SWAP=n} against {CONFIG_MEMCG=y,CONFIG_MEMCG_SWAP=y, cgroup_disable=memory} configuration on an 8-core ARM64 Android device.Signed-off-by Suren Baghdasaryan <[email protected]>Reviewed-by Shakeel Butt <[email protected]>Acked-by Johannes Weiner <[email protected]>
Thanks!
Quoted 61 lines
Thanks!
Quoted 53 lines
--- mm/memcontrol.c | 3 +++ mm/swapfile.c | 3 +++ mm/vmpressure.c | 7 ++++++- 3 files changed, 12 insertions(+), 1 deletion(-)mm/memcontrol.c +3
-0@@ -7305,6 +7305,9 @@ void mem_cgroup_uncharge_swap(swp_entry_t entry, unsigned int nr_pages)struct mem_cgroup *memcg;unsigned short id; +if (mem_cgroup_disabled ()) +return ; + id =swap_cgroup_record (entry,0 , nr_pages);rcu_read_lock (); memcg =mem_cgroup_from_id (id);mm/swapfile.c +3
-0@@ -3778,6 +3778,9 @@ void cgroup_throttle_swaprate(struct page *page, gfp_t gfp_mask)struct swap_info_struct *si, *next;int nid =page_to_nid (page); +if (mem_cgroup_disabled ()) +return ; +if (!(gfp_mask & __GFP_IO))return ;mm/vmpressure.c +6
-2@@ -240,7 +240,12 @@ static void vmpressure_work_fn(struct work_struct *work)void vmpressure (gfp_t gfp,struct mem_cgroup *memcg,bool tree,unsigned long scanned,unsigned long reclaimed) { -struct vmpressure *vmpr =memcg_to_vmpressure (memcg); +struct vmpressure *vmpr; + +if (mem_cgroup_disabled ()) +return ; + + vmpr =memcg_to_vmpressure (memcg);/* * Here we only want to account pressure that userland is able to -- Quoted 1 line
2.32.0.93.g670b81a890-goog
--
Michal Hocko
SUSE Labs