mm, memcg: add mem_cgroup_disabled checks in vmpressure and swap-related functions
Subsystems (5)
Suren Baghdasaryan
reviewerPATCH v4 1/3 mm, memcg: add mem_cgroup_disabled checks in vmpressure and swap-related functions
Add mem_cgroup_disabled check in vmpressure, mem_cgroup_uncharge_swap and cgroup_throttle_swaprate functions. This minimizes the memcg overhead in the pagefault and exit_mmap paths when memcgs are disabled using cgroup_disable=memory command-line option.
This change results in ~2.1% overhead reduction when running PFT test [1] comparing {CONFIG_MEMCG=n, CONFIG_MEMCG_SWAP=n} against {CONFIG_MEMCG=y, CONFIG_MEMCG_SWAP=y, cgroup_disable=memory} configuration on an 8-core ARM64 Android device.
[1] https://lkml.org/lkml/2006/8/29/294 also used in mmtests suite
--- mm/memcontrol.c | 3 +++ mm/swapfile.c | 3 +++ mm/vmpressure.c | 7 ++++++- 3 files changed, 12 insertions(+), 1 deletion(-)
mm/memcontrol.c +3 -0
@@ -7305,6 +7305,9 @@ void mem_cgroup_uncharge_swap(swp_entry_t entry, unsigned int nr_pages)struct mem_cgroup *memcg;unsigned short id; +if (mem_cgroup_disabled ()) +return ; + id =swap_cgroup_record (entry,0 , nr_pages);rcu_read_lock (); memcg =mem_cgroup_from_id (id);
mm/swapfile.c +3 -0
@@ -3778,6 +3778,9 @@ void cgroup_throttle_swaprate(struct page *page, gfp_t gfp_mask)struct swap_info_struct *si, *next;int nid =page_to_nid (page); +if (mem_cgroup_disabled ()) +return ; +if (!(gfp_mask & __GFP_IO))return ;
mm/vmpressure.c +6 -1
@@ -240,7 +240,12 @@ static void vmpressure_work_fn(struct work_struct *work)void vmpressure (gfp_t gfp,struct mem_cgroup *memcg,bool tree,unsigned long scanned,unsigned long reclaimed) { -struct vmpressure *vmpr =memcg_to_vmpressure (memcg); +struct vmpressure *vmpr; + +if (mem_cgroup_disabled ()) +return ; + + vmpr =memcg_to_vmpressure (memcg);/* * Here we only want to account pressure that userland is able to
Signature · 1 line
2.32.0.93.g670b81a890-goog
Suren Baghdasaryan
reviewerPATCH v4 2/3 mm, memcg: inline mem_cgroup_{charge/uncharge} to improve disabled memcg config
Inline mem_cgroup_{charge/uncharge} and mem_cgroup_uncharge_list functions functions to perform mem_cgroup_disabled static key check inline before calling the main body of the function. This minimizes the memcg overhead in the pagefault and exit_mmap paths when memcgs are disabled using cgroup_disable=memory command-line option.
This change results in ~0.4% overhead reduction when running PFT test [1] comparing {CONFIG_MEMCG=n} against {CONFIG_MEMCG=y, cgroup_disable=memory} configuration on an 8-core ARM64 Android device.
[1] https://lkml.org/lkml/2006/8/29/294 also used in mmtests suite
--- include/linux/memcontrol.h | 28 +++++++++++++++++++++++++--- mm/memcontrol.c | 33 ++++++++++++--------------------- 2 files changed, 37 insertions(+), 24 deletions(-)
include/linux/memcontrol.h +25 -3
@@ -693,13 +693,35 @@ static inline bool mem_cgroup_below_min(struct mem_cgroup *memcg)page_counter_read (&memcg->memory); } -int mem_cgroup_charge (struct page *page,struct mm_struct *mm,gfp_t gfp_mask); +int __mem_cgroup_charge (struct page *page,struct mm_struct *mm, +gfp_t gfp_mask); +static inline int mem_cgroup_charge (struct page *page,struct mm_struct *mm, +gfp_t gfp_mask) +{ +if (mem_cgroup_disabled ()) +return 0 ; +return __mem_cgroup_charge (page, mm, gfp_mask); +} +int mem_cgroup_swapin_charge_page (struct page *page,struct mm_struct *mm,gfp_t gfp,swp_entry_t entry);void mem_cgroup_swapin_uncharge_swap (swp_entry_t entry); -void mem_cgroup_uncharge (struct page *page); -void mem_cgroup_uncharge_list (struct list_head *page_list); +void __mem_cgroup_uncharge (struct page *page); +static inline void mem_cgroup_uncharge (struct page *page) +{ +if (mem_cgroup_disabled ()) +return ; +__mem_cgroup_uncharge (page); +} + +void __mem_cgroup_uncharge_list (struct list_head *page_list); +static inline void mem_cgroup_uncharge_list (struct list_head *page_list) +{ +if (mem_cgroup_disabled ()) +return ; +__mem_cgroup_uncharge_list (page_list); +}void mem_cgroup_migrate (struct page *oldpage,struct page *newpage);
mm/memcontrol.c +12 -21
@@ -6701,8 +6701,7 @@ void mem_cgroup_calculate_protection(struct mem_cgroup *root, atomic_long_read(&parent->memory.children_low_usage))); } -static int __mem_cgroup_charge (struct page *page,struct mem_cgroup *memcg, -gfp_t gfp) +static int charge_memcg (struct page *page,struct mem_cgroup *memcg,gfp_t gfp) {unsigned int nr_pages =thp_nr_pages (page);int ret;
@@ -6723,7 +6722,7 @@ static int __mem_cgroup_charge(struct page *page, struct mem_cgroup *memcg, }/** -* mem_cgroup_charge - charge a newly allocated page to a cgroup +* __mem_cgroup_charge - charge a newly allocated page to a cgroup * @page: page to charge * @mm: mm context of the victim * @gfp_mask: reclaim mode
@@ -6736,16 +6735,14 @@ static int __mem_cgroup_charge(struct page *page, struct mem_cgroup *memcg,* * Returns 0 on success. Otherwise, an error code is returned. */ -int mem_cgroup_charge (struct page *page,struct mm_struct *mm,gfp_t gfp_mask) +int __mem_cgroup_charge (struct page *page,struct mm_struct *mm, +gfp_t gfp_mask) {struct mem_cgroup *memcg;int ret; -if (mem_cgroup_disabled ()) -return 0 ; - memcg =get_mem_cgroup_from_mm (mm); - ret =__mem_cgroup_charge (page, memcg, gfp_mask); + ret =charge_memcg (page, memcg, gfp_mask);css_put (&memcg->css);return ret;
@@ -6780,7 +6777,7 @@ int mem_cgroup_swapin_charge_page(struct page *page, struct mm_struct *mm, memcg =get_mem_cgroup_from_mm (mm);rcu_read_unlock (); - ret =__mem_cgroup_charge (page, memcg, gfp); + ret =charge_memcg (page, memcg, gfp);css_put (&memcg->css);return ret;
@@ -6916,18 +6913,15 @@ static void uncharge_page(struct page *page, struct uncharge_gather *ug) }/** -* mem_cgroup_uncharge - uncharge a page +* __mem_cgroup_uncharge - uncharge a page * @page: page to uncharge * -* Uncharge a page previously charged with mem_cgroup_charge(). +* Uncharge a page previously charged with __mem_cgroup_charge(). */ -void mem_cgroup_uncharge (struct page *page) +void __mem_cgroup_uncharge (struct page *page) {struct uncharge_gather ug; -if (mem_cgroup_disabled ()) -return ; -/* Don't touch page->lru of any random page, pre-check: */ if (!page_memcg (page))return ;
@@ -6938,20 +6932,17 @@ void mem_cgroup_uncharge(struct page *page) }/** -* mem_cgroup_uncharge_list - uncharge a list of page +* __mem_cgroup_uncharge_list - uncharge a list of page * @page_list: list of pages to uncharge * * Uncharge a list of pages previously charged with -* mem_cgroup_charge(). +* __mem_cgroup_charge(). */ -void mem_cgroup_uncharge_list (struct list_head *page_list) +void __mem_cgroup_uncharge_list (struct list_head *page_list) {struct uncharge_gather ug;struct page *page; -if (mem_cgroup_disabled ()) -return ; -uncharge_gather_clear (&ug);list_for_each_entry (page, page_list, lru)uncharge_page (page, &ug);
Signature · 1 line
2.32.0.93.g670b81a890-goog
Suren Baghdasaryan
reviewerPATCH v4 3/3 mm, memcg: inline swap-related functions to improve disabled memcg config
In reply to Suren BaghdasaryanInline mem_cgroup_try_charge_swap, mem_cgroup_uncharge_swap and cgroup_throttle_swaprate functions to perform mem_cgroup_disabled static key check inline before calling the main body of the function. This minimizes the memcg overhead in the pagefault and exit_mmap paths when memcgs are disabled using cgroup_disable=memory command-line option. This change results in ~1% overhead reduction when running PFT test [1] comparing {CONFIG_MEMCG=n} against {CONFIG_MEMCG=y, cgroup_disable=memory} configuration on an 8-core ARM64 Android device.
[1] https://lkml.org/lkml/2006/8/29/294 also used in mmtests suite
--- include/linux/swap.h | 26 +++++++++++++++++++++++--- mm/memcontrol.c | 14 ++++---------- mm/swapfile.c | 5 +---- 3 files changed, 28 insertions(+), 17 deletions(-)
include/linux/swap.h +23 -3
@@ -721,7 +721,13 @@ static inline int mem_cgroup_swappiness(struct mem_cgroup *mem)#endif #if defined(CONFIG_SWAP ) && defined(CONFIG_MEMCG ) && defined(CONFIG_BLK_CGROUP ) -extern void cgroup_throttle_swaprate (struct page *page,gfp_t gfp_mask); +extern void __cgroup_throttle_swaprate (struct page *page,gfp_t gfp_mask); +static inline void cgroup_throttle_swaprate (struct page *page,gfp_t gfp_mask) +{ +if (mem_cgroup_disabled ()) +return ; +__cgroup_throttle_swaprate (page, gfp_mask); +}#else static inline void cgroup_throttle_swaprate (struct page *page,gfp_t gfp_mask) {
@@ -730,8 +736,22 @@ static inline void cgroup_throttle_swaprate(struct page *page, gfp_t gfp_mask)#ifdef CONFIG_MEMCG_SWAP extern void mem_cgroup_swapout (struct page *page,swp_entry_t entry); -extern int mem_cgroup_try_charge_swap (struct page *page,swp_entry_t entry); -extern void mem_cgroup_uncharge_swap (swp_entry_t entry,unsigned int nr_pages); +extern int __mem_cgroup_try_charge_swap (struct page *page,swp_entry_t entry); +static inline int mem_cgroup_try_charge_swap (struct page *page,swp_entry_t entry) +{ +if (mem_cgroup_disabled ()) +return 0 ; +return __mem_cgroup_try_charge_swap (page, entry); +} + +extern void __mem_cgroup_uncharge_swap (swp_entry_t entry,unsigned int nr_pages); +static inline void mem_cgroup_uncharge_swap (swp_entry_t entry,unsigned int nr_pages) +{ +if (mem_cgroup_disabled ()) +return ; +__mem_cgroup_uncharge_swap (entry, nr_pages); +} +extern long mem_cgroup_get_nr_swap_pages (struct mem_cgroup *memcg);extern bool mem_cgroup_swap_full (struct page *page);#else
mm/memcontrol.c +4 -10
@@ -7234,7 +7234,7 @@ void mem_cgroup_swapout(struct page *page, swp_entry_t entry) }/** -* mem_cgroup_try_charge_swap - try charging swap space for a page +* __mem_cgroup_try_charge_swap - try charging swap space for a page * @page: page being added to swap * @entry: swap entry to charge *
@@ -7242,16 +7242,13 @@ void mem_cgroup_swapout(struct page *page, swp_entry_t entry)* * Returns 0 on success, -ENOMEM on failure. */ -int mem_cgroup_try_charge_swap (struct page *page,swp_entry_t entry) +int __mem_cgroup_try_charge_swap (struct page *page,swp_entry_t entry) {unsigned int nr_pages =thp_nr_pages (page);struct page_counter *counter;struct mem_cgroup *memcg;unsigned short oldid; -if (mem_cgroup_disabled ()) -return 0 ; -if (!cgroup_subsys_on_dfl (memory_cgrp_subsys))return 0 ;
@@ -7287,18 +7284,15 @@ int mem_cgroup_try_charge_swap(struct page *page, swp_entry_t entry) }/** -* mem_cgroup_uncharge_swap - uncharge swap space +* __mem_cgroup_uncharge_swap - uncharge swap space * @entry: swap entry to uncharge * @nr_pages: the amount of swap space to uncharge */ -void mem_cgroup_uncharge_swap (swp_entry_t entry,unsigned int nr_pages) +void __mem_cgroup_uncharge_swap (swp_entry_t entry,unsigned int nr_pages) {struct mem_cgroup *memcg;unsigned short id; -if (mem_cgroup_disabled ()) -return ; - id =swap_cgroup_record (entry,0 , nr_pages);rcu_read_lock (); memcg =mem_cgroup_from_id (id);
mm/swapfile.c +1 -4
@@ -3773,14 +3773,11 @@ static void free_swap_count_continuations(struct swap_info_struct *si) }#if defined(CONFIG_MEMCG ) && defined(CONFIG_BLK_CGROUP ) -void cgroup_throttle_swaprate (struct page *page,gfp_t gfp_mask) +void __cgroup_throttle_swaprate (struct page *page,gfp_t gfp_mask) {struct swap_info_struct *si, *next;int nid =page_to_nid (page); -if (mem_cgroup_disabled ()) -return ; -if (!(gfp_mask & __GFP_IO))return ;
Signature · 1 line
2.32.0.93.g670b81a890-goog