Skip to contentSkip to conversation

mm, memcg: add mem_cgroup_disabled checks in vmpressure and swap-related functions

PATCH Superseded Suren Baghdasaryan 5 participants 20 messages Jul 10, 2021 – Jul 18, 2021

Subsystems (5)

v3 of 4 Superseded Latest is v4 →

lore
Share & tools
lore

Suren Baghdasaryan

author reviewer
#0 lore raw

PATCH v3 1/3 mm, memcg: add mem_cgroup_disabled checks in vmpressure and swap-related functions

Add mem_cgroup_disabled check in vmpressure, mem_cgroup_uncharge_swap and cgroup_throttle_swaprate functions. This minimizes the memcg overhead in the pagefault and exit_mmap paths when memcgs are disabled using cgroup_disable=memory command-line option.
This change results in ~2.1% overhead reduction when running PFT test comparing {CONFIG_MEMCG=n, CONFIG_MEMCG_SWAP=n} against {CONFIG_MEMCG=y, CONFIG_MEMCG_SWAP=y, cgroup_disable=memory} configuration on an 8-core ARM64 Android device.

---
 mm/memcontrol.c | 3 +++
 mm/swapfile.c   | 3 +++
 mm/vmpressure.c | 7 ++++++-
 3 files changed, 12 insertions(+), 1 deletion(-)
mm/memcontrol.c +3 -0
index ae1f5d0cb581..a228cd51c4bd 100644
@@ -7305,6 +7305,9 @@ void mem_cgroup_uncharge_swap(swp_entry_t entry, unsigned int nr_pages)
 	struct mem_cgroup *memcg;
 	unsigned short id;
 
+	if (mem_cgroup_disabled())
+		return;
+
 	id = swap_cgroup_record(entry, 0, nr_pages);
 	rcu_read_lock();
 	memcg = mem_cgroup_from_id(id);
mm/swapfile.c +3 -0
index 1e07d1c776f2..707fa0481bb4 100644
@@ -3778,6 +3778,9 @@ void cgroup_throttle_swaprate(struct page *page, gfp_t gfp_mask)
 	struct swap_info_struct *si, *next;
 	int nid = page_to_nid(page);
 
+	if (mem_cgroup_disabled())
+		return;
+
 	if (!(gfp_mask & __GFP_IO))
 		return;
 
mm/vmpressure.c +6 -1
index d69019fc3789..9b172561fded 100644
@@ -240,7 +240,12 @@ static void vmpressure_work_fn(struct work_struct *work)
 void vmpressure(gfp_t gfp, struct mem_cgroup *memcg, bool tree,
 		unsigned long scanned, unsigned long reclaimed)
 {
-	struct vmpressure *vmpr = memcg_to_vmpressure(memcg);
+	struct vmpressure *vmpr;
+
+	if (mem_cgroup_disabled())
+		return;
+
+	vmpr = memcg_to_vmpressure(memcg);
 
 	/*
 	 * Here we only want to account pressure that userland is able to
Signature · 1 line
2.32.0.93.g670b81a890-goog
author reviewer
#1 lore raw

PATCH v3 2/3 mm, memcg: inline mem_cgroup_{charge/uncharge} to improve disabled memcg config

Inline mem_cgroup_{charge/uncharge} and mem_cgroup_uncharge_list functions functions to perform mem_cgroup_disabled static key check inline before calling the main body of the function. This minimizes the memcg overhead in the pagefault and exit_mmap paths when memcgs are disabled using cgroup_disable=memory command-line option.
This change results in ~0.4% overhead reduction when running PFT test comparing {CONFIG_MEMCG=n} against {CONFIG_MEMCG=y, cgroup_disable=memory} configurationon on an 8-core ARM64 Android device.

---
 include/linux/memcontrol.h | 28 +++++++++++++++++++++++++---
 mm/memcontrol.c            | 29 ++++++++++-------------------
 2 files changed, 35 insertions(+), 22 deletions(-)
include/linux/memcontrol.h +25 -3
index bfe5c486f4ad..39fa88051a42 100644
@@ -693,13 +693,35 @@ static inline bool mem_cgroup_below_min(struct mem_cgroup *memcg)
 		page_counter_read(&memcg->memory);
 }
 
-int mem_cgroup_charge(struct page *page, struct mm_struct *mm, gfp_t gfp_mask);
+int __mem_cgroup_charge(struct page *page, struct mm_struct *mm,
+			gfp_t gfp_mask);
+static inline int mem_cgroup_charge(struct page *page, struct mm_struct *mm,
+				    gfp_t gfp_mask)
+{
+	if (mem_cgroup_disabled())
+		return 0;
+	return __mem_cgroup_charge(page, mm, gfp_mask);
+}
+
 int mem_cgroup_swapin_charge_page(struct page *page, struct mm_struct *mm,
 				  gfp_t gfp, swp_entry_t entry);
 void mem_cgroup_swapin_uncharge_swap(swp_entry_t entry);
 
-void mem_cgroup_uncharge(struct page *page);
-void mem_cgroup_uncharge_list(struct list_head *page_list);
+void __mem_cgroup_uncharge(struct page *page);
+static inline void mem_cgroup_uncharge(struct page *page)
+{
+	if (mem_cgroup_disabled())
+		return;
+	__mem_cgroup_uncharge(page);
+}
+
+void __mem_cgroup_uncharge_list(struct list_head *page_list);
+static inline void mem_cgroup_uncharge_list(struct list_head *page_list)
+{
+	if (mem_cgroup_disabled())
+		return;
+	__mem_cgroup_uncharge_list(page_list);
+}
 
 void mem_cgroup_migrate(struct page *oldpage, struct page *newpage);
 
mm/memcontrol.c +10 -19
index a228cd51c4bd..cdaf7003b43d 100644
@@ -6701,8 +6701,7 @@ void mem_cgroup_calculate_protection(struct mem_cgroup *root,
 			atomic_long_read(&parent->memory.children_low_usage)));
 }
 
-static int __mem_cgroup_charge(struct page *page, struct mem_cgroup *memcg,
-			       gfp_t gfp)
+static int charge_memcg(struct page *page, struct mem_cgroup *memcg, gfp_t gfp)
 {
 	unsigned int nr_pages = thp_nr_pages(page);
 	int ret;
@@ -6723,7 +6722,7 @@ static int __mem_cgroup_charge(struct page *page, struct mem_cgroup *memcg,
 }
 
 /**
- * mem_cgroup_charge - charge a newly allocated page to a cgroup
+ * __mem_cgroup_charge - charge a newly allocated page to a cgroup
  * @page: page to charge
  * @mm: mm context of the victim
  * @gfp_mask: reclaim mode
@@ -6736,16 +6735,14 @@ static int __mem_cgroup_charge(struct page *page, struct mem_cgroup *memcg,
  *
  * Returns 0 on success. Otherwise, an error code is returned.
  */
-int mem_cgroup_charge(struct page *page, struct mm_struct *mm, gfp_t gfp_mask)
+int __mem_cgroup_charge(struct page *page, struct mm_struct *mm,
+			gfp_t gfp_mask)
 {
 	struct mem_cgroup *memcg;
 	int ret;
 
-	if (mem_cgroup_disabled())
-		return 0;
-
 	memcg = get_mem_cgroup_from_mm(mm);
-	ret = __mem_cgroup_charge(page, memcg, gfp_mask);
+	ret = charge_memcg(page, memcg, gfp_mask);
 	css_put(&memcg->css);
 
 	return ret;
@@ -6780,7 +6777,7 @@ int mem_cgroup_swapin_charge_page(struct page *page, struct mm_struct *mm,
 		memcg = get_mem_cgroup_from_mm(mm);
 	rcu_read_unlock();
 
-	ret = __mem_cgroup_charge(page, memcg, gfp);
+	ret = charge_memcg(page, memcg, gfp);
 
 	css_put(&memcg->css);
 	return ret;
@@ -6916,18 +6913,15 @@ static void uncharge_page(struct page *page, struct uncharge_gather *ug)
 }
 
 /**
- * mem_cgroup_uncharge - uncharge a page
+ * __mem_cgroup_uncharge - uncharge a page
  * @page: page to uncharge
  *
  * Uncharge a page previously charged with mem_cgroup_charge().
  */
-void mem_cgroup_uncharge(struct page *page)
+void __mem_cgroup_uncharge(struct page *page)
 {
 	struct uncharge_gather ug;
 
-	if (mem_cgroup_disabled())
-		return;
-
 	/* Don't touch page->lru of any random page, pre-check: */
 	if (!page_memcg(page))
 		return;
@@ -6938,20 +6932,17 @@ void mem_cgroup_uncharge(struct page *page)
 }
 
 /**
- * mem_cgroup_uncharge_list - uncharge a list of page
+ * __mem_cgroup_uncharge_list - uncharge a list of page
  * @page_list: list of pages to uncharge
  *
  * Uncharge a list of pages previously charged with
  * mem_cgroup_charge().
  */
-void mem_cgroup_uncharge_list(struct list_head *page_list)
+void __mem_cgroup_uncharge_list(struct list_head *page_list)
 {
 	struct uncharge_gather ug;
 	struct page *page;
 
-	if (mem_cgroup_disabled())
-		return;
-
 	uncharge_gather_clear(&ug);
 	list_for_each_entry(page, page_list, lru)
 		uncharge_page(page, &ug);
Signature · 1 line
2.32.0.93.g670b81a890-goog
Reviewed-by on v3 2/3
#2 lore raw

Re: [External] [PATCH v3 2/3] mm, memcg: inline mem_cgroup_{charge/uncharge} to improve disabled memcg config

Quoted 10 lines · Suren Baghdasaryan

Inline mem_cgroup_{charge/uncharge} and mem_cgroup_uncharge_list functions functions to perform mem_cgroup_disabled static key check inline before calling the main body of the function. This minimizes the memcg overhead in the pagefault and exit_mmap paths when memcgs are disabled using cgroup_disable=memory command-line option.
This change results in ~0.4% overhead reduction when running PFT test comparing {CONFIG_MEMCG=n} against {CONFIG_MEMCG=y, cgroup_disable=memory} configurationon on an 8-core ARM64 Android device.

Quoted 2 lines

But some nits below.

Quoted 101 lines
---
 include/linux/memcontrol.h | 28 +++++++++++++++++++++++++---
 mm/memcontrol.c            | 29 ++++++++++-------------------
 2 files changed, 35 insertions(+), 22 deletions(-)
include/linux/memcontrol.h +25 -3
index bfe5c486f4ad..39fa88051a42 100644
@@ -693,13 +693,35 @@ static inline bool mem_cgroup_below_min(struct mem_cgroup *memcg)
                page_counter_read(&memcg->memory);
 }
 
-int mem_cgroup_charge(struct page *page, struct mm_struct *mm, gfp_t gfp_mask);
+int __mem_cgroup_charge(struct page *page, struct mm_struct *mm,
+                       gfp_t gfp_mask);
+static inline int mem_cgroup_charge(struct page *page, struct mm_struct *mm,
+                                   gfp_t gfp_mask)
+{
+       if (mem_cgroup_disabled())
+               return 0;
+       return __mem_cgroup_charge(page, mm, gfp_mask);
+}
+
 int mem_cgroup_swapin_charge_page(struct page *page, struct mm_struct *mm,
                                  gfp_t gfp, swp_entry_t entry);
 void mem_cgroup_swapin_uncharge_swap(swp_entry_t entry);
 
-void mem_cgroup_uncharge(struct page *page);
-void mem_cgroup_uncharge_list(struct list_head *page_list);
+void __mem_cgroup_uncharge(struct page *page);
+static inline void mem_cgroup_uncharge(struct page *page)
+{
+       if (mem_cgroup_disabled())
+               return;
+       __mem_cgroup_uncharge(page);
+}
+
+void __mem_cgroup_uncharge_list(struct list_head *page_list);
+static inline void mem_cgroup_uncharge_list(struct list_head *page_list)
+{
+       if (mem_cgroup_disabled())
+               return;
+       __mem_cgroup_uncharge_list(page_list);
+}
 
 void mem_cgroup_migrate(struct page *oldpage, struct page *newpage);
 
mm/memcontrol.c +6 -9
index a228cd51c4bd..cdaf7003b43d 100644
@@ -6701,8 +6701,7 @@ void mem_cgroup_calculate_protection(struct mem_cgroup *root,
                        atomic_long_read(&parent->memory.children_low_usage)));
 }
 
-static int __mem_cgroup_charge(struct page *page, struct mem_cgroup *memcg,
-                              gfp_t gfp)
+static int charge_memcg(struct page *page, struct mem_cgroup *memcg, gfp_t gfp)
 {
        unsigned int nr_pages = thp_nr_pages(page);
        int ret;
@@ -6723,7 +6722,7 @@ static int __mem_cgroup_charge(struct page *page, struct mem_cgroup *memcg,
 }
 
 /**
- * mem_cgroup_charge - charge a newly allocated page to a cgroup
+ * __mem_cgroup_charge - charge a newly allocated page to a cgroup
  * @page: page to charge
  * @mm: mm context of the victim
  * @gfp_mask: reclaim mode
@@ -6736,16 +6735,14 @@ static int __mem_cgroup_charge(struct page *page, struct mem_cgroup *memcg,
  *
  * Returns 0 on success. Otherwise, an error code is returned.
  */
-int mem_cgroup_charge(struct page *page, struct mm_struct *mm, gfp_t gfp_mask)
+int __mem_cgroup_charge(struct page *page, struct mm_struct *mm,
+                       gfp_t gfp_mask)
 {
        struct mem_cgroup *memcg;
        int ret;
 
-       if (mem_cgroup_disabled())
-               return 0;
-
        memcg = get_mem_cgroup_from_mm(mm);
-       ret = __mem_cgroup_charge(page, memcg, gfp_mask);
+       ret = charge_memcg(page, memcg, gfp_mask);
        css_put(&memcg->css);
 
        return ret;
@@ -6780,7 +6777,7 @@ int mem_cgroup_swapin_charge_page(struct page *page, struct mm_struct *mm,
                memcg = get_mem_cgroup_from_mm(mm);
        rcu_read_unlock();
 
-       ret = __mem_cgroup_charge(page, memcg, gfp);
+       ret = charge_memcg(page, memcg, gfp);
 
        css_put(&memcg->css);
        return ret;

mm/memcontrol.c static void uncharge_page(struct page *page, struct uncharge_gather *ug) · line 6919

Quoted 8 lines
mm/memcontrol.c +1 -1
@@ -6916,18 +6913,15 @@ static void uncharge_page(struct page *page, struct uncharge_gather *ug)
 }
 
 /**
- * mem_cgroup_uncharge - uncharge a page
+ * __mem_cgroup_uncharge - uncharge a page
  * @page: page to uncharge
  *
  * Uncharge a page previously charged with mem_cgroup_charge().

The comment here also needs to be updated.

mem_cgroup_uncharge() -> __mem_cgroup_uncharge()

Quoted 13 lines
mm/memcontrol.c +1 -4
  */
-void mem_cgroup_uncharge(struct page *page)
+void __mem_cgroup_uncharge(struct page *page)
 {
        struct uncharge_gather ug;
 
-       if (mem_cgroup_disabled())
-               return;
-
        /* Don't touch page->lru of any random page, pre-check: */
        if (!page_memcg(page))
                return;

mm/memcontrol.c void mem_cgroup_uncharge(struct page *page) · line 6939

Quoted 9 lines
mm/memcontrol.c +1 -1
@@ -6938,20 +6932,17 @@ void mem_cgroup_uncharge(struct page *page)
 }
 
 /**
- * mem_cgroup_uncharge_list - uncharge a list of page
+ * __mem_cgroup_uncharge_list - uncharge a list of page
  * @page_list: list of pages to uncharge
  *
  * Uncharge a list of pages previously charged with
  * mem_cgroup_charge().

Should be __mem_cgroup_charge().

Thanks.

Quoted 16 lines
mm/memcontrol.c +1 -5
  */
-void mem_cgroup_uncharge_list(struct list_head *page_list)
+void __mem_cgroup_uncharge_list(struct list_head *page_list)
 {
        struct uncharge_gather ug;
        struct page *page;
 
-       if (mem_cgroup_disabled())
-               return;
-
        uncharge_gather_clear(&ug);
        list_for_each_entry(page, page_list, lru)
                uncharge_page(page, &ug);
--

2.32.0.93.g670b81a890-goog

author reviewer
#3 lore raw

Re: [External] [PATCH v3 2/3] mm, memcg: inline mem_cgroup_{charge/uncharge} to improve disabled memcg config

Quoted 148 lines · Muchun Song
Quoted 10 lines · Suren Baghdasaryan

Inline mem_cgroup_{charge/uncharge} and mem_cgroup_uncharge_list functions functions to perform mem_cgroup_disabled static key check inline before calling the main body of the function. This minimizes the memcg overhead in the pagefault and exit_mmap paths when memcgs are disabled using cgroup_disable=memory command-line option.
This change results in ~0.4% overhead reduction when running PFT test comparing {CONFIG_MEMCG=n} against {CONFIG_MEMCG=y, cgroup_disable=memory} configurationon on an 8-core ARM64 Android device.

Quoted 2 lines

But some nits below.

Quoted 101 lines
---
 include/linux/memcontrol.h | 28 +++++++++++++++++++++++++---
 mm/memcontrol.c            | 29 ++++++++++-------------------
 2 files changed, 35 insertions(+), 22 deletions(-)
include/linux/memcontrol.h +25 -3
index bfe5c486f4ad..39fa88051a42 100644
@@ -693,13 +693,35 @@ static inline bool mem_cgroup_below_min(struct mem_cgroup *memcg)
                page_counter_read(&memcg->memory);
 }
 
-int mem_cgroup_charge(struct page *page, struct mm_struct *mm, gfp_t gfp_mask);
+int __mem_cgroup_charge(struct page *page, struct mm_struct *mm,
+                       gfp_t gfp_mask);
+static inline int mem_cgroup_charge(struct page *page, struct mm_struct *mm,
+                                   gfp_t gfp_mask)
+{
+       if (mem_cgroup_disabled())
+               return 0;
+       return __mem_cgroup_charge(page, mm, gfp_mask);
+}
+
 int mem_cgroup_swapin_charge_page(struct page *page, struct mm_struct *mm,
                                  gfp_t gfp, swp_entry_t entry);
 void mem_cgroup_swapin_uncharge_swap(swp_entry_t entry);
 
-void mem_cgroup_uncharge(struct page *page);
-void mem_cgroup_uncharge_list(struct list_head *page_list);
+void __mem_cgroup_uncharge(struct page *page);
+static inline void mem_cgroup_uncharge(struct page *page)
+{
+       if (mem_cgroup_disabled())
+               return;
+       __mem_cgroup_uncharge(page);
+}
+
+void __mem_cgroup_uncharge_list(struct list_head *page_list);
+static inline void mem_cgroup_uncharge_list(struct list_head *page_list)
+{
+       if (mem_cgroup_disabled())
+               return;
+       __mem_cgroup_uncharge_list(page_list);
+}
 
 void mem_cgroup_migrate(struct page *oldpage, struct page *newpage);
 
mm/memcontrol.c +6 -9
index a228cd51c4bd..cdaf7003b43d 100644
@@ -6701,8 +6701,7 @@ void mem_cgroup_calculate_protection(struct mem_cgroup *root,
                        atomic_long_read(&parent->memory.children_low_usage)));
 }
 
-static int __mem_cgroup_charge(struct page *page, struct mem_cgroup *memcg,
-                              gfp_t gfp)
+static int charge_memcg(struct page *page, struct mem_cgroup *memcg, gfp_t gfp)
 {
        unsigned int nr_pages = thp_nr_pages(page);
        int ret;
@@ -6723,7 +6722,7 @@ static int __mem_cgroup_charge(struct page *page, struct mem_cgroup *memcg,
 }
 
 /**
- * mem_cgroup_charge - charge a newly allocated page to a cgroup
+ * __mem_cgroup_charge - charge a newly allocated page to a cgroup
  * @page: page to charge
  * @mm: mm context of the victim
  * @gfp_mask: reclaim mode
@@ -6736,16 +6735,14 @@ static int __mem_cgroup_charge(struct page *page, struct mem_cgroup *memcg,
  *
  * Returns 0 on success. Otherwise, an error code is returned.
  */
-int mem_cgroup_charge(struct page *page, struct mm_struct *mm, gfp_t gfp_mask)
+int __mem_cgroup_charge(struct page *page, struct mm_struct *mm,
+                       gfp_t gfp_mask)
 {
        struct mem_cgroup *memcg;
        int ret;
 
-       if (mem_cgroup_disabled())
-               return 0;
-
        memcg = get_mem_cgroup_from_mm(mm);
-       ret = __mem_cgroup_charge(page, memcg, gfp_mask);
+       ret = charge_memcg(page, memcg, gfp_mask);
        css_put(&memcg->css);
 
        return ret;
@@ -6780,7 +6777,7 @@ int mem_cgroup_swapin_charge_page(struct page *page, struct mm_struct *mm,
                memcg = get_mem_cgroup_from_mm(mm);
        rcu_read_unlock();
 
-       ret = __mem_cgroup_charge(page, memcg, gfp);
+       ret = charge_memcg(page, memcg, gfp);
 
        css_put(&memcg->css);
        return ret;

mm/memcontrol.c static void uncharge_page(struct page *page, struct uncharge_gather *ug) · line 6919

Quoted 8 lines
mm/memcontrol.c +1 -1
@@ -6916,18 +6913,15 @@ static void uncharge_page(struct page *page, struct uncharge_gather *ug)
 }
 
 /**
- * mem_cgroup_uncharge - uncharge a page
+ * __mem_cgroup_uncharge - uncharge a page
  * @page: page to uncharge
  *
  * Uncharge a page previously charged with mem_cgroup_charge().

The comment here also needs to be updated.

mem_cgroup_uncharge() -> __mem_cgroup_uncharge()

Quoted 15 lines
mm/memcontrol.c +1 -4
  */
-void mem_cgroup_uncharge(struct page *page)
+void __mem_cgroup_uncharge(struct page *page)
 {
        struct uncharge_gather ug;
 
-       if (mem_cgroup_disabled())
-               return;
-
        /* Don't touch page->lru of any random page, pre-check: */
        if (!page_memcg(page))
                return;
@@ -6938,20 +6932,17 @@ void mem_cgroup_uncharge(struct page *page)
 }
 
Quoted 9 lines

mm/memcontrol.c void mem_cgroup_uncharge(struct page *page) · line 6939

Quoted 7 lines
mm/memcontrol.c +1 -1
 /**
- * mem_cgroup_uncharge_list - uncharge a list of page
+ * __mem_cgroup_uncharge_list - uncharge a list of page
  * @page_list: list of pages to uncharge
  *
  * Uncharge a list of pages previously charged with
  * mem_cgroup_charge().

Should be __mem_cgroup_charge().

Fixed and posted in v4 at:
https://lore.kernel.org/patchwork/project/lkml/list/?series=507943
Thanks!

Quoted 19 lines

Thanks.

Quoted 16 lines
mm/memcontrol.c +1 -5
  */
-void mem_cgroup_uncharge_list(struct list_head *page_list)
+void __mem_cgroup_uncharge_list(struct list_head *page_list)
 {
        struct uncharge_gather ug;
        struct page *page;
 
-       if (mem_cgroup_disabled())
-               return;
-
        uncharge_gather_clear(&ug);
        list_for_each_entry(page, page_list, lru)
                uncharge_page(page, &ug);
--

2.32.0.93.g670b81a890-goog

reviewer Acked-by on v3 2/3
#4 lore raw

Re: [PATCH v3 2/3] mm, memcg: inline mem_cgroup_{charge/uncharge} to improve disabled memcg config

In reply to Suren Baghdasaryan
Quoted 11 lines · Suren Baghdasaryan

Inline mem_cgroup_{charge/uncharge} and mem_cgroup_uncharge_list functions functions to perform mem_cgroup_disabled static key check inline before calling the main body of the function. This minimizes the memcg overhead in the pagefault and exit_mmap paths when memcgs are disabled using cgroup_disable=memory command-line option.
This change results in ~0.4% overhead reduction when running PFT test comparing {CONFIG_MEMCG=n} against {CONFIG_MEMCG=y, cgroup_disable=memory} configurationon on an 8-core ARM64 Android device.

With doc updated as suggested by Muchun Song

Thanks!

Quoted 146 lines
---
 include/linux/memcontrol.h | 28 +++++++++++++++++++++++++---
 mm/memcontrol.c            | 29 ++++++++++-------------------
 2 files changed, 35 insertions(+), 22 deletions(-)
include/linux/memcontrol.h +25 -3
index bfe5c486f4ad..39fa88051a42 100644
@@ -693,13 +693,35 @@ static inline bool mem_cgroup_below_min(struct mem_cgroup *memcg)
 		page_counter_read(&memcg->memory);
 }
 
-int mem_cgroup_charge(struct page *page, struct mm_struct *mm, gfp_t gfp_mask);
+int __mem_cgroup_charge(struct page *page, struct mm_struct *mm,
+			gfp_t gfp_mask);
+static inline int mem_cgroup_charge(struct page *page, struct mm_struct *mm,
+				    gfp_t gfp_mask)
+{
+	if (mem_cgroup_disabled())
+		return 0;
+	return __mem_cgroup_charge(page, mm, gfp_mask);
+}
+
 int mem_cgroup_swapin_charge_page(struct page *page, struct mm_struct *mm,
 				  gfp_t gfp, swp_entry_t entry);
 void mem_cgroup_swapin_uncharge_swap(swp_entry_t entry);
 
-void mem_cgroup_uncharge(struct page *page);
-void mem_cgroup_uncharge_list(struct list_head *page_list);
+void __mem_cgroup_uncharge(struct page *page);
+static inline void mem_cgroup_uncharge(struct page *page)
+{
+	if (mem_cgroup_disabled())
+		return;
+	__mem_cgroup_uncharge(page);
+}
+
+void __mem_cgroup_uncharge_list(struct list_head *page_list);
+static inline void mem_cgroup_uncharge_list(struct list_head *page_list)
+{
+	if (mem_cgroup_disabled())
+		return;
+	__mem_cgroup_uncharge_list(page_list);
+}
 
 void mem_cgroup_migrate(struct page *oldpage, struct page *newpage);
 
mm/memcontrol.c +10 -19
index a228cd51c4bd..cdaf7003b43d 100644
@@ -6701,8 +6701,7 @@ void mem_cgroup_calculate_protection(struct mem_cgroup *root,
 			atomic_long_read(&parent->memory.children_low_usage)));
 }
 
-static int __mem_cgroup_charge(struct page *page, struct mem_cgroup *memcg,
-			       gfp_t gfp)
+static int charge_memcg(struct page *page, struct mem_cgroup *memcg, gfp_t gfp)
 {
 	unsigned int nr_pages = thp_nr_pages(page);
 	int ret;
@@ -6723,7 +6722,7 @@ static int __mem_cgroup_charge(struct page *page, struct mem_cgroup *memcg,
 }
 
 /**
- * mem_cgroup_charge - charge a newly allocated page to a cgroup
+ * __mem_cgroup_charge - charge a newly allocated page to a cgroup
  * @page: page to charge
  * @mm: mm context of the victim
  * @gfp_mask: reclaim mode
@@ -6736,16 +6735,14 @@ static int __mem_cgroup_charge(struct page *page, struct mem_cgroup *memcg,
  *
  * Returns 0 on success. Otherwise, an error code is returned.
  */
-int mem_cgroup_charge(struct page *page, struct mm_struct *mm, gfp_t gfp_mask)
+int __mem_cgroup_charge(struct page *page, struct mm_struct *mm,
+			gfp_t gfp_mask)
 {
 	struct mem_cgroup *memcg;
 	int ret;
 
-	if (mem_cgroup_disabled())
-		return 0;
-
 	memcg = get_mem_cgroup_from_mm(mm);
-	ret = __mem_cgroup_charge(page, memcg, gfp_mask);
+	ret = charge_memcg(page, memcg, gfp_mask);
 	css_put(&memcg->css);
 
 	return ret;
@@ -6780,7 +6777,7 @@ int mem_cgroup_swapin_charge_page(struct page *page, struct mm_struct *mm,
 		memcg = get_mem_cgroup_from_mm(mm);
 	rcu_read_unlock();
 
-	ret = __mem_cgroup_charge(page, memcg, gfp);
+	ret = charge_memcg(page, memcg, gfp);
 
 	css_put(&memcg->css);
 	return ret;
@@ -6916,18 +6913,15 @@ static void uncharge_page(struct page *page, struct uncharge_gather *ug)
 }
 
 /**
- * mem_cgroup_uncharge - uncharge a page
+ * __mem_cgroup_uncharge - uncharge a page
  * @page: page to uncharge
  *
  * Uncharge a page previously charged with mem_cgroup_charge().
  */
-void mem_cgroup_uncharge(struct page *page)
+void __mem_cgroup_uncharge(struct page *page)
 {
 	struct uncharge_gather ug;
 
-	if (mem_cgroup_disabled())
-		return;
-
 	/* Don't touch page->lru of any random page, pre-check: */
 	if (!page_memcg(page))
 		return;
@@ -6938,20 +6932,17 @@ void mem_cgroup_uncharge(struct page *page)
 }
 
 /**
- * mem_cgroup_uncharge_list - uncharge a list of page
+ * __mem_cgroup_uncharge_list - uncharge a list of page
  * @page_list: list of pages to uncharge
  *
  * Uncharge a list of pages previously charged with
  * mem_cgroup_charge().
  */
-void mem_cgroup_uncharge_list(struct list_head *page_list)
+void __mem_cgroup_uncharge_list(struct list_head *page_list)
 {
 	struct uncharge_gather ug;
 	struct page *page;
 
-	if (mem_cgroup_disabled())
-		return;
-
 	uncharge_gather_clear(&ug);
 	list_for_each_entry(page, page_list, lru)
 		uncharge_page(page, &ug);
Signature · 1 line
2.32.0.93.g670b81a890-goog
Signature · 2 lines
Michal Hocko
SUSE Labs
author reviewer
#5 lore raw

Re: [PATCH v3 2/3] mm, memcg: inline mem_cgroup_{charge/uncharge} to improve disabled memcg config

Quoted 11 lines · Michal Hocko
Quoted 9 lines · Suren Baghdasaryan

Inline mem_cgroup_{charge/uncharge} and mem_cgroup_uncharge_list functions functions to perform mem_cgroup_disabled static key check inline before calling the main body of the function. This minimizes the memcg overhead in the pagefault and exit_mmap paths when memcgs are disabled using cgroup_disable=memory command-line option.
This change results in ~0.4% overhead reduction when running PFT test comparing {CONFIG_MEMCG=n} against {CONFIG_MEMCG=y, cgroup_disable=memory} configurationon on an 8-core ARM64 Android device.

Quoted 5 lines
Quoted 2 lines

With doc updated as suggested by Muchun Song

Thanks! Will fix the comment and post v4 later today.

Quoted 153 lines

Thanks!

Quoted 145 lines
---
 include/linux/memcontrol.h | 28 +++++++++++++++++++++++++---
 mm/memcontrol.c            | 29 ++++++++++-------------------
 2 files changed, 35 insertions(+), 22 deletions(-)
include/linux/memcontrol.h +25 -3
index bfe5c486f4ad..39fa88051a42 100644
@@ -693,13 +693,35 @@ static inline bool mem_cgroup_below_min(struct mem_cgroup *memcg)
              page_counter_read(&memcg->memory);
 }
 
-int mem_cgroup_charge(struct page *page, struct mm_struct *mm, gfp_t gfp_mask);
+int __mem_cgroup_charge(struct page *page, struct mm_struct *mm,
+                     gfp_t gfp_mask);
+static inline int mem_cgroup_charge(struct page *page, struct mm_struct *mm,
+                                 gfp_t gfp_mask)
+{
+     if (mem_cgroup_disabled())
+             return 0;
+     return __mem_cgroup_charge(page, mm, gfp_mask);
+}
+
 int mem_cgroup_swapin_charge_page(struct page *page, struct mm_struct *mm,
                                gfp_t gfp, swp_entry_t entry);
 void mem_cgroup_swapin_uncharge_swap(swp_entry_t entry);
 
-void mem_cgroup_uncharge(struct page *page);
-void mem_cgroup_uncharge_list(struct list_head *page_list);
+void __mem_cgroup_uncharge(struct page *page);
+static inline void mem_cgroup_uncharge(struct page *page)
+{
+     if (mem_cgroup_disabled())
+             return;
+     __mem_cgroup_uncharge(page);
+}
+
+void __mem_cgroup_uncharge_list(struct list_head *page_list);
+static inline void mem_cgroup_uncharge_list(struct list_head *page_list)
+{
+     if (mem_cgroup_disabled())
+             return;
+     __mem_cgroup_uncharge_list(page_list);
+}
 
 void mem_cgroup_migrate(struct page *oldpage, struct page *newpage);
 
mm/memcontrol.c +10 -20
index a228cd51c4bd..cdaf7003b43d 100644
@@ -6701,8 +6701,7 @@ void mem_cgroup_calculate_protection(struct mem_cgroup *root,
                      atomic_long_read(&parent->memory.children_low_usage)));
 }
 
-static int __mem_cgroup_charge(struct page *page, struct mem_cgroup *memcg,
-                            gfp_t gfp)
+static int charge_memcg(struct page *page, struct mem_cgroup *memcg, gfp_t gfp)
 {
      unsigned int nr_pages = thp_nr_pages(page);
      int ret;
@@ -6723,7 +6722,7 @@ static int __mem_cgroup_charge(struct page *page, struct mem_cgroup *memcg,
 }
 
 /**
- * mem_cgroup_charge - charge a newly allocated page to a cgroup
+ * __mem_cgroup_charge - charge a newly allocated page to a cgroup
  * @page: page to charge
  * @mm: mm context of the victim
  * @gfp_mask: reclaim mode
@@ -6736,16 +6735,14 @@ static int __mem_cgroup_charge(struct page *page, struct mem_cgroup *memcg,
  *
  * Returns 0 on success. Otherwise, an error code is returned.
  */
-int mem_cgroup_charge(struct page *page, struct mm_struct *mm, gfp_t gfp_mask)
+int __mem_cgroup_charge(struct page *page, struct mm_struct *mm,
+                     gfp_t gfp_mask)
 {
      struct mem_cgroup *memcg;
      int ret;
 
-     if (mem_cgroup_disabled())
-             return 0;
-
      memcg = get_mem_cgroup_from_mm(mm);
-     ret = __mem_cgroup_charge(page, memcg, gfp_mask);
+     ret = charge_memcg(page, memcg, gfp_mask);
      css_put(&memcg->css);
 
      return ret;
@@ -6780,7 +6777,7 @@ int mem_cgroup_swapin_charge_page(struct page *page, struct mm_struct *mm,
              memcg = get_mem_cgroup_from_mm(mm);
      rcu_read_unlock();
 
-     ret = __mem_cgroup_charge(page, memcg, gfp);
+     ret = charge_memcg(page, memcg, gfp);
 
      css_put(&memcg->css);
      return ret;
@@ -6916,18 +6913,15 @@ static void uncharge_page(struct page *page, struct uncharge_gather *ug)
 }
 
 /**
- * mem_cgroup_uncharge - uncharge a page
+ * __mem_cgroup_uncharge - uncharge a page
  * @page: page to uncharge
  *
  * Uncharge a page previously charged with mem_cgroup_charge().
  */
-void mem_cgroup_uncharge(struct page *page)
+void __mem_cgroup_uncharge(struct page *page)
 {
      struct uncharge_gather ug;
 
-     if (mem_cgroup_disabled())
-             return;
-
      /* Don't touch page->lru of any random page, pre-check: */
      if (!page_memcg(page))
              return;
@@ -6938,20 +6932,17 @@ void mem_cgroup_uncharge(struct page *page)
 }
 
 /**
- * mem_cgroup_uncharge_list - uncharge a list of page
+ * __mem_cgroup_uncharge_list - uncharge a list of page
  * @page_list: list of pages to uncharge
  *
  * Uncharge a list of pages previously charged with
  * mem_cgroup_charge().
  */
-void mem_cgroup_uncharge_list(struct list_head *page_list)
+void __mem_cgroup_uncharge_list(struct list_head *page_list)
 {
      struct uncharge_gather ug;
      struct page *page;
 
-     if (mem_cgroup_disabled())
-             return;
-
      uncharge_gather_clear(&ug);
      list_for_each_entry(page, page_list, lru)
              uncharge_page(page, &ug);
--
Quoted 1 line

2.32.0.93.g670b81a890-goog

--
Michal Hocko
SUSE Labs

#6 lore raw

Re: [PATCH v3 2/3] mm, memcg: inline mem_cgroup_{charge/uncharge} to improve disabled memcg config

In reply to Suren Baghdasaryan

mm/memcontrol.c static int __mem_cgroup_charge(struct page *page, struct mem_cgroup *memcg, · line 6728

Quoted 9 lines · Suren Baghdasaryan
mm/memcontrol.c +1 -1
@@ -6723,7 +6722,7 @@ static int __mem_cgroup_charge(struct page *page, struct mem_cgroup *memcg,
 }
 
 /**
- * mem_cgroup_charge - charge a newly allocated page to a cgroup
+ * __mem_cgroup_charge - charge a newly allocated page to a cgroup
  * @page: page to charge
  * @mm: mm context of the victim
  * @gfp_mask: reclaim mode

This patch conflicts with the folio work, so I'm just rebasing the folio patches on top of this, and I think this part of the patch is a mistake. We don't want to document the __mem_cgroup_charge() function. That's an implementation detail. This patch should instead have moved the kernel-doc to memcontrol.h and continued to document mem_cgroup_charge().

author reviewer
#7 lore raw

Re: [PATCH v3 2/3] mm, memcg: inline mem_cgroup_{charge/uncharge} to improve disabled memcg config

Quoted 5 lines · Matthew Wilcox
Quoted 3 lines · Suren Baghdasaryan
(quoted diff) +0 -0
@@ -6723,7 +6722,7 @@ static int __mem_cgroup_charge(struct page *page, struct mem_cgroup *memcg,
 }
 
Quoted 12 lines

mm/memcontrol.c static int __mem_cgroup_charge(struct page *page, struct mem_cgroup *memcg, · line 6728

Quoted 6 lines
mm/memcontrol.c +1 -1
 /**
- * mem_cgroup_charge - charge a newly allocated page to a cgroup
+ * __mem_cgroup_charge - charge a newly allocated page to a cgroup
  * @page: page to charge
  * @mm: mm context of the victim
  * @gfp_mask: reclaim mode

This patch conflicts with the folio work, so I'm just rebasing the folio patches on top of this, and I think this part of the patch is a mistake. We don't want to document the __mem_cgroup_charge() function. That's an implementation detail. This patch should instead have moved the kernel-doc to memcontrol.h and continued to document mem_cgroup_charge().

Ack.
There was a v4 version of this patch:
https://lore.kernel.org/patchwork/patch/1458907 which was picked up by Andrew already. If others agree that documentation should be moved into the header file then I'll gladly post another version. Or I can post a separate patch moving the documentation only. Whatever works best. Andrew, Michal, Johannes, WDYT?

#8 lore raw

Re: [PATCH v3 2/3] mm, memcg: inline mem_cgroup_{charge/uncharge} to improve disabled memcg config

Quoted 19 lines · Suren Baghdasaryan
Quoted 5 lines · Matthew Wilcox
Quoted 3 lines · Suren Baghdasaryan
(quoted diff) +0 -0
@@ -6723,7 +6722,7 @@ static int __mem_cgroup_charge(struct page *page, struct mem_cgroup *memcg,
 }
 
Quoted 12 lines

mm/memcontrol.c static int __mem_cgroup_charge(struct page *page, struct mem_cgroup *memcg, · line 6728

Quoted 6 lines
mm/memcontrol.c +1 -1
 /**
- * mem_cgroup_charge - charge a newly allocated page to a cgroup
+ * __mem_cgroup_charge - charge a newly allocated page to a cgroup
  * @page: page to charge
  * @mm: mm context of the victim
  * @gfp_mask: reclaim mode

This patch conflicts with the folio work, so I'm just rebasing the folio patches on top of this, and I think this part of the patch is a mistake. We don't want to document the __mem_cgroup_charge() function. That's an implementation detail. This patch should instead have moved the kernel-doc to memcontrol.h and continued to document mem_cgroup_charge().

Quoted 7 lines

Ack.
There was a v4 version of this patch:
https://lore.kernel.org/patchwork/patch/1458907 which was picked up by Andrew already. If others agree that documentation should be moved into the header file then I'll gladly post another version. Or I can post a separate patch moving the documentation only. Whatever works best. Andrew, Michal, Johannes, WDYT?

At this point, I've moved the documentation as part of the folio patch. I'd rather not redo that patch again ...

author reviewer
#9 lore raw

Re: [PATCH v3 2/3] mm, memcg: inline mem_cgroup_{charge/uncharge} to improve disabled memcg config

Quoted 29 lines · Matthew Wilcox
Quoted 19 lines · Suren Baghdasaryan
Quoted 5 lines · Matthew Wilcox
Quoted 3 lines · Suren Baghdasaryan
(quoted diff) +0 -0
@@ -6723,7 +6722,7 @@ static int __mem_cgroup_charge(struct page *page, struct mem_cgroup *memcg,
 }
 
Quoted 12 lines

mm/memcontrol.c static int __mem_cgroup_charge(struct page *page, struct mem_cgroup *memcg, · line 6728

Quoted 6 lines
mm/memcontrol.c +1 -1
 /**
- * mem_cgroup_charge - charge a newly allocated page to a cgroup
+ * __mem_cgroup_charge - charge a newly allocated page to a cgroup
  * @page: page to charge
  * @mm: mm context of the victim
  * @gfp_mask: reclaim mode

This patch conflicts with the folio work, so I'm just rebasing the folio patches on top of this, and I think this part of the patch is a mistake. We don't want to document the __mem_cgroup_charge() function. That's an implementation detail. This patch should instead have moved the kernel-doc to memcontrol.h and continued to document mem_cgroup_charge().

Quoted 7 lines

Ack.
There was a v4 version of this patch:
https://lore.kernel.org/patchwork/patch/1458907 which was picked up by Andrew already. If others agree that documentation should be moved into the header file then I'll gladly post another version. Or I can post a separate patch moving the documentation only. Whatever works best. Andrew, Michal, Johannes, WDYT?

Quoted 2 lines

At this point, I've moved the documentation as part of the folio patch. I'd rather not redo that patch again ...

Ok. If you need me to redo anything please let me know.

author reviewer
#10 lore raw

PATCH v3 3/3 mm, memcg: inline swap-related functions to improve disabled memcg config

In reply to Suren Baghdasaryan

Inline mem_cgroup_try_charge_swap, mem_cgroup_uncharge_swap and cgroup_throttle_swaprate functions to perform mem_cgroup_disabled static key check inline before calling the main body of the function. This minimizes the memcg overhead in the pagefault and exit_mmap paths when memcgs are disabled using cgroup_disable=memory command-line option. This change results in ~1% overhead reduction when running PFT test comparing {CONFIG_MEMCG=n} against {CONFIG_MEMCG=y, cgroup_disable=memory} configuration on an 8-core ARM64 Android device.

---
 include/linux/swap.h | 26 +++++++++++++++++++++++---
 mm/memcontrol.c      | 14 ++++----------
 mm/swapfile.c        |  5 +----
 3 files changed, 28 insertions(+), 17 deletions(-)
include/linux/swap.h +23 -3
index 6f5a43251593..f30d26b0f71d 100644
@@ -721,7 +721,13 @@ static inline int mem_cgroup_swappiness(struct mem_cgroup *mem)
 #endif
 
 #if defined(CONFIG_SWAP) && defined(CONFIG_MEMCG) && defined(CONFIG_BLK_CGROUP)
-extern void cgroup_throttle_swaprate(struct page *page, gfp_t gfp_mask);
+extern void __cgroup_throttle_swaprate(struct page *page, gfp_t gfp_mask);
+static inline  void cgroup_throttle_swaprate(struct page *page, gfp_t gfp_mask)
+{
+	if (mem_cgroup_disabled())
+		return;
+	__cgroup_throttle_swaprate(page, gfp_mask);
+}
 #else
 static inline void cgroup_throttle_swaprate(struct page *page, gfp_t gfp_mask)
 {
@@ -730,8 +736,22 @@ static inline void cgroup_throttle_swaprate(struct page *page, gfp_t gfp_mask)
 
 #ifdef CONFIG_MEMCG_SWAP
 extern void mem_cgroup_swapout(struct page *page, swp_entry_t entry);
-extern int mem_cgroup_try_charge_swap(struct page *page, swp_entry_t entry);
-extern void mem_cgroup_uncharge_swap(swp_entry_t entry, unsigned int nr_pages);
+extern int __mem_cgroup_try_charge_swap(struct page *page, swp_entry_t entry);
+static inline int mem_cgroup_try_charge_swap(struct page *page, swp_entry_t entry)
+{
+	if (mem_cgroup_disabled())
+		return 0;
+	return __mem_cgroup_try_charge_swap(page, entry);
+}
+
+extern void __mem_cgroup_uncharge_swap(swp_entry_t entry, unsigned int nr_pages);
+static inline void mem_cgroup_uncharge_swap(swp_entry_t entry, unsigned int nr_pages)
+{
+	if (mem_cgroup_disabled())
+		return;
+	__mem_cgroup_uncharge_swap(entry, nr_pages);
+}
+
 extern long mem_cgroup_get_nr_swap_pages(struct mem_cgroup *memcg);
 extern bool mem_cgroup_swap_full(struct page *page);
 #else
mm/memcontrol.c +4 -10
index cdaf7003b43d..0b05322836ec 100644
@@ -7234,7 +7234,7 @@ void mem_cgroup_swapout(struct page *page, swp_entry_t entry)
 }
 
 /**
- * mem_cgroup_try_charge_swap - try charging swap space for a page
+ * __mem_cgroup_try_charge_swap - try charging swap space for a page
  * @page: page being added to swap
  * @entry: swap entry to charge
  *
@@ -7242,16 +7242,13 @@ void mem_cgroup_swapout(struct page *page, swp_entry_t entry)
  *
  * Returns 0 on success, -ENOMEM on failure.
  */
-int mem_cgroup_try_charge_swap(struct page *page, swp_entry_t entry)
+int __mem_cgroup_try_charge_swap(struct page *page, swp_entry_t entry)
 {
 	unsigned int nr_pages = thp_nr_pages(page);
 	struct page_counter *counter;
 	struct mem_cgroup *memcg;
 	unsigned short oldid;
 
-	if (mem_cgroup_disabled())
-		return 0;
-
 	if (!cgroup_subsys_on_dfl(memory_cgrp_subsys))
 		return 0;
 
@@ -7287,18 +7284,15 @@ int mem_cgroup_try_charge_swap(struct page *page, swp_entry_t entry)
 }
 
 /**
- * mem_cgroup_uncharge_swap - uncharge swap space
+ * __mem_cgroup_uncharge_swap - uncharge swap space
  * @entry: swap entry to uncharge
  * @nr_pages: the amount of swap space to uncharge
  */
-void mem_cgroup_uncharge_swap(swp_entry_t entry, unsigned int nr_pages)
+void __mem_cgroup_uncharge_swap(swp_entry_t entry, unsigned int nr_pages)
 {
 	struct mem_cgroup *memcg;
 	unsigned short id;
 
-	if (mem_cgroup_disabled())
-		return;
-
 	id = swap_cgroup_record(entry, 0, nr_pages);
 	rcu_read_lock();
 	memcg = mem_cgroup_from_id(id);
mm/swapfile.c +1 -4
index 707fa0481bb4..04a0c83f1313 100644
@@ -3773,14 +3773,11 @@ static void free_swap_count_continuations(struct swap_info_struct *si)
 }
 
 #if defined(CONFIG_MEMCG) && defined(CONFIG_BLK_CGROUP)
-void cgroup_throttle_swaprate(struct page *page, gfp_t gfp_mask)
+void __cgroup_throttle_swaprate(struct page *page, gfp_t gfp_mask)
 {
 	struct swap_info_struct *si, *next;
 	int nid = page_to_nid(page);
 
-	if (mem_cgroup_disabled())
-		return;
-
 	if (!(gfp_mask & __GFP_IO))
 		return;
 
Signature · 1 line
2.32.0.93.g670b81a890-goog
Reviewed-by on v3 3/3
#11 lore raw

Re: [External] [PATCH v3 3/3] mm, memcg: inline swap-related functions to improve disabled memcg config

Quoted 10 lines · Suren Baghdasaryan

Inline mem_cgroup_try_charge_swap, mem_cgroup_uncharge_swap and cgroup_throttle_swaprate functions to perform mem_cgroup_disabled static key check inline before calling the main body of the function. This minimizes the memcg overhead in the pagefault and exit_mmap paths when memcgs are disabled using cgroup_disable=memory command-line option. This change results in ~1% overhead reduction when running PFT test comparing {CONFIG_MEMCG=n} against {CONFIG_MEMCG=y, cgroup_disable=memory} configuration on an 8-core ARM64 Android device.

Quoted 3 lines

LGTM.

Thanks.

reviewer Acked-by on v3 3/3
#12 lore raw

Re: [PATCH v3 3/3] mm, memcg: inline swap-related functions to improve disabled memcg config

In reply to Suren Baghdasaryan
Quoted 9 lines · Suren Baghdasaryan

Inline mem_cgroup_try_charge_swap, mem_cgroup_uncharge_swap and cgroup_throttle_swaprate functions to perform mem_cgroup_disabled static key check inline before calling the main body of the function. This minimizes the memcg overhead in the pagefault and exit_mmap paths when memcgs are disabled using cgroup_disable=memory command-line option. This change results in ~1% overhead reduction when running PFT test comparing {CONFIG_MEMCG=n} against {CONFIG_MEMCG=y, cgroup_disable=memory} configuration on an 8-core ARM64 Android device.

Quoted 3 lines

I find it a bit surprising to see such a big difference over a function call in a slow path like swap in/out.

Anyway the change makes sense.

Thanks!

Quoted 124 lines
---
 include/linux/swap.h | 26 +++++++++++++++++++++++---
 mm/memcontrol.c      | 14 ++++----------
 mm/swapfile.c        |  5 +----
 3 files changed, 28 insertions(+), 17 deletions(-)
include/linux/swap.h +23 -3
index 6f5a43251593..f30d26b0f71d 100644
@@ -721,7 +721,13 @@ static inline int mem_cgroup_swappiness(struct mem_cgroup *mem)
 #endif
 
 #if defined(CONFIG_SWAP) && defined(CONFIG_MEMCG) && defined(CONFIG_BLK_CGROUP)
-extern void cgroup_throttle_swaprate(struct page *page, gfp_t gfp_mask);
+extern void __cgroup_throttle_swaprate(struct page *page, gfp_t gfp_mask);
+static inline  void cgroup_throttle_swaprate(struct page *page, gfp_t gfp_mask)
+{
+	if (mem_cgroup_disabled())
+		return;
+	__cgroup_throttle_swaprate(page, gfp_mask);
+}
 #else
 static inline void cgroup_throttle_swaprate(struct page *page, gfp_t gfp_mask)
 {
@@ -730,8 +736,22 @@ static inline void cgroup_throttle_swaprate(struct page *page, gfp_t gfp_mask)
 
 #ifdef CONFIG_MEMCG_SWAP
 extern void mem_cgroup_swapout(struct page *page, swp_entry_t entry);
-extern int mem_cgroup_try_charge_swap(struct page *page, swp_entry_t entry);
-extern void mem_cgroup_uncharge_swap(swp_entry_t entry, unsigned int nr_pages);
+extern int __mem_cgroup_try_charge_swap(struct page *page, swp_entry_t entry);
+static inline int mem_cgroup_try_charge_swap(struct page *page, swp_entry_t entry)
+{
+	if (mem_cgroup_disabled())
+		return 0;
+	return __mem_cgroup_try_charge_swap(page, entry);
+}
+
+extern void __mem_cgroup_uncharge_swap(swp_entry_t entry, unsigned int nr_pages);
+static inline void mem_cgroup_uncharge_swap(swp_entry_t entry, unsigned int nr_pages)
+{
+	if (mem_cgroup_disabled())
+		return;
+	__mem_cgroup_uncharge_swap(entry, nr_pages);
+}
+
 extern long mem_cgroup_get_nr_swap_pages(struct mem_cgroup *memcg);
 extern bool mem_cgroup_swap_full(struct page *page);
 #else
mm/memcontrol.c +4 -10
index cdaf7003b43d..0b05322836ec 100644
@@ -7234,7 +7234,7 @@ void mem_cgroup_swapout(struct page *page, swp_entry_t entry)
 }
 
 /**
- * mem_cgroup_try_charge_swap - try charging swap space for a page
+ * __mem_cgroup_try_charge_swap - try charging swap space for a page
  * @page: page being added to swap
  * @entry: swap entry to charge
  *
@@ -7242,16 +7242,13 @@ void mem_cgroup_swapout(struct page *page, swp_entry_t entry)
  *
  * Returns 0 on success, -ENOMEM on failure.
  */
-int mem_cgroup_try_charge_swap(struct page *page, swp_entry_t entry)
+int __mem_cgroup_try_charge_swap(struct page *page, swp_entry_t entry)
 {
 	unsigned int nr_pages = thp_nr_pages(page);
 	struct page_counter *counter;
 	struct mem_cgroup *memcg;
 	unsigned short oldid;
 
-	if (mem_cgroup_disabled())
-		return 0;
-
 	if (!cgroup_subsys_on_dfl(memory_cgrp_subsys))
 		return 0;
 
@@ -7287,18 +7284,15 @@ int mem_cgroup_try_charge_swap(struct page *page, swp_entry_t entry)
 }
 
 /**
- * mem_cgroup_uncharge_swap - uncharge swap space
+ * __mem_cgroup_uncharge_swap - uncharge swap space
  * @entry: swap entry to uncharge
  * @nr_pages: the amount of swap space to uncharge
  */
-void mem_cgroup_uncharge_swap(swp_entry_t entry, unsigned int nr_pages)
+void __mem_cgroup_uncharge_swap(swp_entry_t entry, unsigned int nr_pages)
 {
 	struct mem_cgroup *memcg;
 	unsigned short id;
 
-	if (mem_cgroup_disabled())
-		return;
-
 	id = swap_cgroup_record(entry, 0, nr_pages);
 	rcu_read_lock();
 	memcg = mem_cgroup_from_id(id);
mm/swapfile.c +1 -4
index 707fa0481bb4..04a0c83f1313 100644
@@ -3773,14 +3773,11 @@ static void free_swap_count_continuations(struct swap_info_struct *si)
 }
 
 #if defined(CONFIG_MEMCG) && defined(CONFIG_BLK_CGROUP)
-void cgroup_throttle_swaprate(struct page *page, gfp_t gfp_mask)
+void __cgroup_throttle_swaprate(struct page *page, gfp_t gfp_mask)
 {
 	struct swap_info_struct *si, *next;
 	int nid = page_to_nid(page);
 
-	if (mem_cgroup_disabled())
-		return;
-
 	if (!(gfp_mask & __GFP_IO))
 		return;
 
Signature · 1 line
2.32.0.93.g670b81a890-goog
Signature · 2 lines
Michal Hocko
SUSE Labs
author reviewer
#13 lore raw

Re: [PATCH v3 3/3] mm, memcg: inline swap-related functions to improve disabled memcg config

Quoted 11 lines · Michal Hocko
Quoted 9 lines · Suren Baghdasaryan

Inline mem_cgroup_try_charge_swap, mem_cgroup_uncharge_swap and cgroup_throttle_swaprate functions to perform mem_cgroup_disabled static key check inline before calling the main body of the function. This minimizes the memcg overhead in the pagefault and exit_mmap paths when memcgs are disabled using cgroup_disable=memory command-line option. This change results in ~1% overhead reduction when running PFT test comparing {CONFIG_MEMCG=n} against {CONFIG_MEMCG=y, cgroup_disable=memory} configuration on an 8-core ARM64 Android device.

Quoted 6 lines
Quoted 3 lines

I find it a bit surprising to see such a big difference over a function call in a slow path like swap in/out.

Might be due to the nature of the test. It is designed to generate an avalanche of anonymous pagefaults and that might be the reason swap functions are hit this hard.

Quoted 135 lines

Anyway the change makes sense.

Thanks!

Quoted 123 lines
---
 include/linux/swap.h | 26 +++++++++++++++++++++++---
 mm/memcontrol.c      | 14 ++++----------
 mm/swapfile.c        |  5 +----
 3 files changed, 28 insertions(+), 17 deletions(-)
include/linux/swap.h +23 -3
index 6f5a43251593..f30d26b0f71d 100644
@@ -721,7 +721,13 @@ static inline int mem_cgroup_swappiness(struct mem_cgroup *mem)
 #endif
 
 #if defined(CONFIG_SWAP) && defined(CONFIG_MEMCG) && defined(CONFIG_BLK_CGROUP)
-extern void cgroup_throttle_swaprate(struct page *page, gfp_t gfp_mask);
+extern void __cgroup_throttle_swaprate(struct page *page, gfp_t gfp_mask);
+static inline  void cgroup_throttle_swaprate(struct page *page, gfp_t gfp_mask)
+{
+     if (mem_cgroup_disabled())
+             return;
+     __cgroup_throttle_swaprate(page, gfp_mask);
+}
 #else
 static inline void cgroup_throttle_swaprate(struct page *page, gfp_t gfp_mask)
 {
@@ -730,8 +736,22 @@ static inline void cgroup_throttle_swaprate(struct page *page, gfp_t gfp_mask)
 
 #ifdef CONFIG_MEMCG_SWAP
 extern void mem_cgroup_swapout(struct page *page, swp_entry_t entry);
-extern int mem_cgroup_try_charge_swap(struct page *page, swp_entry_t entry);
-extern void mem_cgroup_uncharge_swap(swp_entry_t entry, unsigned int nr_pages);
+extern int __mem_cgroup_try_charge_swap(struct page *page, swp_entry_t entry);
+static inline int mem_cgroup_try_charge_swap(struct page *page, swp_entry_t entry)
+{
+     if (mem_cgroup_disabled())
+             return 0;
+     return __mem_cgroup_try_charge_swap(page, entry);
+}
+
+extern void __mem_cgroup_uncharge_swap(swp_entry_t entry, unsigned int nr_pages);
+static inline void mem_cgroup_uncharge_swap(swp_entry_t entry, unsigned int nr_pages)
+{
+     if (mem_cgroup_disabled())
+             return;
+     __mem_cgroup_uncharge_swap(entry, nr_pages);
+}
+
 extern long mem_cgroup_get_nr_swap_pages(struct mem_cgroup *memcg);
 extern bool mem_cgroup_swap_full(struct page *page);
 #else
mm/memcontrol.c +4 -10
index cdaf7003b43d..0b05322836ec 100644
@@ -7234,7 +7234,7 @@ void mem_cgroup_swapout(struct page *page, swp_entry_t entry)
 }
 
 /**
- * mem_cgroup_try_charge_swap - try charging swap space for a page
+ * __mem_cgroup_try_charge_swap - try charging swap space for a page
  * @page: page being added to swap
  * @entry: swap entry to charge
  *
@@ -7242,16 +7242,13 @@ void mem_cgroup_swapout(struct page *page, swp_entry_t entry)
  *
  * Returns 0 on success, -ENOMEM on failure.
  */
-int mem_cgroup_try_charge_swap(struct page *page, swp_entry_t entry)
+int __mem_cgroup_try_charge_swap(struct page *page, swp_entry_t entry)
 {
      unsigned int nr_pages = thp_nr_pages(page);
      struct page_counter *counter;
      struct mem_cgroup *memcg;
      unsigned short oldid;
 
-     if (mem_cgroup_disabled())
-             return 0;
-
      if (!cgroup_subsys_on_dfl(memory_cgrp_subsys))
              return 0;
 
@@ -7287,18 +7284,15 @@ int mem_cgroup_try_charge_swap(struct page *page, swp_entry_t entry)
 }
 
 /**
- * mem_cgroup_uncharge_swap - uncharge swap space
+ * __mem_cgroup_uncharge_swap - uncharge swap space
  * @entry: swap entry to uncharge
  * @nr_pages: the amount of swap space to uncharge
  */
-void mem_cgroup_uncharge_swap(swp_entry_t entry, unsigned int nr_pages)
+void __mem_cgroup_uncharge_swap(swp_entry_t entry, unsigned int nr_pages)
 {
      struct mem_cgroup *memcg;
      unsigned short id;
 
-     if (mem_cgroup_disabled())
-             return;
-
      id = swap_cgroup_record(entry, 0, nr_pages);
      rcu_read_lock();
      memcg = mem_cgroup_from_id(id);
mm/swapfile.c +1 -5
index 707fa0481bb4..04a0c83f1313 100644
@@ -3773,14 +3773,11 @@ static void free_swap_count_continuations(struct swap_info_struct *si)
 }
 
 #if defined(CONFIG_MEMCG) && defined(CONFIG_BLK_CGROUP)
-void cgroup_throttle_swaprate(struct page *page, gfp_t gfp_mask)
+void __cgroup_throttle_swaprate(struct page *page, gfp_t gfp_mask)
 {
      struct swap_info_struct *si, *next;
      int nid = page_to_nid(page);
 
-     if (mem_cgroup_disabled())
-             return;
-
      if (!(gfp_mask & __GFP_IO))
              return;
 
--
Quoted 1 line

2.32.0.93.g670b81a890-goog

--
Michal Hocko
SUSE Labs

#14 lore raw
In reply to Suren Baghdasaryan
Quoted 37 lines · Suren Baghdasaryan

Add mem_cgroup_disabled check in vmpressure, mem_cgroup_uncharge_swap and cgroup_throttle_swaprate functions. This minimizes the memcg overhead in the pagefault and exit_mmap paths when memcgs are disabled using cgroup_disable=memory command-line option.
This change results in ~2.1% overhead reduction when running PFT test comparing {CONFIG_MEMCG=n, CONFIG_MEMCG_SWAP=n} against {CONFIG_MEMCG=y, CONFIG_MEMCG_SWAP=y, cgroup_disable=memory} configuration on an 8-core ARM64 Android device.

---
 mm/memcontrol.c | 3 +++
 mm/swapfile.c   | 3 +++
 mm/vmpressure.c | 7 ++++++-
 3 files changed, 12 insertions(+), 1 deletion(-)
mm/memcontrol.c +3 -0
index ae1f5d0cb581..a228cd51c4bd 100644
@@ -7305,6 +7305,9 @@ void mem_cgroup_uncharge_swap(swp_entry_t entry, unsigned int nr_pages)
 	struct mem_cgroup *memcg;
 	unsigned short id;
 
+	if (mem_cgroup_disabled())
+		return;
+
 	id = swap_cgroup_record(entry, 0, nr_pages);
 	rcu_read_lock();
 	memcg = mem_cgroup_from_id(id);
mm/swapfile.c +0 -0
index 1e07d1c776f2..707fa0481bb4 100644

mm/swapfile.c void cgroup_throttle_swaprate(struct page *page, gfp_t gfp_mask) · line 3783

Quoted 6 lines
mm/swapfile.c +3 -0
@@ -3778,6 +3778,9 @@ void cgroup_throttle_swaprate(struct page *page, gfp_t gfp_mask)
 	struct swap_info_struct *si, *next;
 	int nid = page_to_nid(page);
 
+	if (mem_cgroup_disabled())
+		return;
+

Many thanks for your patch. But I'am somewhat confused about this change. IMO, cgroup_throttle_swaprate() is only related to blk_cgroup and it seems it's irrelevant to mem_cgroup. Could you please have a explanation for me?

Thanks!

Quoted 22 lines
mm/swapfile.c +0 -0
 	if (!(gfp_mask & __GFP_IO))
 		return;
 
mm/vmpressure.c +6 -1
index d69019fc3789..9b172561fded 100644
@@ -240,7 +240,12 @@ static void vmpressure_work_fn(struct work_struct *work)
 void vmpressure(gfp_t gfp, struct mem_cgroup *memcg, bool tree,
 		unsigned long scanned, unsigned long reclaimed)
 {
-	struct vmpressure *vmpr = memcg_to_vmpressure(memcg);
+	struct vmpressure *vmpr;
+
+	if (mem_cgroup_disabled())
+		return;
+
+	vmpr = memcg_to_vmpressure(memcg);
 
 	/*
 	 * Here we only want to account pressure that userland is able to
author reviewer
#15 lore raw
Quoted 42 lines · Miaohe Lin
Quoted 40 lines · Suren Baghdasaryan

Add mem_cgroup_disabled check in vmpressure, mem_cgroup_uncharge_swap and cgroup_throttle_swaprate functions. This minimizes the memcg overhead in the pagefault and exit_mmap paths when memcgs are disabled using cgroup_disable=memory command-line option.
This change results in ~2.1% overhead reduction when running PFT test comparing {CONFIG_MEMCG=n, CONFIG_MEMCG_SWAP=n} against {CONFIG_MEMCG=y, CONFIG_MEMCG_SWAP=y, cgroup_disable=memory} configuration on an 8-core ARM64 Android device.

---
 mm/memcontrol.c | 3 +++
 mm/swapfile.c   | 3 +++
 mm/vmpressure.c | 7 ++++++-
 3 files changed, 12 insertions(+), 1 deletion(-)
mm/memcontrol.c +3 -0
index ae1f5d0cb581..a228cd51c4bd 100644
@@ -7305,6 +7305,9 @@ void mem_cgroup_uncharge_swap(swp_entry_t entry, unsigned int nr_pages)
      struct mem_cgroup *memcg;
      unsigned short id;
 
+     if (mem_cgroup_disabled())
+             return;
+
      id = swap_cgroup_record(entry, 0, nr_pages);
      rcu_read_lock();
      memcg = mem_cgroup_from_id(id);
mm/swapfile.c +0 -0
index 1e07d1c776f2..707fa0481bb4 100644
@@ -3778,6 +3778,9 @@ void cgroup_throttle_swaprate(struct page *page, gfp_t gfp_mask)
      struct swap_info_struct *si, *next;
      int nid = page_to_nid(page);
 
Quoted 7 lines

mm/swapfile.c void cgroup_throttle_swaprate(struct page *page, gfp_t gfp_mask) · line 3783

Quoted 3 lines
mm/swapfile.c +3 -0
+     if (mem_cgroup_disabled())
+             return;
+

Many thanks for your patch. But I'am somewhat confused about this change. IMO, cgroup_throttle_swaprate() is only related to blk_cgroup and it seems it's irrelevant to mem_cgroup. Could you please have a explanation for me?

cgroup_throttle_swaprate() is a NoOp when CONFIG_MEMCG=n (see:
https://elixir.bootlin.com/linux/latest/source/include/linux/swap.h#L699),
therefore I assume we can safely skip it when memcgs are disabled via "cgroup_disable=memory". From perf results I also see no hits on this function when CONFIG_MEMCG=n.
However, looking into the code, I'm not sure why it should depend on CONFIG_MEMCG. But it's Friday night and I might be missing some details here...

Quoted 26 lines

Thanks!

Quoted 22 lines
mm/swapfile.c +0 -0
      if (!(gfp_mask & __GFP_IO))
              return;
 
mm/vmpressure.c +6 -1
index d69019fc3789..9b172561fded 100644
@@ -240,7 +240,12 @@ static void vmpressure_work_fn(struct work_struct *work)
 void vmpressure(gfp_t gfp, struct mem_cgroup *memcg, bool tree,
              unsigned long scanned, unsigned long reclaimed)
 {
-     struct vmpressure *vmpr = memcg_to_vmpressure(memcg);
+     struct vmpressure *vmpr;
+
+     if (mem_cgroup_disabled())
+             return;
+
+     vmpr = memcg_to_vmpressure(memcg);
 
      /*
       * Here we only want to account pressure that userland is able to
#16 lore raw
Quoted 51 lines · Suren Baghdasaryan
Quoted 42 lines · Miaohe Lin
Quoted 40 lines · Suren Baghdasaryan

Add mem_cgroup_disabled check in vmpressure, mem_cgroup_uncharge_swap and cgroup_throttle_swaprate functions. This minimizes the memcg overhead in the pagefault and exit_mmap paths when memcgs are disabled using cgroup_disable=memory command-line option.
This change results in ~2.1% overhead reduction when running PFT test comparing {CONFIG_MEMCG=n, CONFIG_MEMCG_SWAP=n} against {CONFIG_MEMCG=y, CONFIG_MEMCG_SWAP=y, cgroup_disable=memory} configuration on an 8-core ARM64 Android device.

---
 mm/memcontrol.c | 3 +++
 mm/swapfile.c   | 3 +++
 mm/vmpressure.c | 7 ++++++-
 3 files changed, 12 insertions(+), 1 deletion(-)
mm/memcontrol.c +3 -0
index ae1f5d0cb581..a228cd51c4bd 100644
@@ -7305,6 +7305,9 @@ void mem_cgroup_uncharge_swap(swp_entry_t entry, unsigned int nr_pages)
      struct mem_cgroup *memcg;
      unsigned short id;
 
+     if (mem_cgroup_disabled())
+             return;
+
      id = swap_cgroup_record(entry, 0, nr_pages);
      rcu_read_lock();
      memcg = mem_cgroup_from_id(id);
mm/swapfile.c +0 -0
index 1e07d1c776f2..707fa0481bb4 100644
@@ -3778,6 +3778,9 @@ void cgroup_throttle_swaprate(struct page *page, gfp_t gfp_mask)
      struct swap_info_struct *si, *next;
      int nid = page_to_nid(page);
 
Quoted 7 lines

mm/swapfile.c void cgroup_throttle_swaprate(struct page *page, gfp_t gfp_mask) · line 3783

Quoted 3 lines
mm/swapfile.c +3 -0
+     if (mem_cgroup_disabled())
+             return;
+

Many thanks for your patch. But I'am somewhat confused about this change. IMO, cgroup_throttle_swaprate() is only related to blk_cgroup and it seems it's irrelevant to mem_cgroup. Could you please have a explanation for me?

Quoted 2 lines

cgroup_throttle_swaprate() is a NoOp when CONFIG_MEMCG=n (see:
https://elixir.bootlin.com/linux/latest/source/include/linux/swap.h#L699),

I browsed the git history related to cgroup_throttle_swaprate() and found this:

"""
mm: memcontrol: move out cgroup swaprate throttling

The cgroup swaprate throttling is about matching new anon allocations to the rate of available IO when that is being throttled. It's the io controller hooking into the VM, rather than a memory controller thing.
"""

It seems cgroup_throttle_swaprate() is working with memory allocations. So mem_cgroup matters this way. But I'am not sure...

Quoted 6 lines

therefore I assume we can safely skip it when memcgs are disabled via "cgroup_disable=memory". From perf results I also see no hits on this function when CONFIG_MEMCG=n.
However, looking into the code, I'm not sure why it should depend on CONFIG_MEMCG. But it's Friday night and I might be missing some details here...

Many thanks for your replay at Friday night. :)

Quoted 29 lines
Quoted 23 lines

Thanks!

Quoted 19 lines
mm/swapfile.c +0 -0
      if (!(gfp_mask & __GFP_IO))
              return;
 
mm/vmpressure.c +6 -1
index d69019fc3789..9b172561fded 100644
@@ -240,7 +240,12 @@ static void vmpressure_work_fn(struct work_struct *work)
 void vmpressure(gfp_t gfp, struct mem_cgroup *memcg, bool tree,
              unsigned long scanned, unsigned long reclaimed)
 {
-     struct vmpressure *vmpr = memcg_to_vmpressure(memcg);
+     struct vmpressure *vmpr;
+
+     if (mem_cgroup_disabled())
+             return;
+
+     vmpr = memcg_to_vmpressure(memcg);
 
Quoted 4 lines

mm/vmpressure.c static void vmpressure_work_fn(struct work_struct *work) · line 251

Quoted 3 lines
mm/vmpressure.c +0 -0
      /*
       * Here we only want to account pressure that userland is able to

.

Reviewed-by on v3 1/3
#17 lore raw
In reply to Suren Baghdasaryan
Quoted 10 lines · Suren Baghdasaryan

Add mem_cgroup_disabled check in vmpressure, mem_cgroup_uncharge_swap and cgroup_throttle_swaprate functions. This minimizes the memcg overhead in the pagefault and exit_mmap paths when memcgs are disabled using cgroup_disable=memory command-line option.
This change results in ~2.1% overhead reduction when running PFT test comparing {CONFIG_MEMCG=n, CONFIG_MEMCG_SWAP=n} against {CONFIG_MEMCG=y, CONFIG_MEMCG_SWAP=y, cgroup_disable=memory} configuration on an 8-core ARM64 Android device.

Quoted 3 lines

The changes are straightforward. LGTM.

reviewer Acked-by on v3 1/3
#18 lore raw
In reply to Suren Baghdasaryan
Quoted 5 lines · Suren Baghdasaryan

Add mem_cgroup_disabled check in vmpressure, mem_cgroup_uncharge_swap and cgroup_throttle_swaprate functions. This minimizes the memcg overhead in the pagefault and exit_mmap paths when memcgs are disabled using cgroup_disable=memory command-line option.
This change results in ~2.1% overhead reduction when running PFT test

What is PFT test?

Quoted 7 lines

comparing {CONFIG_MEMCG=n, CONFIG_MEMCG_SWAP=n} against {CONFIG_MEMCG=y, CONFIG_MEMCG_SWAP=y, cgroup_disable=memory} configuration on an 8-core ARM64 Android device.

Thanks!

Quoted 54 lines
---
 mm/memcontrol.c | 3 +++
 mm/swapfile.c   | 3 +++
 mm/vmpressure.c | 7 ++++++-
 3 files changed, 12 insertions(+), 1 deletion(-)
mm/memcontrol.c +3 -0
index ae1f5d0cb581..a228cd51c4bd 100644
@@ -7305,6 +7305,9 @@ void mem_cgroup_uncharge_swap(swp_entry_t entry, unsigned int nr_pages)
 	struct mem_cgroup *memcg;
 	unsigned short id;
 
+	if (mem_cgroup_disabled())
+		return;
+
 	id = swap_cgroup_record(entry, 0, nr_pages);
 	rcu_read_lock();
 	memcg = mem_cgroup_from_id(id);
mm/swapfile.c +3 -0
index 1e07d1c776f2..707fa0481bb4 100644
@@ -3778,6 +3778,9 @@ void cgroup_throttle_swaprate(struct page *page, gfp_t gfp_mask)
 	struct swap_info_struct *si, *next;
 	int nid = page_to_nid(page);
 
+	if (mem_cgroup_disabled())
+		return;
+
 	if (!(gfp_mask & __GFP_IO))
 		return;
 
mm/vmpressure.c +6 -1
index d69019fc3789..9b172561fded 100644
@@ -240,7 +240,12 @@ static void vmpressure_work_fn(struct work_struct *work)
 void vmpressure(gfp_t gfp, struct mem_cgroup *memcg, bool tree,
 		unsigned long scanned, unsigned long reclaimed)
 {
-	struct vmpressure *vmpr = memcg_to_vmpressure(memcg);
+	struct vmpressure *vmpr;
+
+	if (mem_cgroup_disabled())
+		return;
+
+	vmpr = memcg_to_vmpressure(memcg);
 
 	/*
 	 * Here we only want to account pressure that userland is able to
Signature · 1 line
2.32.0.93.g670b81a890-goog
Signature · 2 lines
Michal Hocko
SUSE Labs
author reviewer
#19 lore raw
Quoted 9 lines · Michal Hocko
Quoted 5 lines · Suren Baghdasaryan

Add mem_cgroup_disabled check in vmpressure, mem_cgroup_uncharge_swap and cgroup_throttle_swaprate functions. This minimizes the memcg overhead in the pagefault and exit_mmap paths when memcgs are disabled using cgroup_disable=memory command-line option.
This change results in ~2.1% overhead reduction when running PFT test

What is PFT test?

Christoph Lamenter’s pagefault tool
(https://lkml.org/lkml/2006/8/29/294). I'll add the link in the description for clarity.

Quoted 10 lines
Quoted 8 lines

comparing {CONFIG_MEMCG=n, CONFIG_MEMCG_SWAP=n} against {CONFIG_MEMCG=y, CONFIG_MEMCG_SWAP=y, cgroup_disable=memory} configuration on an 8-core ARM64 Android device.

Thanks!

Quoted 61 lines

Thanks!

Quoted 53 lines
---
 mm/memcontrol.c | 3 +++
 mm/swapfile.c   | 3 +++
 mm/vmpressure.c | 7 ++++++-
 3 files changed, 12 insertions(+), 1 deletion(-)
mm/memcontrol.c +3 -0
index ae1f5d0cb581..a228cd51c4bd 100644
@@ -7305,6 +7305,9 @@ void mem_cgroup_uncharge_swap(swp_entry_t entry, unsigned int nr_pages)
      struct mem_cgroup *memcg;
      unsigned short id;
 
+     if (mem_cgroup_disabled())
+             return;
+
      id = swap_cgroup_record(entry, 0, nr_pages);
      rcu_read_lock();
      memcg = mem_cgroup_from_id(id);
mm/swapfile.c +3 -0
index 1e07d1c776f2..707fa0481bb4 100644
@@ -3778,6 +3778,9 @@ void cgroup_throttle_swaprate(struct page *page, gfp_t gfp_mask)
      struct swap_info_struct *si, *next;
      int nid = page_to_nid(page);
 
+     if (mem_cgroup_disabled())
+             return;
+
      if (!(gfp_mask & __GFP_IO))
              return;
 
mm/vmpressure.c +6 -2
index d69019fc3789..9b172561fded 100644
@@ -240,7 +240,12 @@ static void vmpressure_work_fn(struct work_struct *work)
 void vmpressure(gfp_t gfp, struct mem_cgroup *memcg, bool tree,
              unsigned long scanned, unsigned long reclaimed)
 {
-     struct vmpressure *vmpr = memcg_to_vmpressure(memcg);
+     struct vmpressure *vmpr;
+
+     if (mem_cgroup_disabled())
+             return;
+
+     vmpr = memcg_to_vmpressure(memcg);
 
      /*
       * Here we only want to account pressure that userland is able to
--
Quoted 1 line

2.32.0.93.g670b81a890-goog

--
Michal Hocko
SUSE Labs