Cgroup accounting of memory tier usage

If we need to restrict toptier memory usage for a cgroup, we need to retrieve usage of toptier memory efficiently. Add a page counter to track toptier memory usage directly so its value can be returned right away.

---
 include/linux/memcontrol.h |  1 +
 mm/memcontrol.c            | 50 ++++++++++++++++++++++++++++++++------
 2 files changed, 43 insertions(+), 8 deletions(-)

include/linux/memcontrol.h +1 -0

index 9ecead1042b9..b4f727cba1de 100644
@@ -241,6 +241,7 @@ struct mem_cgroup {
 
        /* Accounted resources */
        struct page_counter memory;             /* Both v1 & v2 */
+       struct page_counter toptier;
 
        union {
                struct page_counter swap;       /* v2 only */

mm/memcontrol.c +8 -0

index 2f6e95e6d200..2f20ec2712b8 100644
@@ -848,6 +848,23 @@ static void mem_cgroup_charge_statistics(struct
 mem_cgroup *memcg,
        __this_cpu_add(memcg->vmstats_percpu->nr_page_events, nr_pages);
 }
 
+static inline void mem_cgroup_charge_toptier(struct mem_cgroup *memcg,
+                                            int nid,
+                                            int nr_pages)
+{
+       if (!node_is_toptier(nid) || !memcg)
+               return;
+
+       if (nr_pages >= 0) {

mm/memcontrol.c +0 -0

                                       enum mem_cgroup_events_target

target)
{

mm/memcontrol.c +2 -0

@@ -3027,6 +3044,8 @@ int __memcg_kmem_charge_page(struct page *page,
 gfp_t gfp, int order)
                if (!ret) {
                        page->memcg_data = (unsigned long)objcg |
                                MEMCG_DATA_KMEM;
+                       mem_cgroup_charge_toptier(page_memcg(page),
+                                       page_to_nid(page), 1 << order);
                        return 0;
                }

                obj_cgroup_put(objcg);

mm/memcontrol.c +2 -0

@@ -3050,6 +3069,8 @@ void __memcg_kmem_uncharge_page(struct page *page,
 int order)
 
        objcg = __folio_objcg(folio);
        obj_cgroup_uncharge_pages(objcg, nr_pages);
+       mem_cgroup_charge_toptier(page_memcg(page),
+                       page_to_nid(page), -nr_pages);
        folio->memcg_data = 0;
        obj_cgroup_put(objcg);

}

mm/memcontrol.c +3 -6

@@ -3947,13 +3968,10 @@ unsigned long mem_cgroup_memtier_usage(struct
 mem_cgroup *memcg,
 
 unsigned long mem_cgroup_toptier_usage(struct mem_cgroup *memcg)
 {
-       struct memory_tier *top_tier;
-
-       top_tier = list_first_entry(&memory_tiers, struct memory_tier,
 list);
-       if (top_tier)
-               return mem_cgroup_memtier_usage(memcg, top_tier);
-       else
+       if (!memcg)
                return 0;
+
+       return page_counter_read(&memcg->toptier);
 }

#endif /* CONFIG_NUMA */

mm/memcontrol.c +2 -0

@@ -5228,11 +5246,13 @@ mem_cgroup_css_alloc(struct cgroup_subsys_state
 *parent_css)
                memcg->oom_kill_disable = parent->oom_kill_disable;
 
                page_counter_init(&memcg->memory, &parent->memory);
+               page_counter_init(&memcg->toptier, &parent->toptier);
                page_counter_init(&memcg->swap, &parent->swap);
                page_counter_init(&memcg->kmem, &parent->kmem);
                page_counter_init(&memcg->tcpmem, &parent->tcpmem);
        } else {
                page_counter_init(&memcg->memory, NULL);
+               page_counter_init(&memcg->toptier, NULL);
                page_counter_init(&memcg->swap, NULL);
                page_counter_init(&memcg->kmem, NULL);

                page_counter_init(&memcg->tcpmem, NULL);

mm/memcontrol.c +3 -0

@@ -5678,6 +5698,8 @@ static int mem_cgroup_move_account(struct page *page,
        memcg_check_events(to, nid);
        mem_cgroup_charge_statistics(from, -nr_pages);
        memcg_check_events(from, nid);
+       mem_cgroup_charge_toptier(to, nid, nr_pages);
+       mem_cgroup_charge_toptier(from, nid, -nr_pages);
        local_irq_enable();
 out_unlock:
        folio_unlock(folio);

@@ -6761,6 +6783,7 @@ static int charge_memcg(struct folio *folio, struct
 mem_cgroup *memcg,
 
        local_irq_disable();
        mem_cgroup_charge_statistics(memcg, nr_pages);
+       mem_cgroup_charge_toptier(memcg, folio_nid(folio), nr_pages);
        memcg_check_events(memcg, folio_nid(folio));
        local_irq_enable();

out:

mm/memcontrol.c +2 -0

@@ -6853,6 +6876,7 @@ struct uncharge_gather {
        unsigned long nr_memory;
        unsigned long pgpgout;
        unsigned long nr_kmem;
+       unsigned long nr_toptier;
        int nid;
 };
 

@@ -6867,6 +6891,7 @@ static void uncharge_batch(const struct
 uncharge_gather *ug)
 
        if (ug->nr_memory) {
                page_counter_uncharge(&ug->memcg->memory, ug->nr_memory);
+               page_counter_uncharge(&ug->memcg->toptier, ug->nr_toptier);
                if (do_memsw_account())
                        page_counter_uncharge(&ug->memcg->memsw,

ug->nr_memory);
                if (ug->nr_kmem)

mm/memcontrol.c +7 -1

@@ -6929,12 +6954,18 @@ static void uncharge_folio(struct folio *folio,
 struct uncharge_gather *ug)
                ug->nr_memory += nr_pages;
                ug->nr_kmem += nr_pages;
 
+               if (node_is_toptier(folio_nid(folio)))
+                       ug->nr_toptier += nr_pages;
+
                folio->memcg_data = 0;
                obj_cgroup_put(objcg);
        } else {
                /* LRU pages aren't accounted at the root level */
-               if (!mem_cgroup_is_root(memcg))
+               if (!mem_cgroup_is_root(memcg)) {
                        ug->nr_memory += nr_pages;
+                       if (node_is_toptier(folio_nid(folio)))
+                               ug->nr_toptier += nr_pages;
+               }
                ug->pgpgout++;
 

                folio->memcg_data = 0;

mm/memcontrol.c +1 -0

@@ -7011,6 +7042,7 @@ void mem_cgroup_migrate(struct folio *old, struct
 folio *new)
        /* Force-charge the new page. The old one will be freed soon */
        if (!mem_cgroup_is_root(memcg)) {
                page_counter_charge(&memcg->memory, nr_pages);
+               mem_cgroup_charge_toptier(memcg, folio_nid(new), nr_pages);
                if (do_memsw_account())
                        page_counter_charge(&memcg->memsw, nr_pages);

        }

mm/memcontrol.c +3 -3

@@ -7231,8 +7263,10 @@ void mem_cgroup_swapout(struct folio *folio,
 swp_entry_t entry)
 
        folio->memcg_data = 0;
 
-       if (!mem_cgroup_is_root(memcg))
+       if (!mem_cgroup_is_root(memcg)) {
                page_counter_uncharge(&memcg->memory, nr_entries);
+               mem_cgroup_charge_toptier(memcg, folio_nid(folio),
-nr_entries);
+       }
 
        if (!cgroup_memory_noswap && memcg != swap_memcg) {
                if (!mem_cgroup_is_root(swap_memcg))
--

2.35.1

If we need to restrict toptier memory usage for a cgroup, we need to retrieve usage of toptier memory efficiently. Add a page counter to track toptier memory usage directly so its value can be returned right away.

---
 include/linux/memcontrol.h |  1 +
 mm/memcontrol.c            | 50 ++++++++++++++++++++++++++++++++------
 2 files changed, 43 insertions(+), 8 deletions(-)

include/linux/memcontrol.h +1 -0

index 9ecead1042b9..b4f727cba1de 100644
@@ -241,6 +241,7 @@ struct mem_cgroup {
 
        /* Accounted resources */
        struct page_counter memory;             /* Both v1 & v2 */
+       struct page_counter toptier;
 
        union {
                struct page_counter swap;       /* v2 only */

mm/memcontrol.c +6 -0

index 2f6e95e6d200..2f20ec2712b8 100644
@@ -848,6 +848,23 @@ static void mem_cgroup_charge_statistics(struct mem_cgroup *memcg,
        __this_cpu_add(memcg->vmstats_percpu->nr_page_events, nr_pages);
 }
 
+static inline void mem_cgroup_charge_toptier(struct mem_cgroup *memcg,
+                                            int nid,
+                                            int nr_pages)
+{
+       if (!node_is_toptier(nid) || !memcg)
+               return;

mm/memcontrol.c +25 -9

 static bool mem_cgroup_event_ratelimit(struct mem_cgroup *memcg,
                                       enum mem_cgroup_events_target target)
 {

@@ -3027,6 +3044,8 @@ int __memcg_kmem_charge_page(struct page *page, gfp_t gfp, int order)
                if (!ret) {
                        page->memcg_data = (unsigned long)objcg |
                                MEMCG_DATA_KMEM;
+                       mem_cgroup_charge_toptier(page_memcg(page),
+                                       page_to_nid(page), 1 << order);
                        return 0;
                }
                obj_cgroup_put(objcg);

@@ -3050,6 +3069,8 @@ void __memcg_kmem_uncharge_page(struct page *page, int order)
 
        objcg = __folio_objcg(folio);
        obj_cgroup_uncharge_pages(objcg, nr_pages);
+       mem_cgroup_charge_toptier(page_memcg(page),
+                       page_to_nid(page), -nr_pages);
        folio->memcg_data = 0;
        obj_cgroup_put(objcg);
 }

@@ -3947,13 +3968,10 @@ unsigned long mem_cgroup_memtier_usage(struct mem_cgroup *memcg,
 
 unsigned long mem_cgroup_toptier_usage(struct mem_cgroup *memcg)
 {
-       struct memory_tier *top_tier;
-
-       top_tier = list_first_entry(&memory_tiers, struct memory_tier, list);
-       if (top_tier)
-               return mem_cgroup_memtier_usage(memcg, top_tier);
-       else
+       if (!memcg)
                return 0;
+
+       return page_counter_read(&memcg->toptier);
 }
 
 #endif /* CONFIG_NUMA */

@@ -5228,11 +5246,13 @@ mem_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
                memcg->oom_kill_disable = parent->oom_kill_disable;
 
                page_counter_init(&memcg->memory, &parent->memory);
+               page_counter_init(&memcg->toptier, &parent->toptier);
                page_counter_init(&memcg->swap, &parent->swap);
                page_counter_init(&memcg->kmem, &parent->kmem);
                page_counter_init(&memcg->tcpmem, &parent->tcpmem);
        } else {
                page_counter_init(&memcg->memory, NULL);
+               page_counter_init(&memcg->toptier, NULL);
                page_counter_init(&memcg->swap, NULL);
                page_counter_init(&memcg->kmem, NULL);
                page_counter_init(&memcg->tcpmem, NULL);

@@ -5678,6 +5698,8 @@ static int mem_cgroup_move_account(struct page *page,
        memcg_check_events(to, nid);
        mem_cgroup_charge_statistics(from, -nr_pages);
        memcg_check_events(from, nid);
+       mem_cgroup_charge_toptier(to, nid, nr_pages);
+       mem_cgroup_charge_toptier(from, nid, -nr_pages);
        local_irq_enable();
 out_unlock:
        folio_unlock(folio);

@@ -6761,6 +6783,7 @@ static int charge_memcg(struct folio *folio, struct mem_cgroup *memcg,
 
        local_irq_disable();
        mem_cgroup_charge_statistics(memcg, nr_pages);
+       mem_cgroup_charge_toptier(memcg, folio_nid(folio), nr_pages);
        memcg_check_events(memcg, folio_nid(folio));
        local_irq_enable();
 out:

@@ -6853,6 +6876,7 @@ struct uncharge_gather {
        unsigned long nr_memory;
        unsigned long pgpgout;
        unsigned long nr_kmem;
+       unsigned long nr_toptier;
        int nid;
 };
 

@@ -6867,6 +6891,7 @@ static void uncharge_batch(const struct uncharge_gather *ug)
 
        if (ug->nr_memory) {
                page_counter_uncharge(&ug->memcg->memory, ug->nr_memory);
+               page_counter_uncharge(&ug->memcg->toptier, ug->nr_toptier);
                if (do_memsw_account())
                        page_counter_uncharge(&ug->memcg->memsw, ug->nr_memory);
                if (ug->nr_kmem)

@@ -6929,12 +6954,18 @@ static void uncharge_folio(struct folio *folio, struct uncharge_gather *ug)
                ug->nr_memory += nr_pages;
                ug->nr_kmem += nr_pages;
 
+               if (node_is_toptier(folio_nid(folio)))
+                       ug->nr_toptier += nr_pages;
+
                folio->memcg_data = 0;
                obj_cgroup_put(objcg);
        } else {
                /* LRU pages aren't accounted at the root level */
-               if (!mem_cgroup_is_root(memcg))
+               if (!mem_cgroup_is_root(memcg)) {
                        ug->nr_memory += nr_pages;
+                       if (node_is_toptier(folio_nid(folio)))
+                               ug->nr_toptier += nr_pages;
+               }
                ug->pgpgout++;
 
                folio->memcg_data = 0;

@@ -7011,6 +7042,7 @@ void mem_cgroup_migrate(struct folio *old, struct folio *new)
        /* Force-charge the new page. The old one will be freed soon */
        if (!mem_cgroup_is_root(memcg)) {
                page_counter_charge(&memcg->memory, nr_pages);
+               mem_cgroup_charge_toptier(memcg, folio_nid(new), nr_pages);
                if (do_memsw_account())
                        page_counter_charge(&memcg->memsw, nr_pages);
        }

@@ -7231,8 +7263,10 @@ void mem_cgroup_swapout(struct folio *folio, swp_entry_t entry)
 
        folio->memcg_data = 0;
 
-       if (!mem_cgroup_is_root(memcg))
+       if (!mem_cgroup_is_root(memcg)) {
                page_counter_uncharge(&memcg->memory, nr_entries);
+               mem_cgroup_charge_toptier(memcg, folio_nid(folio), -nr_entries);
+       }
 
        if (!cgroup_memory_noswap && memcg != swap_memcg) {
                if (!mem_cgroup_is_root(swap_memcg))
--

2.35.1

For controlling usage of a top tiered memory by a cgroup, accounting of top tier memory usage is needed. This patch set implements the following:

Patch 1 introduces interface and simple implementation to retrieve
cgroup tiered memory usage
Patch 2 introduces more efficient accounting with top tier memory page counter Patch 3 provides a sysfs interface to repot the the top tiered memory
usage.

The patchset works with Aneesh's v6 memory-tiering implementation [1]. It is a preparatory patch set before introducing features to control top tiered memory in cgroups.

I'll like to first get feedback to see if
(1) Controllng the topmost tiered memory is enough
or
(2) Multiple tiers at the top levels need to be grouped into "toptier" or

(3) There are use cases not covered by (1) and (2).

Thanks.

Tim

[1] https://lore.kernel.org/linux-mm/[email protected]/

Tim Chen (3):
  mm/memory-tiers Add functions for tier memory usage in a cgroup mm/memory-tiers: Use page counter to track toptier memory usage mm/memory-tiers: Show toptier memory usage for cgroup

 include/linux/memcontrol.h   |  1 +
 include/linux/memory-tiers.h |  2 +
 mm/memcontrol.c              | 86 +++++++++++++++++++++++++++++++++++-
 mm/memory-tiers.c            |  3 +-
 4 files changed, 89 insertions(+), 3 deletions(-)

For controlling usage of a top tiered memory by a cgroup, accounting of top tier memory usage is needed. This patch set implements the following:

Patch 1 introduces interface and simple implementation to retrieve
cgroup tiered memory usage
Patch 2 introduces more efficient accounting with top tier memory page counter Patch 3 provides a sysfs interface to repot the the top tiered memory
usage.

The patchset works with Aneesh's v6 memory-tiering implementation [1]. It is a preparatory patch set before introducing features to control top tiered memory in cgroups.

I'll like to first get feedback to see if
(1) Controllng the topmost tiered memory is enough
or
(2) Multiple tiers at the top levels need to be grouped into "toptier" or
(3) There are use cases not covered by (1) and (2).

Quoted 4 lines · Tim Chen

For controlling usage of a top tiered memory by a cgroup, accounting of top tier memory usage is needed. This patch set implements the following:

Quoted 5 lines

Patch 1 introduces interface and simple implementation to retrieve
cgroup tiered memory usage
Patch 2 introduces more efficient accounting with top tier memory page counter Patch 3 provides a sysfs interface to repot the the top tiered memory
usage.

I guess you meant cgroupfs here, right?

Quoted 11 lines

The patchset works with Aneesh's v6 memory-tiering implementation [1]. It is a preparatory patch set before introducing features to control top tiered memory in cgroups.

I'll like to first get feedback to see if
(1) Controllng the topmost tiered memory is enough
or
(2) Multiple tiers at the top levels need to be grouped into "toptier" or
(3) There are use cases not covered by (1) and (2).

I would start by asking why do we need a dedicated interface in the first place. Why the existing numa_stat is not a proper interface. Right now we only report LRU per node stats. Is this insufficient? What is userspace expect to do based on the reported data?

On Wed, 2022-06-15 at 13:11 +0200, Michal Hocko wrote:

Quoted 11 lines

Quoted 10 lines

The patchset works with Aneesh's v6 memory-tiering implementation [1]. It is a preparatory patch set before introducing features to control top tiered memory in cgroups.

I'll like to first get feedback to see if
(1) Controllng the topmost tiered memory is enough
or
(2) Multiple tiers at the top levels need to be grouped into "toptier" or
(3) There are use cases not covered by (1) and (2).

Quoted 4 lines

I would start by asking why do we need a dedicated interface in the first place. Why the existing numa_stat is not a proper interface. Right now we only report LRU per node stats. Is this insufficient? What is userspace expect to do based on the reported data?

Exporting the toptier information here is convenient for me for debugging purpose of seeing whether a cgroup's toptier usage is under control. Otherwise writing a script to parse numastat and the memtier heirachy will work too. Exporting toptier usage directly is optional and we don't have to do it.

Quoted 14 lines · Tim Chen

For controlling usage of a top tiered memory by a cgroup, accounting of top tier memory usage is needed. This patch set implements the following:

Patch 1 introduces interface and simple implementation to retrieve
cgroup tiered memory usage
Patch 2 introduces more efficient accounting with top tier memory page counter Patch 3 provides a sysfs interface to repot the the top tiered memory
usage.

The patchset works with Aneesh's v6 memory-tiering implementation [1]. It is a preparatory patch set before introducing features to control top tiered memory in cgroups.

Quoted 5 lines

I'll like to first get feedback to see if
(1) Controllng the topmost tiered memory is enough
or
(2) Multiple tiers at the top levels need to be grouped into "toptier" or

If we combine top-N tiers, I think the better name could be "fast-tier", in contrast to "slow-tier".

Quoted 1 line

(3) There are use cases not covered by (1) and (2).

Is it necessary to control memory usage of each tier (except the lowest/slowest)? I am not the right person to answer the question, but I want to ask it.

When we don't know which pages are being charged, we should still charge the usage to toptier (assuming that toptier always include the default tier), e.g. from try_charge_memcg().

The idea is that when lower tier memory is not used, memcg->toptier and memcg->memory should have the same value. Otherwise, it can cause confusions about where the pages of (memcg->memory - memcg->toptier) go.

Tim Chen <[email protected]> writes:

+unsigned long mem_cgroup_toptier_usage(struct mem_cgroup *memcg)

mm/memcontrol.c static int memcg_numa_stat_show(struct seq_file *m, void *v) · line 3957

Quoted 9 lines

mm/memcontrol.c +9 -0

+{
+	struct memory_tier *top_tier;
+
+	top_tier = list_first_entry(&memory_tiers, struct memory_tier, list);
+	if (top_tier)
+		return mem_cgroup_memtier_usage(memcg, top_tier);
+	else
+		return 0;
+}

As discussed here, we would want to consider all memory tiers that got compute as top tier.

https://lore.kernel.org/linux-mm/[email protected]

V6 patchset actually walk the full memory tier hierarchy reverse and consider any memory tier with higher or equal rank value than the first memory tier with CPU as top tier.

https://lore.kernel.org/linux-mm/[email protected]