mm: memcontrol: track LRU counts in the vmstats array
Subsystems (5)
A message that is not in the archive (sent off-list or not yet received): [email protected]
Johannes Weiner
maintainerPATCH 1/6 mm: memcontrol: track LRU counts in the vmstats array
The memcg code currently maintains private per-zone breakdowns of the LRU counters. This is necessary for reclaim decisions which are still zone-based, but there are a variety of users of these counters that only want the aggregate per-lruvec or per-memcg LRU counts, and they need to painfully sum up the zone counters on each request for that.
These would be better served using the memcg vmstats arrays, which track VM statistics at the desired scope already. They just don't have the LRU counts right now.
So to kick off the conversion, begin tracking LRU counts in those.
--- include/linux/mm_inline.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
include/linux/mm_inline.h +1 -1
@@ -29,7 +29,7 @@ static __always_inline void __update_lru_size(struct lruvec *lruvec, {struct pglist_data *pgdat =lruvec_pgdat (lruvec); -__mod_node_page_state (pgdat,NR_LRU_BASE + lru, nr_pages); +__mod_lruvec_state (lruvec,NR_LRU_BASE + lru, nr_pages);__mod_zone_page_state (&pgdat->node_zones[zid],NR_ZONE_LRU_BASE + lru, nr_pages); }
Signature · 1 line
2.20.1
Johannes Weiner
maintainerPATCH 2/6 mm: memcontrol: replace zone summing with lruvec_page_state()
In reply to a message not in the archiveInstead of adding up the zone counters, use lruvec_page_state() to get the node state directly. This is a bit cheaper and more stream-lined.
--- include/linux/memcontrol.h | 18 ------------------ mm/memcontrol.c | 2 +- mm/vmscan.c | 2 +- 3 files changed, 2 insertions(+), 20 deletions(-)
include/linux/memcontrol.h +0 -18
@@ -517,19 +517,6 @@ void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru,unsigned long mem_cgroup_node_nr_lru_pages (struct mem_cgroup *memcg,int nid,unsigned int lru_mask); -static inline -unsigned long mem_cgroup_get_lru_size (struct lruvec *lruvec,enum lru_list lru) -{ -struct mem_cgroup_per_node *mz; -unsigned long nr_pages =0 ; -int zid; - - mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec); -for (zid =0 ; zid <MAX_NR_ZONES ; zid++) - nr_pages += mz->lru_zone_size[zid][lru]; -return nr_pages; -} -static inline unsigned long mem_cgroup_get_zone_lru_size (struct lruvec *lruvec,enum lru_list lru,int zone_idx)
@@ -985,11 +972,6 @@ static inline bool mem_cgroup_online(struct mem_cgroup *memcg)return true; } -static inline unsigned long -mem_cgroup_get_lru_size (struct lruvec *lruvec,enum lru_list lru) -{ -return 0 ; -}static inline unsigned long mem_cgroup_get_zone_lru_size (struct lruvec *lruvec,enum lru_list lru,int zone_idx)
mm/memcontrol.c +1 -1
@@ -737,7 +737,7 @@ unsigned long mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg,for_each_lru (lru) {if (!(BIT (lru) & lru_mask))continue ; - nr +=mem_cgroup_get_lru_size (lruvec, lru); + nr +=lruvec_page_state (lruvec,NR_LRU_BASE + lru); }return nr; }
mm/vmscan.c +1 -1
@@ -346,7 +346,7 @@ unsigned long lruvec_lru_size(struct lruvec *lruvec, enum lru_list lru, int zoneint zid;if (!mem_cgroup_disabled ()) - lru_size =mem_cgroup_get_lru_size (lruvec, lru); + lru_size =lruvec_page_state (lruvec,NR_LRU_BASE + lru);else lru_size =node_page_state (lruvec_pgdat (lruvec),NR_LRU_BASE + lru);
Signature · 1 line
2.20.1
Johannes Weiner
maintainerPATCH 3/6 mm: memcontrol: replace node summing with memcg_page_state()
In reply to a message not in the archiveInstead of adding up the node counters, use memcg_page_state() to get the memcg state directly. This is a bit cheaper and more stream-lined.
--- mm/memcontrol.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-)
mm/memcontrol.c +6 -3
@@ -746,10 +746,13 @@ static unsigned long mem_cgroup_nr_lru_pages(struct mem_cgroup *memcg, unsigned int lru_mask) {unsigned long nr =0 ; -int nid; +enum lru_list lru; -for_each_node_state (nid,N_MEMORY ) - nr +=mem_cgroup_node_nr_lru_pages (memcg, nid, lru_mask); +for_each_lru (lru) { +if (!(BIT (lru) & lru_mask)) +continue ; + nr +=memcg_page_state (memcg,NR_LRU_BASE + lru); + }return nr; }
Signature · 1 line
2.20.1
Johannes Weiner
maintainerPATCH 4/6 mm: memcontrol: push down mem_cgroup_node_nr_lru_pages()
In reply to a message not in the archivemem_cgroup_node_nr_lru_pages() is just a convenience wrapper around lruvec_page_state() that takes bitmasks of lru indexes and aggregates the counts for those.
Replace callsites where the bitmask is simple enough with direct lruvec_page_state() calls.
This removes the last extern user of mem_cgroup_node_nr_lru_pages(), so make that function private again, too.
--- include/linux/memcontrol.h | 10 ---------- mm/memcontrol.c | 10 +++++++--- mm/workingset.c | 5 +++-- 3 files changed, 10 insertions(+), 15 deletions(-)
include/linux/memcontrol.h +0 -10
@@ -514,9 +514,6 @@ int mem_cgroup_select_victim_node(struct mem_cgroup *memcg);void mem_cgroup_update_lru_size (struct lruvec *lruvec,enum lru_list lru,int zid,int nr_pages); -unsigned long mem_cgroup_node_nr_lru_pages (struct mem_cgroup *memcg, -int nid,unsigned int lru_mask); -static inline unsigned long mem_cgroup_get_zone_lru_size (struct lruvec *lruvec,enum lru_list lru,int zone_idx)
@@ -979,13 +976,6 @@ unsigned long mem_cgroup_get_zone_lru_size(struct lruvec *lruvec,return 0 ; } -static inline unsigned long -mem_cgroup_node_nr_lru_pages (struct mem_cgroup *memcg, -int nid,unsigned int lru_mask) -{ -return 0 ; -} -static inline unsigned long mem_cgroup_get_max (struct mem_cgroup *memcg) {return 0 ;
mm/memcontrol.c +7 -3
@@ -725,7 +725,7 @@ static void mem_cgroup_charge_statistics(struct mem_cgroup *memcg,__this_cpu_add (memcg->vmstats_percpu->nr_page_events, nr_pages); } -unsigned long mem_cgroup_node_nr_lru_pages (struct mem_cgroup *memcg, +static unsigned long mem_cgroup_node_nr_lru_pages (struct mem_cgroup *memcg,int nid,unsigned int lru_mask) {struct lruvec *lruvec =mem_cgroup_lruvec (NODE_DATA (nid), memcg);
@@ -1430,11 +1430,15 @@ static bool mem_cgroup_out_of_memory(struct mem_cgroup *memcg, gfp_t gfp_mask,static bool test_mem_cgroup_node_reclaimable (struct mem_cgroup *memcg,int nid,bool noswap) { -if (mem_cgroup_node_nr_lru_pages (memcg, nid,LRU_ALL_FILE )) +struct lruvec *lruvec =mem_cgroup_lruvec (NODE_DATA (nid), memcg); + +if (lruvec_page_state (lruvec,NR_INACTIVE_FILE ) || +lruvec_page_state (lruvec,NR_ACTIVE_FILE ))return true;if (noswap || !total_swap_pages)return false; -if (mem_cgroup_node_nr_lru_pages (memcg, nid,LRU_ALL_ANON )) +if (lruvec_page_state (lruvec,NR_INACTIVE_ANON ) || +lruvec_page_state (lruvec,NR_ACTIVE_ANON ))return true;return false;
mm/workingset.c +3 -2
@@ -427,10 +427,11 @@ static unsigned long count_shadow_nodes(struct shrinker *shrinker,#ifdef CONFIG_MEMCG if (sc->memcg) {struct lruvec *lruvec; +int i; - pages =mem_cgroup_node_nr_lru_pages (sc->memcg, sc->nid, -LRU_ALL ); lruvec =mem_cgroup_lruvec (NODE_DATA (sc->nid), sc->memcg); +for (pages =0 , i =0 ; i <NR_LRU_LISTS ; i++) + pages +=lruvec_page_state (lruvec,NR_LRU_BASE + i); pages +=lruvec_page_state (lruvec,NR_SLAB_RECLAIMABLE ); pages +=lruvec_page_state (lruvec,NR_SLAB_UNRECLAIMABLE ); }else
Signature · 1 line
2.20.1
Johannes Weiner
maintainerPATCH 5/6 mm: memcontrol: push down mem_cgroup_nr_lru_pages()
In reply to a message not in the archivemem_cgroup_nr_lru_pages() is just a convenience wrapper around memcg_page_state() that takes bitmasks of lru indexes and aggregates the counts for those.
Replace callsites where the bitmask is simple enough with direct memcg_page_state() call(s).
--- mm/memcontrol.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-)
mm/memcontrol.c +7 -6
@@ -1361,7 +1361,7 @@ void mem_cgroup_print_oom_meminfo(struct mem_cgroup *memcg)for (i =0 ; i <NR_LRU_LISTS ; i++)pr_cont (" %s:%luKB" , mem_cgroup_lru_names[i], -K (mem_cgroup_nr_lru_pages (iter,BIT (i)))); +K (memcg_page_state (iter,NR_LRU_BASE + i)));pr_cont ("\n" ); }
@@ -3016,8 +3016,8 @@ static void accumulate_vmstats(struct mem_cgroup *memcg, ? acc->vmevents_array[i] : i);for (i =0 ; i <NR_LRU_LISTS ; i++) - acc->lru_pages[i] += -mem_cgroup_nr_lru_pages (mi,BIT (i)); + acc->lru_pages[i] +=memcg_page_state (mi, +NR_LRU_BASE + i); } }
@@ -3447,7 +3447,8 @@ static int memcg_stat_show(struct seq_file *m, void *v)for (i =0 ; i <NR_LRU_LISTS ; i++)seq_printf (m,"%s %lu\n" , mem_cgroup_lru_names[i], -mem_cgroup_nr_lru_pages (memcg,BIT (i)) *PAGE_SIZE ); +memcg_page_state (memcg,NR_LRU_BASE + i) * +PAGE_SIZE );/* Hierarchical information */ memory = memsw =PAGE_COUNTER_MAX ;
@@ -3937,8 +3938,8 @@ void mem_cgroup_wb_stats(struct bdi_writeback *wb, unsigned long *pfilepages,/* this should eventually include NR_UNSTABLE_NFS */ *pwriteback =memcg_page_state (memcg,NR_WRITEBACK ); - *pfilepages =mem_cgroup_nr_lru_pages (memcg, (1 <<LRU_INACTIVE_FILE ) | - (1 <<LRU_ACTIVE_FILE )); + *pfilepages =memcg_page_state (memcg,NR_INACTIVE_FILE ) + +memcg_page_state (memcg,NR_ACTIVE_FILE ); *pheadroom =PAGE_COUNTER_MAX ;while ((parent =parent_mem_cgroup (memcg))) {
Signature · 1 line
2.20.1
Johannes Weiner
maintainerPATCH 6/6 mm: memcontrol: quarantine the mem_cgroup_[node_]nr_lru_pages() API
In reply to a message not in the archiveOnly memcg_numa_stat_show() uses those wrappers and the lru bitmasks, group them together.
--- include/linux/mmzone.h | 5 ---- mm/memcontrol.c | 67 +++++++++++++++++++++++------------------- 2 files changed, 36 insertions(+), 36 deletions(-)
include/linux/mmzone.h +0 -5
@@ -305,11 +305,6 @@ struct lruvec {#endif }; -/* Mask used at gathering information at once (see memcontrol.c) */ -#define LRU_ALL_FILE (BIT(LRU_INACTIVE_FILE) | BIT(LRU_ACTIVE_FILE)) -#define LRU_ALL_ANON (BIT(LRU_INACTIVE_ANON) | BIT(LRU_ACTIVE_ANON)) -#define LRU_ALL ((1 << NR_LRU_LISTS) - 1) -/* Isolate unmapped file */ #define ISOLATE_UNMAPPED ((__force isolate_mode_t)0x2)/* Isolate for asynchronous migration */
mm/memcontrol.c +36 -31
79 lines not on this page · raw message
Signature · 1 line
2.20.1
Roman Gushchin
Reviewed-byRe: your mail
In reply to a message not in the archiveQuoted 8 lines · Johannes Weiner
[resending, rebased on top of latest mmots]
The memcg LRU stats usage is currently a bit messy. Memcg has private per-zone counters because reclaim needs zone granularity sometimes, but we also have plenty of users that need to awkwardly sum them up to node or memcg granularity. Meanwhile the canonical per-memcg vmstats do not track the LRU counts (
NR_INACTIVE_ANONetc.) as you'd expect.
Quoted 4 lines
This series enables LRU count tracking in the per-memcg vmstats array such that
lruvec_page_state()andmemcg_page_state()work on the enumnode_stat_itemitems for the LRU counters. Then it converts all the callers that don't specifically need per-zone numbers over to that.
The updated version looks very good* to me!
Please, feel free to use:
Looking through the patchset, I have a feeling that we're sometimes gathering too much data. Perhaps we don't need the whole set of counters to be per-cpu on both memcg- and memcg-per-node levels. Merging them can save quite a lot of space. Anyway, it's a separate topic.
* except "to" and "subject" of the cover letter
Thanks!