memcg: keep folio's objcg same as its node
Subsystems (4)
Shakeel Butt
maintainerPATCH 1/1 memcg: keep folio's objcg same as its node
memcg_reparent_objcgs() has an inherent assumption that a folio's objcg is the objcg of the folio's node. Folio migration across nodes breaks that assumption: the new folio simply inherits the old folio's objcg while living on a different node.
Once the assumption is broken, the reparenting of the folio's objcg and the reparenting of the folio's LRU list are no longer atomic. memcg_reparent_objcgs() handles one node per iteration and drops all the locks in between, so the objcg gets reparented in the iteration for the objcg's node while the LRU list gets spliced in the iteration for the folio's node. Any LRU operation on that folio in between resolves its lruvec through the objcg, and thus takes the lru_lock of the wrong memcg, not the lru_lock of the list the folio is actually on.
Fix this by selecting the objcg by folio_nid() at charge time, and by re-deriving it for the destination node in mem_cgroup_migrate() and mem_cgroup_replace_folio().
--- mm/memcontrol.c | 33 +++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-)
mm/memcontrol.c +25 -8
@@ -2966,10 +2966,9 @@ struct mem_cgroup *mem_cgroup_from_virt(void *p)return folio_memcg_check (virt_to_folio (p)); } -static struct obj_cgroup *__get_obj_cgroup_from_memcg (struct mem_cgroup *memcg) +static struct obj_cgroup *__get_obj_cgroup_from_memcg (struct mem_cgroup *memcg, +int nid) { -int nid =numa_node_id (); -for (; memcg; memcg =parent_mem_cgroup (memcg)) {struct obj_cgroup *objcg =rcu_dereference (memcg->nodeinfo[nid]->objcg);
@@ -2980,12 +2979,13 @@ static struct obj_cgroup *__get_obj_cgroup_from_memcg(struct mem_cgroup *memcg)return NULL ; } -static inline struct obj_cgroup *get_obj_cgroup_from_memcg (struct mem_cgroup *memcg) +static inline struct obj_cgroup *get_obj_cgroup_from_memcg (struct mem_cgroup *memcg, +int nid) {struct obj_cgroup *objcg;rcu_read_lock (); - objcg =__get_obj_cgroup_from_memcg (memcg); + objcg =__get_obj_cgroup_from_memcg (memcg, nid);rcu_read_unlock ();return objcg;
@@ -3029,7 +3029,7 @@ static struct obj_cgroup *current_objcg_update(void)rcu_read_lock (); memcg =mem_cgroup_from_task (current); - objcg =__get_obj_cgroup_from_memcg (memcg); + objcg =__get_obj_cgroup_from_memcg (memcg,numa_node_id ());rcu_read_unlock ();/*
@@ -5197,7 +5197,7 @@ static int charge_memcg(struct folio *folio, struct mem_cgroup *memcg,int ret =0 ;struct obj_cgroup *objcg; - objcg =get_obj_cgroup_from_memcg (memcg); + objcg =get_obj_cgroup_from_memcg (memcg,folio_nid (folio));/* Do not account at the root objcg level. */ if (!obj_cgroup_is_root (objcg)) ret =try_charge_memcg (memcg, gfp,folio_nr_pages (folio));
@@ -5431,6 +5431,7 @@ void mem_cgroup_replace_folio(struct folio *old, struct folio *new)rcu_read_lock (); memcg =obj_cgroup_memcg (objcg); +/* Force-charge the new page. The old one will be freed soon */ if (!obj_cgroup_is_root (objcg)) {page_counter_charge (&memcg->memory, nr_pages);
@@ -5438,7 +5439,12 @@ void mem_cgroup_replace_folio(struct folio *old, struct folio *new)page_counter_charge (&memcg->memsw, nr_pages); } -obj_cgroup_get (objcg); +/* If replacing folio of different node, get objcg of that node. */ +if (folio_nid (old) !=folio_nid (new)) + objcg =__get_obj_cgroup_from_memcg (memcg,folio_nid (new)); +else +obj_cgroup_get (objcg); +commit_charge (new, objcg);memcg1_commit_charge (new, memcg);rcu_read_unlock ();
@@ -5478,6 +5484,17 @@ void mem_cgroup_migrate(struct folio *old, struct folio *new)if (!objcg)return ; +/* If migrating to different node, get objcg of that node. */ +if (folio_nid (old) !=folio_nid (new)) { +struct obj_cgroup *old_objcg = objcg; + +rcu_read_lock (); + objcg =__get_obj_cgroup_from_memcg (obj_cgroup_memcg (old_objcg), +folio_nid (new)); +rcu_read_unlock (); +obj_cgroup_put (old_objcg); + } +/* Transfer the charge and the objcg ref */ commit_charge (new, objcg);
Signature · 1 line
2.53.0-Meta
Johannes Weiner
maintainerQuoted 14 lines · Shakeel Butt
memcg_reparent_objcgs()has an inherent assumption that a folio's objcg is the objcg of the folio's node. Folio migration across nodes breaks that assumption: the new folio simply inherits the old folio's objcg while living on a different node.Once the assumption is broken, the reparenting of the folio's objcg and the reparenting of the folio's LRU list are no longer atomic.
memcg_reparent_objcgs()handles one node per iteration and drops all the locks in between, so the objcg gets reparented in the iteration for the objcg's node while the LRU list gets spliced in the iteration for the folio's node. Any LRU operation on that folio in between resolves its lruvec through the objcg, and thus takes thelru_lockof the wrong memcg, not thelru_lockof the list the folio is actually on.
Quoted 3 lines
Fix this by selecting the objcg by
folio_nid()at charge time, and by re-deriving it for the destination node inmem_cgroup_migrate()andmem_cgroup_replace_folio().
Nice sleuthing.
Quoted 89 lines
Reported-by Karl Erik Hofseth <[email protected]>Fixes f1cf8d2f36dc ("mm: memcontrol: eliminate the problem of dying memory cgroup for LRU folios")Signed-off-by Shakeel Butt <[email protected]>--- mm/memcontrol.c | 33 +++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-)mm/memcontrol.c +17
-8@@ -2966,10 +2966,9 @@ struct mem_cgroup *mem_cgroup_from_virt(void *p)return folio_memcg_check (virt_to_folio (p)); } -static struct obj_cgroup *__get_obj_cgroup_from_memcg (struct mem_cgroup *memcg) +static struct obj_cgroup *__get_obj_cgroup_from_memcg (struct mem_cgroup *memcg, +int nid) { -int nid =numa_node_id (); -for (; memcg; memcg =parent_mem_cgroup (memcg)) {struct obj_cgroup *objcg =rcu_dereference (memcg->nodeinfo[nid]->objcg);@@ -2980,12 +2979,13 @@ static struct obj_cgroup *__get_obj_cgroup_from_memcg(struct mem_cgroup *memcg)return NULL ; } -static inline struct obj_cgroup *get_obj_cgroup_from_memcg (struct mem_cgroup *memcg) +static inline struct obj_cgroup *get_obj_cgroup_from_memcg (struct mem_cgroup *memcg, +int nid) {struct obj_cgroup *objcg;rcu_read_lock (); - objcg =__get_obj_cgroup_from_memcg (memcg); + objcg =__get_obj_cgroup_from_memcg (memcg, nid);rcu_read_unlock ();return objcg;@@ -3029,7 +3029,7 @@ static struct obj_cgroup *current_objcg_update(void)rcu_read_lock (); memcg =mem_cgroup_from_task (current); - objcg =__get_obj_cgroup_from_memcg (memcg); + objcg =__get_obj_cgroup_from_memcg (memcg,numa_node_id ());rcu_read_unlock ();/* @@ -5197,7 +5197,7 @@ static int charge_memcg(struct folio *folio, struct mem_cgroup *memcg,int ret =0 ;struct obj_cgroup *objcg; - objcg =get_obj_cgroup_from_memcg (memcg); + objcg =get_obj_cgroup_from_memcg (memcg,folio_nid (folio));/* Do not account at the root objcg level. */ if (!obj_cgroup_is_root (objcg)) ret =try_charge_memcg (memcg, gfp,folio_nr_pages (folio));@@ -5431,6 +5431,7 @@ void mem_cgroup_replace_folio(struct folio *old, struct folio *new)rcu_read_lock (); memcg =obj_cgroup_memcg (objcg); +/* Force-charge the new page. The old one will be freed soon */ if (!obj_cgroup_is_root (objcg)) {page_counter_charge (&memcg->memory, nr_pages);@@ -5438,7 +5439,12 @@ void mem_cgroup_replace_folio(struct folio *old, struct folio *new)page_counter_charge (&memcg->memsw, nr_pages); } -obj_cgroup_get (objcg); +/* If replacing folio of different node, get objcg of that node. */ +if (folio_nid (old) !=folio_nid (new)) + objcg =__get_obj_cgroup_from_memcg (memcg,folio_nid (new)); +else +obj_cgroup_get (objcg); +commit_charge (new, objcg);memcg1_commit_charge (new, memcg);rcu_read_unlock ();@@ -5478,6 +5484,17 @@ void mem_cgroup_migrate(struct folio *old, struct folio *new)if (!objcg)return ; +/* If migrating to different node, get objcg of that node. */ +if (folio_nid (old) !=folio_nid (new)) { +struct obj_cgroup *old_objcg = objcg;
mm/memcontrol.c void mem_cgroup_migrate(struct folio *old, struct folio *new) · line 5499
Quoted 10 lines
mm/memcontrol.c +8
-0+ +rcu_read_lock (); + objcg =__get_obj_cgroup_from_memcg (obj_cgroup_memcg (old_objcg), +folio_nid (new)); +rcu_read_unlock (); +obj_cgroup_put (old_objcg); + } +/* Transfer the charge and the objcg ref */ commit_charge (new, objcg);
Just noticed Sashiko pointed this out too, but you put old_objcg while old->memcg_data still references it. I don't see how that could lead to a UAF but it's fragile.
Man, this is hairy. I missed the implications when I suggested to make the objcg per node for simplifcation elsewhere. Now locality actually matters -_-
A comment describing the race with reparenting would be very helpful.
Shakeel Butt
maintainerQuoted 15 lines · Johannes Weiner
Quoted 14 lines · Shakeel Butt
memcg_reparent_objcgs()has an inherent assumption that a folio's objcg is the objcg of the folio's node. Folio migration across nodes breaks that assumption: the new folio simply inherits the old folio's objcg while living on a different node.Once the assumption is broken, the reparenting of the folio's objcg and the reparenting of the folio's LRU list are no longer atomic.
memcg_reparent_objcgs()handles one node per iteration and drops all the locks in between, so the objcg gets reparented in the iteration for the objcg's node while the LRU list gets spliced in the iteration for the folio's node. Any LRU operation on that folio in between resolves its lruvec through the objcg, and thus takes thelru_lockof the wrong memcg, not thelru_lockof the list the folio is actually on.
Quoted 5 lines
Quoted 3 lines
Fix this by selecting the objcg by
folio_nid()at charge time, and by re-deriving it for the destination node inmem_cgroup_migrate()andmem_cgroup_replace_folio().Nice sleuthing.
Thanks and it was actually due to very good reproducer provided by Karl.
Quoted 96 lines
Quoted 96 lines
Reported-by Karl Erik Hofseth <[email protected]>Fixes f1cf8d2f36dc ("mm: memcontrol: eliminate the problem of dying memory cgroup for LRU folios")Signed-off-by Shakeel Butt <[email protected]>--- mm/memcontrol.c | 33 +++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-)mm/memcontrol.c +23
-8@@ -2966,10 +2966,9 @@ struct mem_cgroup *mem_cgroup_from_virt(void *p)return folio_memcg_check (virt_to_folio (p)); } -static struct obj_cgroup *__get_obj_cgroup_from_memcg (struct mem_cgroup *memcg) +static struct obj_cgroup *__get_obj_cgroup_from_memcg (struct mem_cgroup *memcg, +int nid) { -int nid =numa_node_id (); -for (; memcg; memcg =parent_mem_cgroup (memcg)) {struct obj_cgroup *objcg =rcu_dereference (memcg->nodeinfo[nid]->objcg);@@ -2980,12 +2979,13 @@ static struct obj_cgroup *__get_obj_cgroup_from_memcg(struct mem_cgroup *memcg)return NULL ; } -static inline struct obj_cgroup *get_obj_cgroup_from_memcg (struct mem_cgroup *memcg) +static inline struct obj_cgroup *get_obj_cgroup_from_memcg (struct mem_cgroup *memcg, +int nid) {struct obj_cgroup *objcg;rcu_read_lock (); - objcg =__get_obj_cgroup_from_memcg (memcg); + objcg =__get_obj_cgroup_from_memcg (memcg, nid);rcu_read_unlock ();return objcg;@@ -3029,7 +3029,7 @@ static struct obj_cgroup *current_objcg_update(void)rcu_read_lock (); memcg =mem_cgroup_from_task (current); - objcg =__get_obj_cgroup_from_memcg (memcg); + objcg =__get_obj_cgroup_from_memcg (memcg,numa_node_id ());rcu_read_unlock ();/* @@ -5197,7 +5197,7 @@ static int charge_memcg(struct folio *folio, struct mem_cgroup *memcg,int ret =0 ;struct obj_cgroup *objcg; - objcg =get_obj_cgroup_from_memcg (memcg); + objcg =get_obj_cgroup_from_memcg (memcg,folio_nid (folio));/* Do not account at the root objcg level. */ if (!obj_cgroup_is_root (objcg)) ret =try_charge_memcg (memcg, gfp,folio_nr_pages (folio));@@ -5431,6 +5431,7 @@ void mem_cgroup_replace_folio(struct folio *old, struct folio *new)rcu_read_lock (); memcg =obj_cgroup_memcg (objcg); +/* Force-charge the new page. The old one will be freed soon */ if (!obj_cgroup_is_root (objcg)) {page_counter_charge (&memcg->memory, nr_pages);@@ -5438,7 +5439,12 @@ void mem_cgroup_replace_folio(struct folio *old, struct folio *new)page_counter_charge (&memcg->memsw, nr_pages); } -obj_cgroup_get (objcg); +/* If replacing folio of different node, get objcg of that node. */ +if (folio_nid (old) !=folio_nid (new)) + objcg =__get_obj_cgroup_from_memcg (memcg,folio_nid (new)); +else +obj_cgroup_get (objcg); +commit_charge (new, objcg);memcg1_commit_charge (new, memcg);rcu_read_unlock ();@@ -5478,6 +5484,17 @@ void mem_cgroup_migrate(struct folio *old, struct folio *new)if (!objcg)return ; +/* If migrating to different node, get objcg of that node. */ +if (folio_nid (old) !=folio_nid (new)) { +struct obj_cgroup *old_objcg = objcg; + +rcu_read_lock (); + objcg =__get_obj_cgroup_from_memcg (obj_cgroup_memcg (old_objcg), +folio_nid (new)); +rcu_read_unlock (); +obj_cgroup_put (old_objcg);
Quoted 8 lines
mm/memcontrol.c void mem_cgroup_migrate(struct folio *old, struct folio *new) · line 5499
Quoted 4 lines
mm/memcontrol.c +2
-0+ } +/* Transfer the charge and the objcg ref */ commit_charge (new, objcg);Just noticed Sashiko pointed this out too, but you put
old_objcgwhileold->memcg_datastill references it. I don't see how that could lead to a UAF but it's fragile.
Yeah it is complaining about folio_unqueue_deferred_split(old) accessing old->memcg_data internally. I will do the put at the end.
Quoted 4 lines
Man, this is hairy. I missed the implications when I suggested to make the objcg per node for simplifcation elsewhere. Now locality actually matters -_-
Yeah it has become complicated. The other concern Sashiko raised is the case where the destination node's objcg has reached/reparented to root and the source node's objcg is still not root. Since we skip root page counters for LRU pages, this can unbalance them. It's one more check to add.
Quoted 2 lines
A comment describing the race with reparenting would be very helpful.
Will do.