memcg accounting for BPF arena
v3 cover memcg accounting for BPF arena
v2: https://lore.kernel.org/all/[email protected]/
Changes in v2->v3:
- Remove memcg accounting from bpf_map_alloc_pages() as the caller does it already. (Alexei)
- Do memcg set/recover in arena_alloc/free_pages() rather than bpf_arena_alloc/free_pages(), it reduces copy pasting in sleepable/non_sleepable functions.
v1: https://lore.kernel.org/all/[email protected]/
Changes in v1->v2:
- Return both pointers through arguments from bpf_map_memcg_enter and make it return void. (Alexei)
- Add memcg accounting in arena_free_worker (AI)
This set adds memcg accounting logic into arena kfuncs and other places that do allocations in arena.c.
Puranjay Mohan (2):
bpf: syscall: Introduce memcg enter/exit helpers
bpf: arena: Reintroduce memcg accounting
include/linux/bpf.h | 15 ++++++++++++ kernel/bpf/arena.c | 34 +++++++++++++++++++++++---- kernel/bpf/range_tree.c | 5 ++-- kernel/bpf/syscall.c | 52 +++++++++++++++++++---------------------- 4 files changed, 72 insertions(+), 34 deletions(-)
base-commit: c286e7e9d1f1f3d90ad11c37e896f582b02d19c4
Signature · 1 line
2.47.3
PATCH v3 1/2 bpf: syscall: Introduce memcg enter/exit helpers
Introduce bpf_map_memcg_enter() and bpf_map_memcg_exit() helpers to reduce code duplication in memcg context management.
bpf_map_memcg_enter() gets the memcg from the map, sets it as active, and returns both the previous and the now active memcg.
bpf_map_memcg_exit() restores the previous active memcg and releases the reference obtained during enter.
--- include/linux/bpf.h | 15 ++++++++++++ kernel/bpf/syscall.c | 54 +++++++++++++++++++++----------------------- 2 files changed, 41 insertions(+), 28 deletions(-)
include/linux/bpf.h +15 -0
@@ -2608,6 +2608,10 @@ struct bpf_prog *bpf_prog_get_curr_or_next(u32 *id);int bpf_map_alloc_pages (const struct bpf_map *map,int nid,unsigned long nr_pages,struct page **page_array);#ifdef CONFIG_MEMCG +void bpf_map_memcg_enter (const struct bpf_map *map,struct mem_cgroup **old_memcg, +struct mem_cgroup **new_memcg); +void bpf_map_memcg_exit (struct mem_cgroup *old_memcg, +struct mem_cgroup *memcg);void *bpf_map_kmalloc_node (const struct bpf_map *map,size_t size,gfp_t flags,int node); void *bpf_map_kmalloc_nolock(const struct bpf_map *map, size_t size, gfp_t flags,
@@ -2632,6 +2636,17 @@ void __percpu *bpf_map_alloc_percpu(const struct bpf_map *map, size_t size, kvcalloc(_n, _size, _flags)#define bpf_map_alloc_percpu (_map, _size, _align, _flags) \ __alloc_percpu_gfp(_size, _align, _flags) +static inline void bpf_map_memcg_enter (const struct bpf_map *map,struct mem_cgroup **old_memcg, +struct mem_cgroup **new_memcg) +{ + *new_memcg =NULL ; + *old_memcg =NULL ; +} + +static inline void bpf_map_memcg_exit (struct mem_cgroup *old_memcg, +struct mem_cgroup *memcg) +{ +}#endif static inline int
kernel/bpf/syscall.c +26 -28
109 lines not on this page · raw message
Signature · 1 line
2.47.3
PATCH v3 2/2 bpf: arena: Reintroduce memcg accounting
In reply to Puranjay MohanWhen arena allocations were converted from bpf_map_alloc_pages() to kmalloc_nolock() to support non-sleepable contexts, memcg accounting was inadvertently lost. This commit restores proper memory accounting for all arena-related allocations.
All arena related allocations are accounted into memcg of the process that created bpf_arena.
--- kernel/bpf/arena.c | 34 ++++++++++++++++++++++++++++++---- kernel/bpf/range_tree.c | 5 +++-- kernel/bpf/syscall.c | 2 -- 3 files changed, 33 insertions(+), 8 deletions(-)
kernel/bpf/arena.c +30 -4
@@ -360,6 +360,7 @@ static vm_fault_t arena_vm_fault(struct vm_fault *vmf) {struct bpf_map *map = vmf->vma->vm_file->private_data; struct bpf_arena *arena = container_of(map, struct bpf_arena, map); +struct mem_cgroup *new_memcg, *old_memcg;struct page *page;long kbase, kaddr;unsigned long flags;
@@ -377,6 +378,8 @@ static vm_fault_t arena_vm_fault(struct vm_fault *vmf)/* already have a page vmap-ed */ goto out; +bpf_map_memcg_enter (&arena->map, &old_memcg, &new_memcg); +if (arena->map.map_flags &BPF_F_SEGV_ON_FAULT )/* User space requested to segfault when page is not allocated by bpf prog */ goto out_unlock_sigsegv;
@@ -400,12 +403,14 @@ static vm_fault_t arena_vm_fault(struct vm_fault *vmf) goto out_unlock_sigsegv; }flush_vmap_cache (kaddr,PAGE_SIZE ); +bpf_map_memcg_exit (old_memcg, new_memcg); out:page_ref_add (page,1 );raw_res_spin_unlock_irqrestore (&arena->spinlock, flags); vmf->page = page;return 0 ; out_unlock_sigsegv: +bpf_map_memcg_exit (old_memcg, new_memcg);raw_res_spin_unlock_irqrestore (&arena->spinlock, flags);return VM_FAULT_SIGSEGV ; }
@@ -534,6 +539,7 @@ static long arena_alloc_pages(struct bpf_arena *arena, long uaddr, long page_cnt/* user_vm_end/start are fixed before bpf prog runs */ long page_cnt_max = (arena->user_vm_end - arena->user_vm_start) >>PAGE_SHIFT ;u64 kern_vm_start =bpf_arena_get_kern_vm_start (arena); +struct mem_cgroup *new_memcg, *old_memcg;struct apply_range_data data;struct page **pages =NULL ;long remaining, mapped =0 ;
@@ -555,11 +561,14 @@ static long arena_alloc_pages(struct bpf_arena *arena, long uaddr, long page_cntreturn 0 ; } +bpf_map_memcg_enter (&arena->map, &old_memcg, &new_memcg);/* Cap allocation size to KMALLOC_MAX_CACHE_SIZE so kmalloc_nolock() can succeed. */ alloc_pages =min (page_cnt,KMALLOC_MAX_CACHE_SIZE /sizeof (struct page *)); - pages =kmalloc_nolock (alloc_pages *sizeof (struct page *),0 ,NUMA_NO_NODE ); -if (!pages) + pages =kmalloc_nolock (alloc_pages *sizeof (struct page *), __GFP_ACCOUNT,NUMA_NO_NODE ); +if (!pages) { +bpf_map_memcg_exit (old_memcg, new_memcg);return 0 ; + } data.pages = pages;if (raw_res_spin_lock_irqsave (&arena->spinlock, flags))
@@ -617,6 +626,7 @@ static long arena_alloc_pages(struct bpf_arena *arena, long uaddr, long page_cntflush_vmap_cache (kern_vm_start + uaddr32, mapped <<PAGE_SHIFT );raw_res_spin_unlock_irqrestore (&arena->spinlock, flags);kfree_nolock (pages); +bpf_map_memcg_exit (old_memcg, new_memcg);return clear_lo32 (arena->user_vm_start) + uaddr32; out:range_tree_set (&arena->rt, pgoff + mapped, page_cnt - mapped);
@@ -630,6 +640,7 @@ static long arena_alloc_pages(struct bpf_arena *arena, long uaddr, long page_cntraw_res_spin_unlock_irqrestore (&arena->spinlock, flags); out_free_pages:kfree_nolock (pages); +bpf_map_memcg_exit (old_memcg, new_memcg);return 0 ; }
@@ -651,6 +662,7 @@ static void zap_pages(struct bpf_arena *arena, long uaddr, long page_cnt)static void arena_free_pages (struct bpf_arena *arena,long uaddr,long page_cnt,bool sleepable) { +struct mem_cgroup *new_memcg, *old_memcg;u64 full_uaddr, uaddr_end;long kaddr, pgoff;struct page *page;
@@ -671,6 +683,7 @@ static void arena_free_pages(struct bpf_arena *arena, long uaddr, long page_cnt, page_cnt = (uaddr_end - full_uaddr) >>PAGE_SHIFT ; pgoff =compute_pgoff (arena, uaddr); +bpf_map_memcg_enter (&arena->map, &old_memcg, &new_memcg);if (!sleepable) goto defer;
@@ -709,11 +722,13 @@ static void arena_free_pages(struct bpf_arena *arena, long uaddr, long page_cnt,zap_pages (arena, full_uaddr,1 );__free_page (page); } +bpf_map_memcg_exit (old_memcg, new_memcg);return ; defer: - s =kmalloc_nolock (sizeof (struct arena_free_span ),0 ,-1 ); + s =kmalloc_nolock (sizeof (struct arena_free_span ), __GFP_ACCOUNT,-1 ); +bpf_map_memcg_exit (old_memcg, new_memcg);if (!s)/* * If allocation fails in non-sleepable context, pages are intentionally left
@@ -766,6 +781,7 @@ static int arena_reserve_pages(struct bpf_arena *arena, long uaddr, u32 page_cntstatic void arena_free_worker (struct work_struct *work) { struct bpf_arena *arena = container_of(work, struct bpf_arena, free_work); +struct mem_cgroup *new_memcg, *old_memcg;struct llist_node *list, *pos, *t;struct arena_free_span *s;u64 arena_vm_start, user_vm_start;
@@ -780,6 +796,8 @@ static void arena_free_worker(struct work_struct *work)return ; } +bpf_map_memcg_enter (&arena->map, &old_memcg, &new_memcg); +init_llist_head (&free_pages); arena_vm_start =bpf_arena_get_kern_vm_start (arena); user_vm_start =bpf_arena_get_user_vm_start (arena);
@@ -820,6 +838,8 @@ static void arena_free_worker(struct work_struct *work) page = llist_entry(pos, struct page, pcp_llist);__free_page (page); } + +bpf_map_memcg_exit (old_memcg, new_memcg); }static void arena_free_irq (struct irq_work *iw)
@@ -876,7 +896,9 @@ void bpf_arena_free_pages_non_sleepable(void *p__map, void *ptr__ign, u32 page_c __bpf_kfunc int bpf_arena_reserve_pages(void *p__map, void *ptr__ign, u32 page_cnt) { +int ret;struct bpf_map *map = p__map; +struct mem_cgroup *new_memcg, *old_memcg; struct bpf_arena *arena = container_of(map, struct bpf_arena, map);if (map->map_type !=BPF_MAP_TYPE_ARENA )
@@ -885,7 +907,11 @@ __bpf_kfunc int bpf_arena_reserve_pages(void *p__map, void *ptr__ign, u32 page_cif (!page_cnt)return 0 ; -return arena_reserve_pages (arena, (long )ptr__ign, page_cnt); +bpf_map_memcg_enter (map, &old_memcg, &new_memcg); + ret =arena_reserve_pages (arena, (long )ptr__ign, page_cnt); +bpf_map_memcg_exit (old_memcg, new_memcg); + +return ret; }__bpf_kfunc_end_defs ();
kernel/bpf/range_tree.c +3 -2
@@ -149,7 +149,8 @@ int range_tree_clear(struct range_tree *rt, u32 start, u32 len)range_it_insert (rn, rt);/* Add a range */ - new_rn =kmalloc_nolock (sizeof (struct range_node ),0 ,NUMA_NO_NODE ); + new_rn =kmalloc_nolock (sizeof (struct range_node ), __GFP_ACCOUNT, +NUMA_NO_NODE );if (!new_rn)return -ENOMEM ; new_rn->rn_start = last +1 ;
@@ -234,7 +235,7 @@ int range_tree_set(struct range_tree *rt, u32 start, u32 len) right->rn_start = start;range_it_insert (right, rt); }else { - left =kmalloc_nolock (sizeof (struct range_node ),0 ,NUMA_NO_NODE ); + left =kmalloc_nolock (sizeof (struct range_node ), __GFP_ACCOUNT,NUMA_NO_NODE );if (!left)return -ENOMEM ; left->rn_start = start;
kernel/bpf/syscall.c +0 -2
@@ -618,7 +618,6 @@ int bpf_map_alloc_pages(const struct bpf_map *map, int nid,int ret =0 ;struct mem_cgroup *memcg, *old_memcg; -bpf_map_memcg_enter (map, &old_memcg, &memcg);for (i =0 ; i < nr_pages; i++) { pg =__bpf_alloc_page (nid);
@@ -632,7 +631,6 @@ int bpf_map_alloc_pages(const struct bpf_map *map, int nid,break ; } -bpf_map_memcg_exit (old_memcg, memcg);return ret; }
Signature · 1 line
2.47.3
Re: [PATCH bpf-next v3 2/2] bpf: arena: Reintroduce memcg accounting
kernel/bpf/arena.c __bpf_kfunc int bpf_arena_reserve_pages(void *p__map, void *ptr__ign, u32 page_c · line 912
Quoted 10 lines · Puranjay Mohan
if (map->map_type != BPF_MAP_TYPE_ARENA)kernel/bpf/arena.c +3
-1@@ -885,7 +907,11 @@ __bpf_kfunc int bpf_arena_reserve_pages(void *p__map, void *ptr__ign, u32 page_cif (!page_cnt)return 0 ; -return arena_reserve_pages (arena, (long )ptr__ign, page_cnt); +bpf_map_memcg_enter (map, &old_memcg, &new_memcg); + ret =arena_reserve_pages (arena, (long )ptr__ign, page_cnt); +bpf_map_memcg_exit (old_memcg, new_memcg);
This one can also move into arena_reserve_pages() for symmetry with the rest and wrap range_tree_clear() call.
pw-bot: cr
Re: [PATCH bpf-next v3 2/2] bpf: arena: Reintroduce memcg accounting
Quoted 16 lines · Alexei Starovoitov
kernel/bpf/arena.c __bpf_kfunc int bpf_arena_reserve_pages(void *p__map, void *ptr__ign, u32 page_c · line 912
Quoted 10 lines · Puranjay Mohan
if (map->map_type != BPF_MAP_TYPE_ARENA)kernel/bpf/arena.c +3
-1@@ -885,7 +907,11 @@ __bpf_kfunc int bpf_arena_reserve_pages(void *p__map, void *ptr__ign, u32 page_cif (!page_cnt)return 0 ; -return arena_reserve_pages (arena, (long )ptr__ign, page_cnt); +bpf_map_memcg_enter (map, &old_memcg, &new_memcg); + ret =arena_reserve_pages (arena, (long )ptr__ign, page_cnt); +bpf_map_memcg_exit (old_memcg, new_memcg);This one can also move into
arena_reserve_pages()for symmetry with the rest and wraprange_tree_clear()call.
Quoted 1 line
pw-bot: cr
Okay, sending v4 with this now.
kernel test robot
build botRe: [PATCH bpf-next v3 2/2] bpf: arena: Reintroduce memcg accounting
In reply to Puranjay Mohankernel test robot noticed the following build warnings:
Hi Puranjay,
kernel test robot noticed the following build warnings:
[auto build test WARNING on c286e7e9d1f1f3d90ad11c37e896f582b02d19c4]
url: https://github.com/intel-lab-lkp/linux/commits/Puranjay-Mohan/bpf-syscall-Introduce-memcg-enter-exit-helpers/20260102-232059 base: c286e7e9d1f1f3d90ad11c37e896f582b02d19c4 patch link: https://lore.kernel.org/r/20260102151852.570285-3-puranjay%40kernel.org patch subject: [PATCH bpf-next v3 2/2] bpf: arena: Reintroduce memcg accounting config: x86_64-kexec (https://download.01.org/0day-ci/archive/20260103/[email protected]/config) compiler: clang version 20.1.8 (https://github.com/llvm/llvm-project 87f0227cb60147a26a1eeb4fb06e3b505e9c7261) reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20260103/[email protected]/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <[email protected]>
| Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/
All warnings (new ones prefixed by >>):
>> kernel/bpf/syscall.c:619:21: warning: unused variable 'memcg' [-Wunused-variable]
619 | struct mem_cgroup *memcg, *old_memcg;
| ^~~~~
>> kernel/bpf/syscall.c:619:29: warning: unused variable 'old_memcg' [-Wunused-variable]
619 | struct mem_cgroup *memcg, *old_memcg;
| ^~~~~~~~~2 warnings generated.
vim +/memcg +619 kernel/bpf/syscall.c
c9eb8102e21e8c Alexei Starovoitov 2025-02-21 612
c9eb8102e21e8c Alexei Starovoitov 2025-02-21 613 int bpf_map_alloc_pages(const struct bpf_map *map, int nid,
317460317a02a1 Alexei Starovoitov 2024-03-07 614 unsigned long nr_pages, struct page **pages)
317460317a02a1 Alexei Starovoitov 2024-03-07 615 {
317460317a02a1 Alexei Starovoitov 2024-03-07 616 unsigned long i, j;
317460317a02a1 Alexei Starovoitov 2024-03-07 617 struct page *pg;
317460317a02a1 Alexei Starovoitov 2024-03-07 618 int ret = 0;
317460317a02a1 Alexei Starovoitov 2024-03-07 @619 struct mem_cgroup *memcg, *old_memcg;
317460317a02a1 Alexei Starovoitov 2024-03-07 620
317460317a02a1 Alexei Starovoitov 2024-03-07 621 for (i = 0; i < nr_pages; i++) {
c9eb8102e21e8c Alexei Starovoitov 2025-02-21 622 pg = __bpf_alloc_page(nid);
317460317a02a1 Alexei Starovoitov 2024-03-07 623
317460317a02a1 Alexei Starovoitov 2024-03-07 624 if (pg) {
317460317a02a1 Alexei Starovoitov 2024-03-07 625 pages[i] = pg;
317460317a02a1 Alexei Starovoitov 2024-03-07 626 continue;
317460317a02a1 Alexei Starovoitov 2024-03-07 627 }
317460317a02a1 Alexei Starovoitov 2024-03-07 628 for (j = 0; j < i; j++)
c9eb8102e21e8c Alexei Starovoitov 2025-02-21 629 free_pages_nolock(pages[j], 0);
317460317a02a1 Alexei Starovoitov 2024-03-07 630 ret = -ENOMEM;
317460317a02a1 Alexei Starovoitov 2024-03-07 631 break;
317460317a02a1 Alexei Starovoitov 2024-03-07 632 }
317460317a02a1 Alexei Starovoitov 2024-03-07 633
317460317a02a1 Alexei Starovoitov 2024-03-07 634 return ret;
317460317a02a1 Alexei Starovoitov 2024-03-07 635 }
317460317a02a1 Alexei Starovoitov 2024-03-07 636 Signature · 2 lines
0-DAY CI Kernel Test Service https://github.com/intel/lkp-tests/wiki