memcg accounting for BPF arena
v4 cover memcg accounting for BPF arena
v3: https://lore.kernel.org/all/[email protected]/
Changes in v3->v4:
- Do memcg set/recover in arena_reserve_pages() rather than bpf_arena_reserve_pages() for symmetry with other kfuncs (Alexei)
v2: https://lore.kernel.org/all/[email protected]/
Changes in v2->v3:
- Remove memcg accounting from bpf_map_alloc_pages() as the caller does it already. (Alexei)
- Do memcg set/recover in arena_alloc/free_pages() rather than bpf_arena_alloc/free_pages(), it reduces copy pasting in sleepable/non_sleepable functions.
v1: https://lore.kernel.org/all/[email protected]/
Changes in v1->v2:
- Return both pointers through arguments from bpf_map_memcg_enter and make it return void. (Alexei)
- Add memcg accounting in arena_free_worker (AI)
This set adds memcg accounting logic into arena kfuncs and other places that do allocations in arena.c.
Puranjay Mohan (2):
bpf: syscall: Introduce memcg enter/exit helpers
bpf: arena: Reintroduce memcg accounting
include/linux/bpf.h | 15 ++++++++++++ kernel/bpf/arena.c | 29 ++++++++++++++++++++--- kernel/bpf/range_tree.c | 5 ++-- kernel/bpf/syscall.c | 52 +++++++++++++++++++---------------------- 4 files changed, 68 insertions(+), 33 deletions(-)
base-commit: c286e7e9d1f1f3d90ad11c37e896f582b02d19c4
Signature · 1 line
2.47.3
PATCH v4 1/2 bpf: syscall: Introduce memcg enter/exit helpers
Introduce bpf_map_memcg_enter() and bpf_map_memcg_exit() helpers to reduce code duplication in memcg context management.
bpf_map_memcg_enter() gets the memcg from the map, sets it as active, and returns both the previous and the now active memcg.
bpf_map_memcg_exit() restores the previous active memcg and releases the reference obtained during enter.
--- include/linux/bpf.h | 15 ++++++++++++ kernel/bpf/syscall.c | 54 +++++++++++++++++++++----------------------- 2 files changed, 41 insertions(+), 28 deletions(-)
include/linux/bpf.h +15 -0
@@ -2608,6 +2608,10 @@ struct bpf_prog *bpf_prog_get_curr_or_next(u32 *id);int bpf_map_alloc_pages (const struct bpf_map *map,int nid,unsigned long nr_pages,struct page **page_array);#ifdef CONFIG_MEMCG +void bpf_map_memcg_enter (const struct bpf_map *map,struct mem_cgroup **old_memcg, +struct mem_cgroup **new_memcg); +void bpf_map_memcg_exit (struct mem_cgroup *old_memcg, +struct mem_cgroup *memcg);void *bpf_map_kmalloc_node (const struct bpf_map *map,size_t size,gfp_t flags,int node); void *bpf_map_kmalloc_nolock(const struct bpf_map *map, size_t size, gfp_t flags,
@@ -2632,6 +2636,17 @@ void __percpu *bpf_map_alloc_percpu(const struct bpf_map *map, size_t size, kvcalloc(_n, _size, _flags)#define bpf_map_alloc_percpu (_map, _size, _align, _flags) \ __alloc_percpu_gfp(_size, _align, _flags) +static inline void bpf_map_memcg_enter (const struct bpf_map *map,struct mem_cgroup **old_memcg, +struct mem_cgroup **new_memcg) +{ + *new_memcg =NULL ; + *old_memcg =NULL ; +} + +static inline void bpf_map_memcg_exit (struct mem_cgroup *old_memcg, +struct mem_cgroup *memcg) +{ +}#endif static inline int
kernel/bpf/syscall.c +26 -28
109 lines not on this page · raw message
Signature · 1 line
2.47.3
PATCH v4 2/2 bpf: arena: Reintroduce memcg accounting
In reply to Puranjay MohanWhen arena allocations were converted from bpf_map_alloc_pages() to kmalloc_nolock() to support non-sleepable contexts, memcg accounting was inadvertently lost. This commit restores proper memory accounting for all arena-related allocations.
All arena related allocations are accounted into memcg of the process that created bpf_arena.
--- kernel/bpf/arena.c | 29 ++++++++++++++++++++++++++--- kernel/bpf/range_tree.c | 5 +++-- kernel/bpf/syscall.c | 2 -- 3 files changed, 29 insertions(+), 7 deletions(-)
kernel/bpf/arena.c +26 -3
@@ -360,6 +360,7 @@ static vm_fault_t arena_vm_fault(struct vm_fault *vmf) {struct bpf_map *map = vmf->vma->vm_file->private_data; struct bpf_arena *arena = container_of(map, struct bpf_arena, map); +struct mem_cgroup *new_memcg, *old_memcg;struct page *page;long kbase, kaddr;unsigned long flags;
@@ -377,6 +378,8 @@ static vm_fault_t arena_vm_fault(struct vm_fault *vmf)/* already have a page vmap-ed */ goto out; +bpf_map_memcg_enter (&arena->map, &old_memcg, &new_memcg); +if (arena->map.map_flags &BPF_F_SEGV_ON_FAULT )/* User space requested to segfault when page is not allocated by bpf prog */ goto out_unlock_sigsegv;
@@ -400,12 +403,14 @@ static vm_fault_t arena_vm_fault(struct vm_fault *vmf) goto out_unlock_sigsegv; }flush_vmap_cache (kaddr,PAGE_SIZE ); +bpf_map_memcg_exit (old_memcg, new_memcg); out:page_ref_add (page,1 );raw_res_spin_unlock_irqrestore (&arena->spinlock, flags); vmf->page = page;return 0 ; out_unlock_sigsegv: +bpf_map_memcg_exit (old_memcg, new_memcg);raw_res_spin_unlock_irqrestore (&arena->spinlock, flags);return VM_FAULT_SIGSEGV ; }
@@ -534,6 +539,7 @@ static long arena_alloc_pages(struct bpf_arena *arena, long uaddr, long page_cnt/* user_vm_end/start are fixed before bpf prog runs */ long page_cnt_max = (arena->user_vm_end - arena->user_vm_start) >>PAGE_SHIFT ;u64 kern_vm_start =bpf_arena_get_kern_vm_start (arena); +struct mem_cgroup *new_memcg, *old_memcg;struct apply_range_data data;struct page **pages =NULL ;long remaining, mapped =0 ;
@@ -555,11 +561,14 @@ static long arena_alloc_pages(struct bpf_arena *arena, long uaddr, long page_cntreturn 0 ; } +bpf_map_memcg_enter (&arena->map, &old_memcg, &new_memcg);/* Cap allocation size to KMALLOC_MAX_CACHE_SIZE so kmalloc_nolock() can succeed. */ alloc_pages =min (page_cnt,KMALLOC_MAX_CACHE_SIZE /sizeof (struct page *)); - pages =kmalloc_nolock (alloc_pages *sizeof (struct page *),0 ,NUMA_NO_NODE ); -if (!pages) + pages =kmalloc_nolock (alloc_pages *sizeof (struct page *), __GFP_ACCOUNT,NUMA_NO_NODE ); +if (!pages) { +bpf_map_memcg_exit (old_memcg, new_memcg);return 0 ; + } data.pages = pages;if (raw_res_spin_lock_irqsave (&arena->spinlock, flags))
@@ -617,6 +626,7 @@ static long arena_alloc_pages(struct bpf_arena *arena, long uaddr, long page_cntflush_vmap_cache (kern_vm_start + uaddr32, mapped <<PAGE_SHIFT );raw_res_spin_unlock_irqrestore (&arena->spinlock, flags);kfree_nolock (pages); +bpf_map_memcg_exit (old_memcg, new_memcg);return clear_lo32 (arena->user_vm_start) + uaddr32; out:range_tree_set (&arena->rt, pgoff + mapped, page_cnt - mapped);
@@ -630,6 +640,7 @@ static long arena_alloc_pages(struct bpf_arena *arena, long uaddr, long page_cntraw_res_spin_unlock_irqrestore (&arena->spinlock, flags); out_free_pages:kfree_nolock (pages); +bpf_map_memcg_exit (old_memcg, new_memcg);return 0 ; }
@@ -651,6 +662,7 @@ static void zap_pages(struct bpf_arena *arena, long uaddr, long page_cnt)static void arena_free_pages (struct bpf_arena *arena,long uaddr,long page_cnt,bool sleepable) { +struct mem_cgroup *new_memcg, *old_memcg;u64 full_uaddr, uaddr_end;long kaddr, pgoff;struct page *page;
@@ -671,6 +683,7 @@ static void arena_free_pages(struct bpf_arena *arena, long uaddr, long page_cnt, page_cnt = (uaddr_end - full_uaddr) >>PAGE_SHIFT ; pgoff =compute_pgoff (arena, uaddr); +bpf_map_memcg_enter (&arena->map, &old_memcg, &new_memcg);if (!sleepable) goto defer;
@@ -709,11 +722,13 @@ static void arena_free_pages(struct bpf_arena *arena, long uaddr, long page_cnt,zap_pages (arena, full_uaddr,1 );__free_page (page); } +bpf_map_memcg_exit (old_memcg, new_memcg);return ; defer: - s =kmalloc_nolock (sizeof (struct arena_free_span ),0 ,-1 ); + s =kmalloc_nolock (sizeof (struct arena_free_span ), __GFP_ACCOUNT,-1 ); +bpf_map_memcg_exit (old_memcg, new_memcg);if (!s)/* * If allocation fails in non-sleepable context, pages are intentionally left
@@ -735,6 +750,7 @@ static void arena_free_pages(struct bpf_arena *arena, long uaddr, long page_cnt,static int arena_reserve_pages (struct bpf_arena *arena,long uaddr,u32 page_cnt) {long page_cnt_max = (arena->user_vm_end - arena->user_vm_start) >>PAGE_SHIFT ; +struct mem_cgroup *new_memcg, *old_memcg;unsigned long flags;long pgoff;int ret;
@@ -757,7 +773,9 @@ static int arena_reserve_pages(struct bpf_arena *arena, long uaddr, u32 page_cnt }/* "Allocate" the region to prevent it from being allocated. */ +bpf_map_memcg_enter (&arena->map, &old_memcg, &new_memcg); ret =range_tree_clear (&arena->rt, pgoff, page_cnt); +bpf_map_memcg_exit (old_memcg, new_memcg); out:raw_res_spin_unlock_irqrestore (&arena->spinlock, flags);return ret;
@@ -766,6 +784,7 @@ static int arena_reserve_pages(struct bpf_arena *arena, long uaddr, u32 page_cntstatic void arena_free_worker (struct work_struct *work) { struct bpf_arena *arena = container_of(work, struct bpf_arena, free_work); +struct mem_cgroup *new_memcg, *old_memcg;struct llist_node *list, *pos, *t;struct arena_free_span *s;u64 arena_vm_start, user_vm_start;
@@ -780,6 +799,8 @@ static void arena_free_worker(struct work_struct *work)return ; } +bpf_map_memcg_enter (&arena->map, &old_memcg, &new_memcg); +init_llist_head (&free_pages); arena_vm_start =bpf_arena_get_kern_vm_start (arena); user_vm_start =bpf_arena_get_user_vm_start (arena);
@@ -820,6 +841,8 @@ static void arena_free_worker(struct work_struct *work) page = llist_entry(pos, struct page, pcp_llist);__free_page (page); } + +bpf_map_memcg_exit (old_memcg, new_memcg); }static void arena_free_irq (struct irq_work *iw)
kernel/bpf/range_tree.c +3 -2
@@ -149,7 +149,8 @@ int range_tree_clear(struct range_tree *rt, u32 start, u32 len)range_it_insert (rn, rt);/* Add a range */ - new_rn =kmalloc_nolock (sizeof (struct range_node ),0 ,NUMA_NO_NODE ); + new_rn =kmalloc_nolock (sizeof (struct range_node ), __GFP_ACCOUNT, +NUMA_NO_NODE );if (!new_rn)return -ENOMEM ; new_rn->rn_start = last +1 ;
@@ -234,7 +235,7 @@ int range_tree_set(struct range_tree *rt, u32 start, u32 len) right->rn_start = start;range_it_insert (right, rt); }else { - left =kmalloc_nolock (sizeof (struct range_node ),0 ,NUMA_NO_NODE ); + left =kmalloc_nolock (sizeof (struct range_node ), __GFP_ACCOUNT,NUMA_NO_NODE );if (!left)return -ENOMEM ; left->rn_start = start;
kernel/bpf/syscall.c +0 -2
@@ -618,7 +618,6 @@ int bpf_map_alloc_pages(const struct bpf_map *map, int nid,int ret =0 ;struct mem_cgroup *memcg, *old_memcg; -bpf_map_memcg_enter (map, &old_memcg, &memcg);for (i =0 ; i < nr_pages; i++) { pg =__bpf_alloc_page (nid);
@@ -632,7 +631,6 @@ int bpf_map_alloc_pages(const struct bpf_map *map, int nid,break ; } -bpf_map_memcg_exit (old_memcg, memcg);return ret; }
Signature · 1 line
2.47.3
Re: [PATCH bpf-next v4 2/2] bpf: arena: Reintroduce memcg accounting
Quoted 14 lines · Puranjay Mohan
left->rn_start = start;kernel/bpf/syscall.c +0
-1@@ -618,7 +618,6 @@ int bpf_map_alloc_pages(const struct bpf_map *map, int nid,int ret =0 ;struct mem_cgroup *memcg, *old_memcg; -bpf_map_memcg_enter (map, &old_memcg, &memcg);for (i =0 ; i < nr_pages; i++) { pg =__bpf_alloc_page (nid);
kernel/bpf/syscall.c int bpf_map_alloc_pages(const struct bpf_map *map, int nid, · line 634
Quoted 5 lines
kernel/bpf/syscall.c +0
-1@@ -632,7 +631,6 @@ int bpf_map_alloc_pages(const struct bpf_map *map, int nid,break ; } -bpf_map_memcg_exit (old_memcg, memcg);return ret;
Sigh. See CI complains:
../kernel/bpf/syscall.c:619:21: warning: unused variable 'memcg' [-Wunused-variable]
619 | struct mem_cgroup *memcg, *old_memcg;
| ^~~~~../kernel/bpf/syscall.c:619:29: warning: unused variable 'old_memcg' [-Wunused-variable]
619 | struct mem_cgroup *memcg, *old_memcg;
| ^~~~~~~~~
2 warnings generated.
New errors added
pw-bot: cr
Re: [PATCH bpf-next v4 2/2] bpf: arena: Reintroduce memcg accounting
Quoted 35 lines · Alexei Starovoitov
Quoted 14 lines · Puranjay Mohan
left->rn_start = start;kernel/bpf/syscall.c +0
-1@@ -618,7 +618,6 @@ int bpf_map_alloc_pages(const struct bpf_map *map, int nid,int ret =0 ;struct mem_cgroup *memcg, *old_memcg; -bpf_map_memcg_enter (map, &old_memcg, &memcg);for (i =0 ; i < nr_pages; i++) { pg =__bpf_alloc_page (nid);kernel/bpf/syscall.c int bpf_map_alloc_pages(const struct bpf_map *map, int nid, · line 634
Quoted 5 lines
kernel/bpf/syscall.c +0
-1@@ -632,7 +631,6 @@ int bpf_map_alloc_pages(const struct bpf_map *map, int nid,break ; } -bpf_map_memcg_exit (old_memcg, memcg);return ret;Sigh. See CI complains:
../kernel/bpf/syscall.c:619:21: warning: unused variable 'memcg' [-Wunused-variable]
619 | structmem_cgroup*memcg,*old_memcg;
| ^~~~~../kernel/bpf/syscall.c:619:29: warning: unused variable 'old_memcg' [-Wunused-variable]
619 | structmem_cgroup*memcg,*old_memcg;
| ^~~~~~~~~
2 warnings generated.
New errors added
Quoted 1 line
pw-bot: cr
Sorry, sent v5 with fix.