Use kmalloc_nolock() universally in BPF local storage
Subsystems (6)
v2 cover Use kmalloc_nolock() universally in BPF local storage
Socket local storage did not convert to use kmalloc_nolock() since there were observable performance degredation due to kfree_nolock() hitting the slow path and the lack of kfree_rcu()-like batching freeing. Now that these concern were addressed in slub, convert all remaining local storage flavors to use kmalloc_nolock().
v1 -> v2:
- Fix build (CI, Alexei)
Amery Hung (3):
selftests/bpf: Remove kmalloc tracing from local storage create bench
bpf: Use kmalloc_nolock() universally in local storage
bpf: Remove gfp_flags plumbing from bpf_local_storage_update()
include/linux/bpf_local_storage.h | 15 +- kernel/bpf/bpf_cgrp_storage.c | 11 +- kernel/bpf/bpf_inode_storage.c | 11 +- kernel/bpf/bpf_local_storage.c | 146 +++--------------- kernel/bpf/bpf_task_storage.c | 11 +- kernel/bpf/verifier.c | 26 ---- net/core/bpf_sk_storage.c | 23 ++- .../bpf/benchs/bench_local_storage_create.c | 21 +-- .../bpf/progs/bench_local_storage_create.c | 11 -- 9 files changed, 57 insertions(+), 218 deletions(-)
Signature · 1 line
2.52.0
PATCH v2 1/3 selftests/bpf: Remove kmalloc tracing from local storage create bench
Remove the raw_tp/kmalloc BPF program and its associated reporting from the local storage create benchmark. The kmalloc count per create is not a useful metric as different code paths use different allocators (e.g. kmalloc_nolock vs kzalloc), introducing noise that makes the number hard to interpret.
Keep total_creates in the summary output as it is useful for normalizing perf statistics collected alongside the benchmark.
--- .../bpf/benchs/bench_local_storage_create.c | 21 ++++++------------- .../bpf/progs/bench_local_storage_create.c | 11 ---------- 2 files changed, 6 insertions(+), 26 deletions(-)
tools/testing/selftests/bpf/benchs/bench_local_storage_create.c +6 -15
@@ -101,11 +101,6 @@ static void setup(void) } } -if (!bpf_program__attach (skel->progs.kmalloc)) { -fprintf (stderr,"Error attaching bpf program\n" ); -exit (1 ); - } - threads =calloc (env.producer_cnt,sizeof (*threads));if (!threads) {
@@ -140,7 +135,6 @@ static void setup(void)static void measure (struct bench_res *res) { res->hits =atomic_swap (&skel->bss->create_cnts,0 ); - res->drops =atomic_swap (&skel->bss->kmalloc_cnts,0 ); }static void *sk_producer (void *input)
@@ -203,28 +197,25 @@ static void *producer(void *input)static void report_progress (int iter,struct bench_res *res,long delta_ns) { -double creates_per_sec, kmallocs_per_create; +double creates_per_sec; creates_per_sec = res->hits /1000.0 / (delta_ns /1000000000.0 ); - kmallocs_per_create = (double )res->drops / res->hits;printf ("Iter %3d (%7.3lfus): " , iter, (delta_ns -1000000000 ) /1000.0 ); -printf ("creates %8.3lfk/s (%7.3lfk/prod), " , +printf ("creates %8.3lfk/s (%7.3lfk/prod)\n" , creates_per_sec, creates_per_sec / env.producer_cnt); -printf ("%3.2lf kmallocs/create\n" , kmallocs_per_create); }static void report_final (struct bench_res res[],int res_cnt) {double creates_mean =0.0 , creates_stddev =0.0 ; -long total_creates =0 , total_kmallocs =0 ; +long total_creates =0 ;int i;for (i =0 ; i < res_cnt; i++) { creates_mean += res[i].hits /1000.0 / (0.0 + res_cnt); total_creates += res[i].hits; - total_kmallocs += res[i].drops; }if (res_cnt >1 ) {
@@ -234,9 +225,9 @@ static void report_final(struct bench_res res[], int res_cnt) (res_cnt -1.0 ); creates_stddev =sqrt (creates_stddev); } -printf ("Summary: creates %8.3lf \u00B1 %5.3lfk/s (%7.3lfk/prod), " , - creates_mean, creates_stddev, creates_mean / env.producer_cnt); -printf ("%4.2lf kmallocs/create\n" , (double )total_kmallocs / total_creates); +printf ("Summary: creates %8.3lf \u00B1 %5.3lfk/s (%7.3lfk/prod), %ld total\n" , + creates_mean, creates_stddev, creates_mean / env.producer_cnt, + total_creates);if (create_owner_errs || skel->bss->create_errs) printf("%s() errors %ld create_errs %ld\n", storage_type == BPF_MAP_TYPE_SK_STORAGE ?
tools/testing/selftests/bpf/progs/bench_local_storage_create.c +0 -11
@@ -8,7 +8,6 @@long create_errs =0 ;long create_cnts =0 ; -long kmalloc_cnts =0 ;__u32 bench_pid =0 ;struct storage {
@@ -29,16 +28,6 @@ struct { __type(value, struct storage); } task_storage_map SEC(".maps"); -SEC("raw_tp/kmalloc") -int BPF_PROG (kmalloc ,unsigned long call_site,const void *ptr, -size_t bytes_req,size_t bytes_alloc,gfp_t gfp_flags, -int node) -{ -__sync_fetch_and_add (&kmalloc_cnts,1 ); - -return 0 ; -} - SEC("tp_btf/sched_process_fork")int BPF_PROG (sched_process_fork ,struct task_struct *parent,struct task_struct *child) {
Signature · 1 line
2.52.0
PATCH v2 2/3 bpf: Use kmalloc_nolock() universally in local storage
In reply to Amery HungSwitch to kmalloc_nolock() universally in local storage. Socket local storage didn't move to kmalloc_nolock() when BPF memory allocator was replaced by it for performance reasons. Now that kfree_rcu() supports freeing memory allocated by kmalloc_nolock(), we can move the remaining local storages to use kmalloc_nolock() and cleanup the cluttered free paths.
Use kfree() instead of kfree_nolock() in bpf_selem_free_trace_rcu() and bpf_local_storage_free_trace_rcu(). Both callbacks run in process context where spinning is allowed, so kfree_nolock() is unnecessary.
Benchmark:
./bench -p 1 local-storage-create --storage-type socket \
--batch-size {16,32,64}
The benchmark is a microbenchmark stress-testing how fast local storage can be created. There is no measurable throughput change for socket local storage after switching from kzalloc() to kmalloc_nolock().
Socket local storage
batch creation speed diff
--------------- ---- ------------------ ----
Baseline 16 433.9 ± 0.6 k/s
32 434.3 ± 1.4 k/s
64 434.2 ± 0.7 k/sAfter 16 439.0 ± 1.9 k/s +1.2%
32 437.3 ± 2.0 k/s +0.7%
64 435.8 ± 2.5k/s +0.4%Also worth noting that the baseline got a 5% throughput boost when sheaf replaces percpu partial slab recently [0].
[0] https://lore.kernel.org/bpf/[email protected]/
--- include/linux/bpf_local_storage.h | 8 +- kernel/bpf/bpf_cgrp_storage.c | 2 +- kernel/bpf/bpf_inode_storage.c | 2 +- kernel/bpf/bpf_local_storage.c | 130 ++++-------------------------- kernel/bpf/bpf_task_storage.c | 2 +- net/core/bpf_sk_storage.c | 2 +- 6 files changed, 21 insertions(+), 125 deletions(-)
include/linux/bpf_local_storage.h +2 -6
@@ -54,7 +54,6 @@ struct bpf_local_storage_map {u32 bucket_log;u16 elem_size;u16 cache_idx; -bool use_kmalloc_nolock; };struct bpf_local_storage_data {
@@ -86,8 +85,7 @@ struct bpf_local_storage_elem {*/ };atomic_t state; -bool use_kmalloc_nolock; -/* 3 bytes hole */ +/* 4 bytes hole */ /* The data is stored in another cacheline to minimize * the number of cachelines access during a cache hit. */
@@ -104,7 +102,6 @@ struct bpf_local_storage {rqspinlock_t lock;/* Protect adding/removing from the "list" */ u64 mem_charge;/* Copy of mem charged to owner. Protected by "lock" */ refcount_t owner_refcnt;/* Used to pin owner when map_free is uncharging */ -bool use_kmalloc_nolock; };/* U16_MAX is much more than enough for sk local storage
@@ -137,8 +134,7 @@ int bpf_local_storage_map_alloc_check(union bpf_attr *attr);struct bpf_map *bpf_local_storage_map_alloc (union bpf_attr *attr, -struct bpf_local_storage_cache *cache, -bool use_kmalloc_nolock); +struct bpf_local_storage_cache *cache); void __bpf_local_storage_insert_cache(struct bpf_local_storage *local_storage, struct bpf_local_storage_map *smap,
kernel/bpf/bpf_cgrp_storage.c +1 -1
@@ -114,7 +114,7 @@ static int notsupp_get_next_key(struct bpf_map *map, void *key, void *next_key)static struct bpf_map *cgroup_storage_map_alloc (union bpf_attr *attr) { -return bpf_local_storage_map_alloc (attr, &cgroup_cache, true); +return bpf_local_storage_map_alloc (attr, &cgroup_cache); }static void cgroup_storage_map_free (struct bpf_map *map)
kernel/bpf/bpf_inode_storage.c +1 -1
@@ -179,7 +179,7 @@ static int notsupp_get_next_key(struct bpf_map *map, void *key,static struct bpf_map *inode_storage_map_alloc (union bpf_attr *attr) { -return bpf_local_storage_map_alloc (attr, &inode_cache, false); +return bpf_local_storage_map_alloc (attr, &inode_cache); }static void inode_storage_map_free (struct bpf_map *map)
kernel/bpf/bpf_local_storage.c +15 -115
206 lines not on this page · raw message
kernel/bpf/bpf_task_storage.c +1 -1
@@ -212,7 +212,7 @@ static int notsupp_get_next_key(struct bpf_map *map, void *key, void *next_key)static struct bpf_map *task_storage_map_alloc (union bpf_attr *attr) { -return bpf_local_storage_map_alloc (attr, &task_cache, true); +return bpf_local_storage_map_alloc (attr, &task_cache); }static void task_storage_map_free (struct bpf_map *map)
net/core/bpf_sk_storage.c +1 -1
@@ -68,7 +68,7 @@ static void bpf_sk_storage_map_free(struct bpf_map *map)static struct bpf_map *bpf_sk_storage_map_alloc (union bpf_attr *attr) { -return bpf_local_storage_map_alloc (attr, &sk_cache, false); +return bpf_local_storage_map_alloc (attr, &sk_cache); } static int notsupp_get_next_key(struct bpf_map *map, void *key,
Signature · 1 line
2.52.0
[email protected]
AI reviewRe: [PATCH bpf-next v2 2/3] bpf: Use kmalloc_nolock() universally in local storage
With this change, socket and inode local storage now always go through kmalloc_nolock(). kmalloc_nolock_noprof() in mm/slub.c has a hard size limit:
Quoted 4 lines
kernel/bpf/bpf_local_storage.c +0
-0
[ ... ]
kernel/bpf/bpf_local_storage.c bpf_selem_alloc(struct bpf_local_storage_map *smap, void *owner, · line 79
Quoted 13 lines
kernel/bpf/bpf_local_storage.c +2
-7@@ -75,18 +75,12 @@ bpf_selem_alloc(struct bpf_local_storage_map *smap, void *owner,if (mem_charge (smap, owner, smap->elem_size))return NULL ; -if (smap->use_kmalloc_nolock) { - selem =bpf_map_kmalloc_nolock (&smap->map, smap->elem_size, - __GFP_ZERO,NUMA_NO_NODE ); - }else { - selem =bpf_map_kzalloc (&smap->map, smap->elem_size, - gfp_flags | __GFP_NOWARN); - } + selem =bpf_map_kmalloc_nolock (&smap->map, smap->elem_size, + __GFP_ZERO,NUMA_NO_NODE );
With this change, socket and inode local storage now always go through kmalloc_nolock(). kmalloc_nolock_noprof() in mm/slub.c has a hard size limit:
if (unlikely(size > KMALLOC_MAX_CACHE_SIZE))
return NULL;
KMALLOC_MAX_CACHE_SIZE is 1 << (PAGE_SHIFT + 1), which is 8KB on x86_64 with 4KB pages. But BPF_LOCAL_STORAGE_MAX_VALUE_SIZE in bpf_local_storage.h is calculated from KMALLOC_MAX_SIZE (not KMALLOC_MAX_CACHE_SIZE):
#define BPF_LOCAL_STORAGE_MAX_VALUE_SIZE \
min_t(u32, \
(KMALLOC_MAX_SIZE - MAX_BPF_STACK - \
sizeof(struct bpf_local_storage_elem)), \
(U16_MAX - sizeof(struct bpf_local_storage_elem)))This allows value sizes up to ~65KB. Before this patch, socket and inode storage used bpf_map_kzalloc() (backed by regular kmalloc) which could handle those large sizes. After this patch, any elem_size above KMALLOC_MAX_CACHE_SIZE will silently fail: the map creation succeeds via bpf_local_storage_map_alloc_check() but every element allocation returns NULL.
Should BPF_LOCAL_STORAGE_MAX_VALUE_SIZE be updated to use KMALLOC_MAX_CACHE_SIZE instead of KMALLOC_MAX_SIZE now that all storage types go through kmalloc_nolock()?
Slava Imameev raised the same concern for task storage in
https://lore.kernel.org/bpf/[email protected]/
[ ... ]
---
AI reviewed your patch. Please fix the bug or email reply why it's not a bug.
See: https://github.com/kernel-patches/vmtest/blob/master/ci/claude/README.md
CI run summary: https://github.com/kernel-patches/bpf/actions/runs/24272235145
Re: [PATCH bpf-next v2 2/3] bpf: Use kmalloc_nolock() universally in local storage
Quoted 9 lines · On Fri, Apr 10, 2026 at 7:36 PM <[email protected]> wrote:
This allows value sizes up to ~65KB. Before this patch, socket and inode storage used
bpf_map_kzalloc()(backed by regular kmalloc) which could handle those large sizes. After this patch, anyelem_sizeaboveKMALLOC_MAX_CACHE_SIZEwill silently fail: the map creation succeeds viabpf_local_storage_map_alloc_check()but every element allocation returns NULL.
Quoted 6 lines
Should
BPF_LOCAL_STORAGE_MAX_VALUE_SIZEbe updated to useKMALLOC_MAX_CACHE_SIZEinstead ofKMALLOC_MAX_SIZEnow that all storage types go throughkmalloc_nolock()?Slava Imameev raised the same concern for task storage in
https://lore.kernel.org/bpf/[email protected]/
Right. Let's update it, but I don't think it's a regression. On a loaded system kmalloc_large() rarely succeeds for order 2+. That's why kmalloc_nolock() doesn't attempt to bridge that gap. One or two contiguous physical pages is the best one can expect. In early bpf days we picked KMALLOC_MAX_SIZE assuming that it's a realistic max for kmalloc().
It turned out to be wishful thinking.kmalloc_large concept should really be removed.
It deceives users into thinking that it's usable.
Re: [PATCH bpf-next v2 2/3] bpf: Use kmalloc_nolock() universally in local storage
Quoted 16 lines · On Fri, 10 Apr 2026 21:39:00 -0700 Alexei Starovoitov wrote:
Quoted 9 lines
This allows value sizes up to ~65KB. Before this patch, socket and inode storage used
bpf_map_kzalloc()(backed by regular kmalloc) which could handle those large sizes. After this patch, anyelem_sizeaboveKMALLOC_MAX_CACHE_SIZEwill silently fail: the map creation succeeds viabpf_local_storage_map_alloc_check()but every element allocation returns NULL.Quoted 6 lines
Should
BPF_LOCAL_STORAGE_MAX_VALUE_SIZEbe updated to useKMALLOC_MAX_CACHE_SIZEinstead ofKMALLOC_MAX_SIZEnow that all storage types go throughkmalloc_nolock()?Slava Imameev raised the same concern for task storage in
https://lore.kernel.org/bpf/[email protected]/
Quoted 9 lines
Right. Let's update it, but I don't think it's a regression. On a loaded system
kmalloc_large()rarely succeeds for order 2+. That's whykmalloc_nolock()doesn't attempt to bridge that gap. One or two contiguous physical pages is the best one can expect. In early bpf days we pickedKMALLOC_MAX_SIZEassuming that it's a realistic max forkmalloc().
It turned out to be wishful thinking.kmalloc_largeconcept should really be removed.
It deceives users into thinking that it's usable.
Do you think it would be viable to extend task storage to support larger allocations, to restore support for 64KB or maybe less value like 32 KB, using vmalloc or bpf_mem_cache_alloc, with the obvious restrictions that vmalloc imposes? Perhaps we could use bpf_mem_cache_alloc as the primary mechanism with vmalloc as a fallback when the caller context permits?
We've found task storage allocations larger than 8KB quite valuable for scenarios involving processing multiple file paths. Currently, without large task storage support, we're forced to preallocate maps with 12KB+ values and significantly over-provision the number of entries to reduce the probability of free entry depletion. This approach places unnecessary burden on the memory subsystem since much of this pre-allocated memory remains unused.
Even if task storage allocation fails due to lack of contiguous physical memory and vmalloc is not possible, there's an option to maintain an emergency preallocated map of much smaller size compared to when this map serves as the primary mechanism.
With larger task storage allocations, we've implemented a simple memory allocator that operates over task storage. For example, a 16KB task storage can accommodate multiple allocations, one big and couple of small, which has substantially reduced our memory footprint compared to the current map-based approach. We've also experimented successfully with 32KB arenas for workloads requiring even larger working sets.
Re: [PATCH bpf-next v2 2/3] bpf: Use kmalloc_nolock() universally in local storage
In reply to Alexei StarovoitovQuoted 16 lines · On Fri, 10 Apr 2026 21:39:00 -0700 Alexei Starovoitov wrote:
Quoted 9 lines
This allows value sizes up to ~65KB. Before this patch, socket and inode storage used
bpf_map_kzalloc()(backed by regular kmalloc) which could handle those large sizes. After this patch, anyelem_sizeaboveKMALLOC_MAX_CACHE_SIZEwill silently fail: the map creation succeeds viabpf_local_storage_map_alloc_check()but every element allocation returns NULL.Quoted 6 lines
Should
BPF_LOCAL_STORAGE_MAX_VALUE_SIZEbe updated to useKMALLOC_MAX_CACHE_SIZEinstead ofKMALLOC_MAX_SIZEnow that all storage types go throughkmalloc_nolock()?Slava Imameev raised the same concern for task storage in
https://lore.kernel.org/bpf/[email protected]/
Quoted 9 lines
Right. Let's update it, but I don't think it's a regression. On a loaded system
kmalloc_large()rarely succeeds for order 2+. That's whykmalloc_nolock()doesn't attempt to bridge that gap. One or two contiguous physical pages is the best one can expect. In early bpf days we pickedKMALLOC_MAX_SIZEassuming that it's a realistic max forkmalloc().
It turned out to be wishful thinking.kmalloc_largeconcept should really be removed.
It deceives users into thinking that it's usable.
In defense of supporting 8KB-64KB allocations for local storage, we can consider BPF_MAP_TYPE_HASH with BPF_F_NO_PREALLOC as providing similar functionality to replace the missing 8KB-64KB local storage allocation support. However, these map entry allocations can also fail with similar probability since they depend on the same underlying allocator.
Re: [PATCH bpf-next v2 2/3] bpf: Use kmalloc_nolock() universally in local storage
Quoted 27 lines · Slava Imameev
Quoted 16 lines · On Fri, 10 Apr 2026 21:39:00 -0700 Alexei Starovoitov wrote:
Quoted 9 lines
This allows value sizes up to ~65KB. Before this patch, socket and inode storage used
bpf_map_kzalloc()(backed by regular kmalloc) which could handle those large sizes. After this patch, anyelem_sizeaboveKMALLOC_MAX_CACHE_SIZEwill silently fail: the map creation succeeds viabpf_local_storage_map_alloc_check()but every element allocation returns NULL.Quoted 6 lines
Should
BPF_LOCAL_STORAGE_MAX_VALUE_SIZEbe updated to useKMALLOC_MAX_CACHE_SIZEinstead ofKMALLOC_MAX_SIZEnow that all storage types go throughkmalloc_nolock()?Slava Imameev raised the same concern for task storage in
https://lore.kernel.org/bpf/[email protected]/Quoted 9 lines
Right. Let's update it, but I don't think it's a regression. On a loaded system
kmalloc_large()rarely succeeds for order 2+. That's whykmalloc_nolock()doesn't attempt to bridge that gap. One or two contiguous physical pages is the best one can expect. In early bpf days we pickedKMALLOC_MAX_SIZEassuming that it's a realistic max forkmalloc().
It turned out to be wishful thinking.kmalloc_largeconcept should really be removed.
It deceives users into thinking that it's usable.
Quoted 6 lines
In defense of supporting 8KB-64KB allocations for local storage, we can consider
BPF_MAP_TYPE_HASHwithBPF_F_NO_PREALLOCas providing similar functionality to replace the missing 8KB-64KB local storage allocation support. However, these map entry allocations can also fail with similar probability since they depend on the same underlying allocator.
I really hope that 64kb task local storage is not your production code. Severs easily have 50k threads. Sometimes more.
64k * 50k = 3 Gbytes of memory wasted.
You need to redesign it from ground up.
Re: [PATCH bpf-next v2 2/3] bpf: Use kmalloc_nolock() universally in local storage
Quoted 35 lines · On Tue, 14 Apr 2026 19:27:00 -0700 Alexei Starovoitov wrote:
Quoted 27 lines · Slava Imameev
Quoted 16 lines · On Fri, 10 Apr 2026 21:39:00 -0700 Alexei Starovoitov wrote:
Quoted 9 lines
This allows value sizes up to ~65KB. Before this patch, socket and inode storage used
bpf_map_kzalloc()(backed by regular kmalloc) which could handle those large sizes. After this patch, anyelem_sizeaboveKMALLOC_MAX_CACHE_SIZEwill silently fail: the map creation succeeds viabpf_local_storage_map_alloc_check()but every element allocation returns NULL.Quoted 6 lines
Should
BPF_LOCAL_STORAGE_MAX_VALUE_SIZEbe updated to useKMALLOC_MAX_CACHE_SIZEinstead ofKMALLOC_MAX_SIZEnow that all storage types go throughkmalloc_nolock()?Slava Imameev raised the same concern for task storage in
https://urldefense.com/v3/__https://lore.kernel.org/bpf/[email protected]/__;!!BmdzS3_lV9HdKG8!ytFHcGR6fq4YVQZ74Z_LwJ5IKsEaF2vnY03x8-IS51cQyN3SkHYa-6G_vUxk2lW7xvWMNEfSArwyIGXuxeEhe62whEC8AyDpmA$Quoted 9 lines
Right. Let's update it, but I don't think it's a regression. On a loaded system
kmalloc_large()rarely succeeds for order 2+. That's whykmalloc_nolock()doesn't attempt to bridge that gap. One or two contiguous physical pages is the best one can expect. In early bpf days we pickedKMALLOC_MAX_SIZEassuming that it's a realistic max forkmalloc().
It turned out to be wishful thinking.kmalloc_largeconcept should really be removed.
It deceives users into thinking that it's usable.Quoted 6 lines
In defense of supporting 8KB-64KB allocations for local storage, we can consider
BPF_MAP_TYPE_HASHwithBPF_F_NO_PREALLOCas providing similar functionality to replace the missing 8KB-64KB local storage allocation support. However, these map entry allocations can also fail with similar probability since they depend on the same underlying allocator.
Quoted 4 lines
I really hope that 64kb task local storage is not your production code. Severs easily have 50k threads. Sometimes more.
64k * 50k = 3 Gbytes of memory wasted.
You need to redesign it from ground up.
This was a research project to replace LRU maps with task storage. We implemented a garbage collector using a BPF task iterator to release inactive task allocations. While iterating over tens of thousands of tasks might be questionable, this was a proof of concept that, when combined with other measures, could potentially keep memory pressure in the tens of MBs.
8KB would be sufficient for 99.9% of our allocations, but sometimes we need 12KB or more. The alternative to task storage could be BPF_MAP_TYPE_HASH with BPF_F_NO_PREALLOC and a garbage collector, as we want to reduce dependency on preallocated LRU maps.
PATCH v2 3/3 bpf: Remove gfp_flags plumbing from bpf_local_storage_update()
In reply to Amery HungRemove the check that rejects sleepable BPF programs from doing BPF_ANY/BPF_EXIST updates on local storage. This restriction was added in commit b00fa38a9c1c ("bpf: Enable non-atomic allocations in local storage") because kzalloc(GFP_KERNEL) could sleep inside local_storage->lock. This is no longer a concern: all local storage allocations now use kmalloc_nolock() which never sleeps.
In addition, since kmalloc_nolock() only accepts __GFP_ACCOUNT, __GFP_ZERO and __GFP_NO_OBJ_EXT, the gfp_flags parameter plumbing from bpf_*_storage_get() to bpf_local_storage_update() becomes dead code. Remove gfp_flags from bpf_selem_alloc(), bpf_local_storage_alloc() and bpf_local_storage_update(). Drop the hidden 5th argument from bpf_*_storage_get helpers, and remove the verifier patching that injected GFP_KERNEL/GFP_ATOMIC into the fifth argument.
--- include/linux/bpf_local_storage.h | 7 +++---- kernel/bpf/bpf_cgrp_storage.c | 9 ++++----- kernel/bpf/bpf_inode_storage.c | 9 ++++----- kernel/bpf/bpf_local_storage.c | 16 ++++++---------- kernel/bpf/bpf_task_storage.c | 9 ++++----- kernel/bpf/verifier.c | 26 -------------------------- net/core/bpf_sk_storage.c | 21 +++++++++------------ 7 files changed, 30 insertions(+), 67 deletions(-)
include/linux/bpf_local_storage.h +3 -4
@@ -188,7 +188,7 @@ int bpf_selem_link_map(struct bpf_local_storage_map *smap,struct bpf_local_storage_elem *bpf_selem_alloc (struct bpf_local_storage_map *smap,void *owner,void *value, -bool swap_uptrs,gfp_t gfp_flags); +bool swap_uptrs);void bpf_selem_free (struct bpf_local_storage_elem *selem,bool reuse_now);
@@ -196,12 +196,11 @@ void bpf_selem_free(struct bpf_local_storage_elem *selem,int bpf_local_storage_alloc (void *owner,struct bpf_local_storage_map *smap, -struct bpf_local_storage_elem *first_selem, -gfp_t gfp_flags); +struct bpf_local_storage_elem *first_selem);struct bpf_local_storage_data *bpf_local_storage_update (void *owner,struct bpf_local_storage_map *smap, -void *value,u64 map_flags,bool swap_uptrs,gfp_t gfp_flags); +void *value,u64 map_flags,bool swap_uptrs);u64 bpf_local_storage_map_mem_usage (const struct bpf_map *map);
kernel/bpf/bpf_cgrp_storage.c +4 -5
@@ -76,7 +76,7 @@ static long bpf_cgrp_storage_update_elem(struct bpf_map *map, void *key,return PTR_ERR (cgroup); sdata =bpf_local_storage_update (cgroup, (struct bpf_local_storage_map *)map, - value, map_flags, false,GFP_ATOMIC ); + value, map_flags, false);cgroup_put (cgroup);return PTR_ERR_OR_ZERO (sdata); }
@@ -122,9 +122,8 @@ static void cgroup_storage_map_free(struct bpf_map *map)bpf_local_storage_map_free (map, &cgroup_cache); } -/* *gfp_flags* is a hidden argument provided by the verifier */ -BPF_CALL_5(bpf_cgrp_storage_get, struct bpf_map *, map, struct cgroup *, cgroup, - void *, value, u64, flags, gfp_t, gfp_flags) +BPF_CALL_4(bpf_cgrp_storage_get, struct bpf_map *, map, struct cgroup *, cgroup, + void *, value, u64, flags) {struct bpf_local_storage_data *sdata;
@@ -143,7 +142,7 @@ BPF_CALL_5(bpf_cgrp_storage_get, struct bpf_map *, map, struct cgroup *, cgroup,if (!percpu_ref_is_dying (&cgroup->self.refcnt) && (flags &BPF_LOCAL_STORAGE_GET_F_CREATE )) sdata =bpf_local_storage_update (cgroup, (struct bpf_local_storage_map *)map, - value,BPF_NOEXIST , false, gfp_flags); + value,BPF_NOEXIST , false); out:return IS_ERR_OR_NULL (sdata) ? (unsigned long )NULL : (unsigned long )sdata->data;
kernel/bpf/bpf_inode_storage.c +4 -5
@@ -98,7 +98,7 @@ static long bpf_fd_inode_storage_update_elem(struct bpf_map *map, void *key, sdata =bpf_local_storage_update (file_inode (fd_file (f)), (struct bpf_local_storage_map *)map, - value, map_flags, false,GFP_ATOMIC ); + value, map_flags, false);return PTR_ERR_OR_ZERO (sdata); }
@@ -122,9 +122,8 @@ static long bpf_fd_inode_storage_delete_elem(struct bpf_map *map, void *key)return inode_storage_delete (file_inode (fd_file (f)), map); } -/* *gfp_flags* is a hidden argument provided by the verifier */ -BPF_CALL_5(bpf_inode_storage_get, struct bpf_map *, map, struct inode *, inode, - void *, value, u64, flags, gfp_t, gfp_flags) +BPF_CALL_4(bpf_inode_storage_get, struct bpf_map *, map, struct inode *, inode, + void *, value, u64, flags) {struct bpf_local_storage_data *sdata;
@@ -150,7 +149,7 @@ BPF_CALL_5(bpf_inode_storage_get, struct bpf_map *, map, struct inode *, inode,if (flags &BPF_LOCAL_STORAGE_GET_F_CREATE ) { sdata =bpf_local_storage_update ( inode, (struct bpf_local_storage_map *)map, value, -BPF_NOEXIST , false, gfp_flags); +BPF_NOEXIST , false);return IS_ERR (sdata) ? (unsigned long )NULL : (unsigned long )sdata->data; }
kernel/bpf/bpf_local_storage.c +6 -10
@@ -68,7 +68,7 @@ static bool selem_linked_to_map(const struct bpf_local_storage_elem *selem)struct bpf_local_storage_elem *bpf_selem_alloc (struct bpf_local_storage_map *smap,void *owner, -void *value,bool swap_uptrs,gfp_t gfp_flags) +void *value,bool swap_uptrs) {struct bpf_local_storage_elem *selem;
@@ -475,8 +475,7 @@ static int check_flags(const struct bpf_local_storage_data *old_sdata,int bpf_local_storage_alloc (void *owner,struct bpf_local_storage_map *smap, -struct bpf_local_storage_elem *first_selem, -gfp_t gfp_flags) +struct bpf_local_storage_elem *first_selem) {struct bpf_local_storage *prev_storage, *storage;struct bpf_local_storage **owner_storage_ptr;
@@ -546,7 +545,7 @@ int bpf_local_storage_alloc(void *owner,*/ struct bpf_local_storage_data *bpf_local_storage_update (void *owner,struct bpf_local_storage_map *smap, -void *value,u64 map_flags,bool swap_uptrs,gfp_t gfp_flags) +void *value,u64 map_flags,bool swap_uptrs) {struct bpf_local_storage_data *old_sdata =NULL ;struct bpf_local_storage_elem *alloc_selem, *selem =NULL ;
@@ -563,9 +562,6 @@ bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap, !btf_record_has_field(smap->map.record, BPF_SPIN_LOCK)))return ERR_PTR (-EINVAL ); -if (gfp_flags ==GFP_KERNEL && (map_flags & ~BPF_F_LOCK ) !=BPF_NOEXIST ) -return ERR_PTR (-EINVAL ); - local_storage =rcu_dereference_check (*owner_storage (smap, owner),bpf_rcu_lock_held ());if (!local_storage ||hlist_empty (&local_storage->list)) {
@@ -574,11 +570,11 @@ bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap,if (err)return ERR_PTR (err); - selem =bpf_selem_alloc (smap, owner, value, swap_uptrs, gfp_flags); + selem =bpf_selem_alloc (smap, owner, value, swap_uptrs);if (!selem)return ERR_PTR (-ENOMEM ); - err =bpf_local_storage_alloc (owner, smap, selem, gfp_flags); + err =bpf_local_storage_alloc (owner, smap, selem);if (err) {bpf_selem_free (selem, true);mem_uncharge (smap, owner, smap->elem_size);
@@ -608,7 +604,7 @@ bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap,/* A lookup has just been done before and concluded a new selem is * needed. The chance of an unnecessary alloc is unlikely. */ - alloc_selem = selem =bpf_selem_alloc (smap, owner, value, swap_uptrs, gfp_flags); + alloc_selem = selem =bpf_selem_alloc (smap, owner, value, swap_uptrs);if (!alloc_selem)return ERR_PTR (-ENOMEM );
kernel/bpf/bpf_task_storage.c +4 -5
@@ -118,7 +118,7 @@ static long bpf_pid_task_storage_update_elem(struct bpf_map *map, void *key, sdata =bpf_local_storage_update ( task, (struct bpf_local_storage_map *)map, value, map_flags, - true,GFP_ATOMIC ); + true); err =PTR_ERR_OR_ZERO (sdata); out:
@@ -165,9 +165,8 @@ static long bpf_pid_task_storage_delete_elem(struct bpf_map *map, void *key)return err; } -/* *gfp_flags* is a hidden argument provided by the verifier */ -BPF_CALL_5(bpf_task_storage_get, struct bpf_map *, map, struct task_struct *, - task, void *, value, u64, flags, gfp_t, gfp_flags) +BPF_CALL_4(bpf_task_storage_get, struct bpf_map *, map, struct task_struct *, + task, void *, value, u64, flags) {struct bpf_local_storage_data *sdata;
@@ -184,7 +183,7 @@ BPF_CALL_5(bpf_task_storage_get, struct bpf_map *, map, struct task_struct *, (flags & BPF_LOCAL_STORAGE_GET_F_CREATE)) { sdata =bpf_local_storage_update ( task, (struct bpf_local_storage_map *)map, value, -BPF_NOEXIST , false, gfp_flags); +BPF_NOEXIST , false);return IS_ERR (sdata) ? (unsigned long )NULL : (unsigned long )sdata->data; }
kernel/bpf/verifier.c +0 -26
@@ -589,14 +589,6 @@ static bool is_may_goto_insn_at(struct bpf_verifier_env *env, int insn_idx)return bpf_is_may_goto_insn (&env->prog->insnsi[insn_idx]); } -static bool is_storage_get_function (enum bpf_func_id func_id) -{ -return func_id == BPF_FUNC_sk_storage_get || - func_id == BPF_FUNC_inode_storage_get || - func_id == BPF_FUNC_task_storage_get || - func_id == BPF_FUNC_cgrp_storage_get; -} -static bool helper_multiple_ref_obj_use (enum bpf_func_id func_id,const struct bpf_map *map) {
@@ -24414,24 +24406,6 @@ static int do_misc_fixups(struct bpf_verifier_env *env) goto patch_call_imm; } -if (is_storage_get_function (insn->imm)) { -if (env->insn_aux_data[i + delta].non_sleepable) - insn_buf[0 ] =BPF_MOV64_IMM (BPF_REG_5 , (__force __s32 )GFP_ATOMIC ); -else - insn_buf[0 ] =BPF_MOV64_IMM (BPF_REG_5 , (__force __s32 )GFP_KERNEL ); - insn_buf[1 ] = *insn; - cnt =2 ; - - new_prog =bpf_patch_insn_data (env, i + delta, insn_buf, cnt); -if (!new_prog) -return -ENOMEM ; - - delta += cnt -1 ; - env->prog = prog = new_prog; - insn = new_prog->insnsi + i + delta; - goto patch_call_imm; - } -/* bpf_per_cpu_ptr() and bpf_this_cpu_ptr() */ if (env->insn_aux_data[i + delta].call_with_percpu_alloc_ptr) {/* patch with 'r1 = *(u64 *)(r1 + 0)' since for percpu data,
net/core/bpf_sk_storage.c +9 -12
@@ -106,7 +106,7 @@ static long bpf_fd_sk_storage_update_elem(struct bpf_map *map, void *key,if (sock) { sdata =bpf_local_storage_update ( sock->sk, (struct bpf_local_storage_map *)map, value, - map_flags, false,GFP_ATOMIC ); + map_flags, false);sockfd_put (sock);return PTR_ERR_OR_ZERO (sdata); }
@@ -137,7 +137,7 @@ bpf_sk_storage_clone_elem(struct sock *newsk, {struct bpf_local_storage_elem *copy_selem; - copy_selem =bpf_selem_alloc (smap, newsk,NULL , false,GFP_ATOMIC ); + copy_selem =bpf_selem_alloc (smap, newsk,NULL , false);if (!copy_selem)return NULL ;
@@ -202,7 +202,7 @@ int bpf_sk_storage_clone(const struct sock *sk, struct sock *newsk) }bpf_selem_link_storage_nolock (new_sk_storage, copy_selem); }else { - ret =bpf_local_storage_alloc (newsk, smap, copy_selem,GFP_ATOMIC ); + ret =bpf_local_storage_alloc (newsk, smap, copy_selem);if (ret) {bpf_selem_free (copy_selem, true); atomic_sub(smap->elem_size,
@@ -227,9 +227,8 @@ int bpf_sk_storage_clone(const struct sock *sk, struct sock *newsk)return ret; } -/* *gfp_flags* is a hidden argument provided by the verifier */ -BPF_CALL_5(bpf_sk_storage_get, struct bpf_map *, map, struct sock *, sk, - void *, value, u64, flags, gfp_t, gfp_flags) +BPF_CALL_4(bpf_sk_storage_get, struct bpf_map *, map, struct sock *, sk, + void *, value, u64, flags) {struct bpf_local_storage_data *sdata;
@@ -250,7 +249,7 @@ BPF_CALL_5(bpf_sk_storage_get, struct bpf_map *, map, struct sock *, sk, refcount_inc_not_zero(&sk->sk_refcnt)) { sdata =bpf_local_storage_update ( sk, (struct bpf_local_storage_map *)map, value, -BPF_NOEXIST , false, gfp_flags); +BPF_NOEXIST , false);/* sk must be a fullsock (guaranteed by verifier), * so sock_gen_put() is unnecessary. */
@@ -383,16 +382,14 @@ static bool bpf_sk_storage_tracing_allowed(const struct bpf_prog *prog)return false; } -/* *gfp_flags* is a hidden argument provided by the verifier */ -BPF_CALL_5(bpf_sk_storage_get_tracing, struct bpf_map *, map, struct sock *, sk, - void *, value, u64, flags, gfp_t, gfp_flags) +BPF_CALL_4(bpf_sk_storage_get_tracing, struct bpf_map *, map, struct sock *, sk, + void *, value, u64, flags) {WARN_ON_ONCE (!bpf_rcu_lock_held ());if (in_hardirq () ||in_nmi ())return (unsigned long )NULL ; -return (unsigned long )____bpf_sk_storage_get (map, sk, value, flags, - gfp_flags); +return (unsigned long )____bpf_sk_storage_get (map, sk, value, flags); } BPF_CALL_2(bpf_sk_storage_delete_tracing, struct bpf_map *, map,
Signature · 1 line
2.52.0
This series was applied to bpf/bpf-next.git (master) by Alexei Starovoitov <[email protected]>:
Hello:
This series was applied to bpf/bpf-next.git (master) by Alexei Starovoitov <[email protected]>:
Quoted 10 lines · On Fri, 10 Apr 2026 18:54:15 -0700 you wrote:
Socket local storage did not convert to use
kmalloc_nolock()since there were observable performance degredation due tokfree_nolock()hitting the slow path and the lack of kfree_rcu()-like batching freeing. Now that these concern were addressed in slub, convert all remaining local storage flavors to usekmalloc_nolock().v1 -> v2:
- Fix build (CI, Alexei)[...]
Here is the summary with links:
- [bpf-next,v2,1/3] selftests/bpf: Remove kmalloc tracing from local storage create bench
https://git.kernel.org/bpf/bpf-next/c/78ee02a966ad - [bpf-next,v2,2/3] bpf: Use kmalloc_nolock() universally in local storage https://git.kernel.org/bpf/bpf-next/c/5063e7758899 - [bpf-next,v2,3/3] bpf: Remove gfp_flags plumbing from bpf_local_storage_update() https://git.kernel.org/bpf/bpf-next/c/136deea435dc
You are awesome, thank you!
Signature · 2 lines
Deet-doot-dot, I am a bot. https://korg.docs.kernel.org/patchwork/pwbot.html