bpf: Fix usage of trace RCU in local storage.
Subsystems (3)
PATCH v2 1/1 bpf: Fix usage of trace RCU in local storage.
bpf_{sk,task,inode}_storage_free() do not need to use call_rcu_tasks_trace as no BPF program should be accessing the owner as it's being destroyed. The only other reader at this point is bpf_local_storage_map_free() which uses normal RCU.
The only path that needs trace RCU are:
* bpf_local_storage_{delete,update} helpers
* map_{delete,update}_elem() syscalls
--- include/linux/bpf_local_storage.h | 4 ++-- kernel/bpf/bpf_inode_storage.c | 4 ++-- kernel/bpf/bpf_local_storage.c | 29 +++++++++++++++++++---------- kernel/bpf/bpf_task_storage.c | 4 ++-- net/core/bpf_sk_storage.c | 6 +++--- 5 files changed, 28 insertions(+), 19 deletions(-)
include/linux/bpf_local_storage.h +2 -2
@@ -143,9 +143,9 @@ void bpf_selem_link_storage_nolock(struct bpf_local_storage *local_storage,bool bpf_selem_unlink_storage_nolock (struct bpf_local_storage *local_storage,struct bpf_local_storage_elem *selem, -bool uncharge_omem); +bool uncharge_omem,bool use_trace_rcu); -void bpf_selem_unlink (struct bpf_local_storage_elem *selem); +void bpf_selem_unlink (struct bpf_local_storage_elem *selem,bool use_trace_rcu);void bpf_selem_link_map (struct bpf_local_storage_map *smap,struct bpf_local_storage_elem *selem);
kernel/bpf/bpf_inode_storage.c +2 -2
@@ -90,7 +90,7 @@ void bpf_inode_storage_free(struct inode *inode)*/ bpf_selem_unlink_map (selem); free_inode_storage =bpf_selem_unlink_storage_nolock ( - local_storage, selem, false); + local_storage, selem, false, false); }raw_spin_unlock_bh (&local_storage->lock);rcu_read_unlock ();
@@ -149,7 +149,7 @@ static int inode_storage_delete(struct inode *inode, struct bpf_map *map)if (!sdata)return -ENOENT ; -bpf_selem_unlink (SELEM (sdata)); +bpf_selem_unlink (SELEM (sdata), true);return 0 ; }
kernel/bpf/bpf_local_storage.c +19 -10
@@ -106,7 +106,7 @@ static void bpf_selem_free_rcu(struct rcu_head *rcu)*/ bool bpf_selem_unlink_storage_nolock (struct bpf_local_storage *local_storage,struct bpf_local_storage_elem *selem, -bool uncharge_mem) +bool uncharge_mem,bool use_trace_rcu) {struct bpf_local_storage_map *smap;bool free_local_storage;
@@ -150,11 +150,16 @@ bool bpf_selem_unlink_storage_nolock(struct bpf_local_storage *local_storage, SDATA(selem))RCU_INIT_POINTER (local_storage->cache[smap->cache_idx],NULL ); -call_rcu_tasks_trace (&selem->rcu, bpf_selem_free_rcu); +if (use_trace_rcu) +call_rcu_tasks_trace (&selem->rcu, bpf_selem_free_rcu); +else +kfree_rcu (selem, rcu); +return free_local_storage; } -static void __bpf_selem_unlink_storage (struct bpf_local_storage_elem *selem) +static void __bpf_selem_unlink_storage (struct bpf_local_storage_elem *selem, +bool use_trace_rcu) {struct bpf_local_storage *local_storage;bool free_local_storage = false;
@@ -169,12 +174,16 @@ static void __bpf_selem_unlink_storage(struct bpf_local_storage_elem *selem)raw_spin_lock_irqsave (&local_storage->lock, flags);if (likely (selem_linked_to_storage (selem))) free_local_storage =bpf_selem_unlink_storage_nolock ( - local_storage, selem, true); + local_storage, selem, true, use_trace_rcu);raw_spin_unlock_irqrestore (&local_storage->lock, flags); -if (free_local_storage) -call_rcu_tasks_trace (&local_storage->rcu, +if (free_local_storage) { +if (use_trace_rcu) +call_rcu_tasks_trace (&local_storage->rcu, bpf_local_storage_free_rcu); +else +kfree_rcu (local_storage, rcu); + } } void bpf_selem_link_storage_nolock(struct bpf_local_storage *local_storage,
@@ -214,14 +223,14 @@ void bpf_selem_link_map(struct bpf_local_storage_map *smap,raw_spin_unlock_irqrestore (&b->lock, flags); } -void bpf_selem_unlink (struct bpf_local_storage_elem *selem) +void bpf_selem_unlink (struct bpf_local_storage_elem *selem,bool use_trace_rcu) {/* Always unlink from map before unlinking from local_storage * because selem will be freed after successfully unlinked from * the local_storage. */ bpf_selem_unlink_map (selem); -__bpf_selem_unlink_storage (selem); +__bpf_selem_unlink_storage (selem, use_trace_rcu); } struct bpf_local_storage_data *
@@ -466,7 +475,7 @@ bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap,if (old_sdata) {bpf_selem_unlink_map (SELEM (old_sdata));bpf_selem_unlink_storage_nolock (local_storage,SELEM (old_sdata), - false); + false, true); } unlock:
@@ -548,7 +557,7 @@ void bpf_local_storage_map_free(struct bpf_local_storage_map *smap,migrate_disable ();__this_cpu_inc (*busy_counter); } -bpf_selem_unlink (selem); +bpf_selem_unlink (selem, false);if (busy_counter) {__this_cpu_dec (*busy_counter);migrate_enable ();
kernel/bpf/bpf_task_storage.c +2 -2
@@ -102,7 +102,7 @@ void bpf_task_storage_free(struct task_struct *task)*/ bpf_selem_unlink_map (selem); free_task_storage =bpf_selem_unlink_storage_nolock ( - local_storage, selem, false); + local_storage, selem, false, false); }raw_spin_unlock_irqrestore (&local_storage->lock, flags);bpf_task_storage_unlock ();
@@ -192,7 +192,7 @@ static int task_storage_delete(struct task_struct *task, struct bpf_map *map)if (!sdata)return -ENOENT ; -bpf_selem_unlink (SELEM (sdata)); +bpf_selem_unlink (SELEM (sdata), true);return 0 ; }
net/core/bpf_sk_storage.c +3 -3
@@ -40,7 +40,7 @@ static int bpf_sk_storage_del(struct sock *sk, struct bpf_map *map)if (!sdata)return -ENOENT ; -bpf_selem_unlink (SELEM (sdata)); +bpf_selem_unlink (SELEM (sdata), true);return 0 ; }
@@ -75,8 +75,8 @@ void bpf_sk_storage_free(struct sock *sk)* sk_storage. */ bpf_selem_unlink_map (selem); - free_sk_storage =bpf_selem_unlink_storage_nolock (sk_storage, - selem, true); + free_sk_storage =bpf_selem_unlink_storage_nolock ( + sk_storage, selem, true, false); }raw_spin_unlock_bh (&sk_storage->lock);rcu_read_unlock ();
Signature · 1 line
2.36.0.rc0.470.gd361397f0d-goog
Quoted 9 lines · KP Singh
bpf_{sk,task,inode}_storage_free() do not need to use
call_rcu_tasks_traceas no BPF program should be accessing the owner as it's being destroyed. The only other reader at this point isbpf_local_storage_map_free()which uses normal RCU.The only path that needs trace RCU are:
* bpf_local_storage_{delete,update} helpers
* map_{delete,update}_elem() syscalls
This patch was applied to bpf/bpf-next.git (master) by Alexei Starovoitov <[email protected]>:
Hello:
This patch was applied to bpf/bpf-next.git (master) by Alexei Starovoitov <[email protected]>:
Quoted 8 lines · On Mon, 18 Apr 2022 15:51:58 +0000 you wrote:
bpf_{sk,task,inode}_storage_free() do not need to use
call_rcu_tasks_traceas no BPF program should be accessing the owner as it's being destroyed. The only other reader at this point isbpf_local_storage_map_free()which uses normal RCU.The only path that needs trace RCU are:
[...]
Here is the summary with links:
- [v2,bpf-next] bpf: Fix usage of trace RCU in local storage.
https://git.kernel.org/bpf/bpf-next/c/dcf456c9a095
You are awesome, thank you!
Signature · 2 lines
Deet-doot-dot, I am a bot. https://korg.docs.kernel.org/patchwork/pwbot.html