bpf: fix a potential deadlock with bpf_map_do_batch
PATCH 1/1 bpf: fix a potential deadlock with bpf_map_do_batch
Commit 057996380a42 ("bpf: Add batch ops to all htab bpf map") added lookup_and_delete batch operation for hash table. The current implementation has bpf_lru_push_free() inside the bucket lock, which may cause a deadlock.
syzbot reports:
-> #2 (&htab->buckets[i].lock#2){....}:
__raw_spin_lock_irqsave include/linux/spinlock_api_smp.h:110 [inline]
_raw_spin_lock_irqsave+0x95/0xcd kernel/locking/spinlock.c:159
htab_lru_map_delete_node+0xce/0x2f0 kernel/bpf/hashtab.c:593
__bpf_lru_list_shrink_inactive kernel/bpf/bpf_lru_list.c:220 [inline]
__bpf_lru_list_shrink+0xf9/0x470 kernel/bpf/bpf_lru_list.c:266
bpf_lru_list_pop_free_to_local kernel/bpf/bpf_lru_list.c:340 [inline]
bpf_common_lru_pop_free kernel/bpf/bpf_lru_list.c:447 [inline]
bpf_lru_pop_free+0x87c/0x1670 kernel/bpf/bpf_lru_list.c:499
prealloc_lru_pop+0x2c/0xa0 kernel/bpf/hashtab.c:132
__htab_lru_percpu_map_update_elem+0x67e/0xa90 kernel/bpf/hashtab.c:1069
bpf_percpu_hash_update+0x16e/0x210 kernel/bpf/hashtab.c:1585
bpf_map_update_value.isra.0+0x2d7/0x8e0 kernel/bpf/syscall.c:181
generic_map_update_batch+0x41f/0x610 kernel/bpf/syscall.c:1319
bpf_map_do_batch+0x3f5/0x510 kernel/bpf/syscall.c:3348
__do_sys_bpf+0x9b7/0x41e0 kernel/bpf/syscall.c:3460
__se_sys_bpf kernel/bpf/syscall.c:3355 [inline]
__x64_sys_bpf+0x73/0xb0 kernel/bpf/syscall.c:3355
do_syscall_64+0xfa/0x790 arch/x86/entry/common.c:294
entry_SYSCALL_64_after_hwframe+0x49/0xbe-> #0 (&loc_l->lock){....}:
check_prev_add kernel/locking/lockdep.c:2475 [inline]
check_prevs_add kernel/locking/lockdep.c:2580 [inline]
validate_chain kernel/locking/lockdep.c:2970 [inline]
__lock_acquire+0x2596/0x4a00 kernel/locking/lockdep.c:3954
lock_acquire+0x190/0x410 kernel/locking/lockdep.c:4484
__raw_spin_lock_irqsave include/linux/spinlock_api_smp.h:110 [inline]
_raw_spin_lock_irqsave+0x95/0xcd kernel/locking/spinlock.c:159
bpf_common_lru_push_free kernel/bpf/bpf_lru_list.c:516 [inline]
bpf_lru_push_free+0x250/0x5b0 kernel/bpf/bpf_lru_list.c:555
__htab_map_lookup_and_delete_batch+0x8d4/0x1540 kernel/bpf/hashtab.c:1374
htab_lru_map_lookup_and_delete_batch+0x34/0x40 kernel/bpf/hashtab.c:1491
bpf_map_do_batch+0x3f5/0x510 kernel/bpf/syscall.c:3348
__do_sys_bpf+0x1f7d/0x41e0 kernel/bpf/syscall.c:3456
__se_sys_bpf kernel/bpf/syscall.c:3355 [inline]
__x64_sys_bpf+0x73/0xb0 kernel/bpf/syscall.c:3355
do_syscall_64+0xfa/0x790 arch/x86/entry/common.c:294
entry_SYSCALL_64_after_hwframe+0x49/0xbe
Possible unsafe locking scenario: CPU0 CPU2
---- ----
lock(&htab->buckets[i].lock#2);
lock(&l->lock);
lock(&htab->buckets[i].lock#2);
lock(&loc_l->lock);
*** DEADLOCK ***To fix the issue, for htab_lru_map_lookup_and_delete_batch() in CPU0, let us do bpf_lru_push_free() out of the htab bucket lock. This can avoid the above deadlock scenario.
--- kernel/bpf/hashtab.c | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-)
kernel/bpf/hashtab.c +16 -3
@@ -56,6 +56,7 @@ struct htab_elem {union {struct bpf_htab *htab;struct pcpu_freelist_node fnode; +struct htab_elem *link; }; }; };
@@ -1255,6 +1256,7 @@ __htab_map_lookup_and_delete_batch(struct bpf_map *map,void __user *uvalues =u64_to_user_ptr (attr->batch.values);void __user *ukeys =u64_to_user_ptr (attr->batch.keys);void *ubatch =u64_to_user_ptr (attr->batch.in_batch); +struct htab_elem *node_to_free =NULL ;u32 batch, max_count, size, bucket_size;u64 elem_map_flags, map_flags;struct hlist_nulls_head *head;
@@ -1370,9 +1372,13 @@ __htab_map_lookup_and_delete_batch(struct bpf_map *map, }if (do_delete) {hlist_nulls_del_rcu (&l->hash_node); -if (is_lru_map) -bpf_lru_push_free (&htab->lru, &l->lru_node); -else +if (is_lru_map) { +/* link to-be-freed elements together so +* they can freed outside bucket lock region. +*/ + l->link = node_to_free; + node_to_free = l; + }else free_htab_elem (htab, l); } dst_key += key_size;
@@ -1380,6 +1386,13 @@ __htab_map_lookup_and_delete_batch(struct bpf_map *map, }raw_spin_unlock_irqrestore (&b->lock, flags); + +while (node_to_free) { + l = node_to_free; + node_to_free = node_to_free->link; +bpf_lru_push_free (&htab->lru, &l->lru_node); + } +/* If we are not copying data, we can go to next bucket and avoid * unlocking the rcu. */
Signature · 1 line
2.17.1
Quoted 97 lines · Yonghong Song
Commit
057996380a42("bpf: Add batch ops to all htab bpf map") addedlookup_and_deletebatch operation for hash table. The current implementation hasbpf_lru_push_free()inside the bucket lock, which may cause a deadlock.syzbot reports:
-> #2 (&htab->buckets[i].lock#2){....}: __raw_spin_lock_irqsave include/linux/spinlock_api_smp.h:110 [inline] _raw_spin_lock_irqsave+0x95/0xcd kernel/locking/spinlock.c:159 htab_lru_map_delete_node+0xce/0x2f0 kernel/bpf/hashtab.c:593 __bpf_lru_list_shrink_inactive kernel/bpf/bpf_lru_list.c:220 [inline] __bpf_lru_list_shrink+0xf9/0x470 kernel/bpf/bpf_lru_list.c:266 bpf_lru_list_pop_free_to_local kernel/bpf/bpf_lru_list.c:340 [inline] bpf_common_lru_pop_free kernel/bpf/bpf_lru_list.c:447 [inline] bpf_lru_pop_free+0x87c/0x1670 kernel/bpf/bpf_lru_list.c:499 prealloc_lru_pop+0x2c/0xa0 kernel/bpf/hashtab.c:132 __htab_lru_percpu_map_update_elem+0x67e/0xa90 kernel/bpf/hashtab.c:1069 bpf_percpu_hash_update+0x16e/0x210 kernel/bpf/hashtab.c:1585 bpf_map_update_value.isra.0+0x2d7/0x8e0 kernel/bpf/syscall.c:181 generic_map_update_batch+0x41f/0x610 kernel/bpf/syscall.c:1319 bpf_map_do_batch+0x3f5/0x510 kernel/bpf/syscall.c:3348 __do_sys_bpf+0x9b7/0x41e0 kernel/bpf/syscall.c:3460 __se_sys_bpf kernel/bpf/syscall.c:3355 [inline] __x64_sys_bpf+0x73/0xb0 kernel/bpf/syscall.c:3355 do_syscall_64+0xfa/0x790 arch/x86/entry/common.c:294 entry_SYSCALL_64_after_hwframe+0x49/0xbe -> #0 (&loc_l->lock){....}: check_prev_add kernel/locking/lockdep.c:2475 [inline] check_prevs_add kernel/locking/lockdep.c:2580 [inline] validate_chain kernel/locking/lockdep.c:2970 [inline] __lock_acquire+0x2596/0x4a00 kernel/locking/lockdep.c:3954 lock_acquire+0x190/0x410 kernel/locking/lockdep.c:4484 __raw_spin_lock_irqsave include/linux/spinlock_api_smp.h:110 [inline] _raw_spin_lock_irqsave+0x95/0xcd kernel/locking/spinlock.c:159 bpf_common_lru_push_free kernel/bpf/bpf_lru_list.c:516 [inline] bpf_lru_push_free+0x250/0x5b0 kernel/bpf/bpf_lru_list.c:555 __htab_map_lookup_and_delete_batch+0x8d4/0x1540 kernel/bpf/hashtab.c:1374 htab_lru_map_lookup_and_delete_batch+0x34/0x40 kernel/bpf/hashtab.c:1491 bpf_map_do_batch+0x3f5/0x510 kernel/bpf/syscall.c:3348 __do_sys_bpf+0x1f7d/0x41e0 kernel/bpf/syscall.c:3456 __se_sys_bpf kernel/bpf/syscall.c:3355 [inline] __x64_sys_bpf+0x73/0xb0 kernel/bpf/syscall.c:3355 do_syscall_64+0xfa/0x790 arch/x86/entry/common.c:294 entry_SYSCALL_64_after_hwframe+0x49/0xbe Possible unsafe locking scenario: CPU0 CPU2 ---- ---- lock(&htab->buckets[i].lock#2); lock(&l->lock); lock(&htab->buckets[i].lock#2); lock(&loc_l->lock); *** DEADLOCK ***To fix the issue, for
htab_lru_map_lookup_and_delete_batch()in CPU0, let us dobpf_lru_push_free()out of the htab bucket lock. This can avoid the above deadlock scenario.Fixes 057996380a42 ("bpf: Add batch ops to all htab bpf map")Reported-by [email protected]Reported-by [email protected]Suggested-by Hillf Danton <[email protected]>Suggested-by Martin KaFai Lau <[email protected]>Cc Brian Vazquez <[email protected]>Signed-off-by Yonghong Song <[email protected]>--- kernel/bpf/hashtab.c | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-)kernel/bpf/hashtab.c +2
-1@@ -56,6 +56,7 @@ struct htab_elem {union {struct bpf_htab *htab;struct pcpu_freelist_node fnode; +struct htab_elem *link; }; }; };@@ -1255,6 +1256,7 @@ __htab_map_lookup_and_delete_batch(struct bpf_map *map,void __user *uvalues =u64_to_user_ptr (attr->batch.values);void __user *ukeys =u64_to_user_ptr (attr->batch.keys);void *ubatch =u64_to_user_ptr (attr->batch.in_batch); +struct htab_elem *node_to_free =NULL ;u32 batch, max_count, size, bucket_size;u64 elem_map_flags, map_flags;struct hlist_nulls_head *head;@@ -1370,9 +1372,13 @@ __htab_map_lookup_and_delete_batch(struct bpf_map *map, }if (do_delete) {hlist_nulls_del_rcu (&l->hash_node); -if (is_lru_map)
kernel/bpf/hashtab.c __htab_map_lookup_and_delete_batch(struct bpf_map *map, · line 1382
Quoted 10 lines
kernel/bpf/hashtab.c +7
-2-bpf_lru_push_free (&htab->lru, &l->lru_node); - else +if (is_lru_map) { +/* link to-be-freed elements together so +* they can freed outside bucket lock region. +*/ + l->link = node_to_free; + node_to_free = l; + }else free_htab_elem (htab, l);
Nit, we need braces in both branches now, as perprocess/coding-style.rst:
| This does not apply if only one branch of a conditional statement is a single
| statement; in the latter case use braces in both branches:
|
| .. code-block:: c
|
| if (condition) {
| do_this();
| do_that();
| } else {
| otherwise();
| }Quoted 6 lines
kernel/bpf/hashtab.c +0
-0} dst_key += key_size;@@ -1380,6 +1386,13 @@ __htab_map_lookup_and_delete_batch(struct bpf_map *map, }raw_spin_unlock_irqrestore (&b->lock, flags);
kernel/bpf/hashtab.c __htab_map_lookup_and_delete_batch(struct bpf_map *map, · line 1398
Quoted 10 lines
kernel/bpf/hashtab.c +7
-0+ +while (node_to_free) { + l = node_to_free; + node_to_free = node_to_free->link; +bpf_lru_push_free (&htab->lru, &l->lru_node); + } +/* If we are not copying data, we can go to next bucket and avoid * unlocking the rcu. */
Quoted 112 lines · Jakub Sitnicki
Quoted 97 lines · Yonghong Song
Commit
057996380a42("bpf: Add batch ops to all htab bpf map") addedlookup_and_deletebatch operation for hash table. The current implementation hasbpf_lru_push_free()inside the bucket lock, which may cause a deadlock.syzbot reports:
-> #2 (&htab->buckets[i].lock#2){....}: __raw_spin_lock_irqsave include/linux/spinlock_api_smp.h:110 [inline] _raw_spin_lock_irqsave+0x95/0xcd kernel/locking/spinlock.c:159 htab_lru_map_delete_node+0xce/0x2f0 kernel/bpf/hashtab.c:593 __bpf_lru_list_shrink_inactive kernel/bpf/bpf_lru_list.c:220 [inline] __bpf_lru_list_shrink+0xf9/0x470 kernel/bpf/bpf_lru_list.c:266 bpf_lru_list_pop_free_to_local kernel/bpf/bpf_lru_list.c:340 [inline] bpf_common_lru_pop_free kernel/bpf/bpf_lru_list.c:447 [inline] bpf_lru_pop_free+0x87c/0x1670 kernel/bpf/bpf_lru_list.c:499 prealloc_lru_pop+0x2c/0xa0 kernel/bpf/hashtab.c:132 __htab_lru_percpu_map_update_elem+0x67e/0xa90 kernel/bpf/hashtab.c:1069 bpf_percpu_hash_update+0x16e/0x210 kernel/bpf/hashtab.c:1585 bpf_map_update_value.isra.0+0x2d7/0x8e0 kernel/bpf/syscall.c:181 generic_map_update_batch+0x41f/0x610 kernel/bpf/syscall.c:1319 bpf_map_do_batch+0x3f5/0x510 kernel/bpf/syscall.c:3348 __do_sys_bpf+0x9b7/0x41e0 kernel/bpf/syscall.c:3460 __se_sys_bpf kernel/bpf/syscall.c:3355 [inline] __x64_sys_bpf+0x73/0xb0 kernel/bpf/syscall.c:3355 do_syscall_64+0xfa/0x790 arch/x86/entry/common.c:294 entry_SYSCALL_64_after_hwframe+0x49/0xbe -> #0 (&loc_l->lock){....}: check_prev_add kernel/locking/lockdep.c:2475 [inline] check_prevs_add kernel/locking/lockdep.c:2580 [inline] validate_chain kernel/locking/lockdep.c:2970 [inline] __lock_acquire+0x2596/0x4a00 kernel/locking/lockdep.c:3954 lock_acquire+0x190/0x410 kernel/locking/lockdep.c:4484 __raw_spin_lock_irqsave include/linux/spinlock_api_smp.h:110 [inline] _raw_spin_lock_irqsave+0x95/0xcd kernel/locking/spinlock.c:159 bpf_common_lru_push_free kernel/bpf/bpf_lru_list.c:516 [inline] bpf_lru_push_free+0x250/0x5b0 kernel/bpf/bpf_lru_list.c:555 __htab_map_lookup_and_delete_batch+0x8d4/0x1540 kernel/bpf/hashtab.c:1374 htab_lru_map_lookup_and_delete_batch+0x34/0x40 kernel/bpf/hashtab.c:1491 bpf_map_do_batch+0x3f5/0x510 kernel/bpf/syscall.c:3348 __do_sys_bpf+0x1f7d/0x41e0 kernel/bpf/syscall.c:3456 __se_sys_bpf kernel/bpf/syscall.c:3355 [inline] __x64_sys_bpf+0x73/0xb0 kernel/bpf/syscall.c:3355 do_syscall_64+0xfa/0x790 arch/x86/entry/common.c:294 entry_SYSCALL_64_after_hwframe+0x49/0xbe Possible unsafe locking scenario: CPU0 CPU2 ---- ---- lock(&htab->buckets[i].lock#2); lock(&l->lock); lock(&htab->buckets[i].lock#2); lock(&loc_l->lock); *** DEADLOCK ***To fix the issue, for
htab_lru_map_lookup_and_delete_batch()in CPU0, let us dobpf_lru_push_free()out of the htab bucket lock. This can avoid the above deadlock scenario.Fixes 057996380a42 ("bpf: Add batch ops to all htab bpf map")Reported-by [email protected]Reported-by [email protected]Suggested-by Hillf Danton <[email protected]>Suggested-by Martin KaFai Lau <[email protected]>Cc Brian Vazquez <[email protected]>Signed-off-by Yonghong Song <[email protected]>--- kernel/bpf/hashtab.c | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-)kernel/bpf/hashtab.c +2
-1@@ -56,6 +56,7 @@ struct htab_elem {union {struct bpf_htab *htab;struct pcpu_freelist_node fnode; +struct htab_elem *link; }; }; };@@ -1255,6 +1256,7 @@ __htab_map_lookup_and_delete_batch(struct bpf_map *map,void __user *uvalues =u64_to_user_ptr (attr->batch.values);void __user *ukeys =u64_to_user_ptr (attr->batch.keys);void *ubatch =u64_to_user_ptr (attr->batch.in_batch); +struct htab_elem *node_to_free =NULL ;u32 batch, max_count, size, bucket_size;u64 elem_map_flags, map_flags;struct hlist_nulls_head *head;@@ -1370,9 +1372,13 @@ __htab_map_lookup_and_delete_batch(struct bpf_map *map, }if (do_delete) {hlist_nulls_del_rcu (&l->hash_node); -if (is_lru_map)kernel/bpf/hashtab.c __htab_map_lookup_and_delete_batch(struct bpf_map *map, · line 1382
Quoted 10 lines
kernel/bpf/hashtab.c +7
-2-bpf_lru_push_free (&htab->lru, &l->lru_node); - else +if (is_lru_map) { +/* link to-be-freed elements together so +* they can freed outside bucket lock region. +*/ + l->link = node_to_free; + node_to_free = l; + }else free_htab_elem (htab, l);Nit, we need braces in both branches now, as per
process/coding-style.rst:
Quoted 11 lines
| This does not apply if only one branch of a conditional statement is a single
| statement; in the latter case use braces in both branches:
|
| .. code-block:: c
|
| if (condition) {
| do_this();
| do_that();
| } else {
| otherwise();
| }Thanks! Make sense. Will respin.
Quoted 20 lines
Quoted 7 lines
kernel/bpf/hashtab.c +0
-0} dst_key += key_size;@@ -1380,6 +1386,13 @@ __htab_map_lookup_and_delete_batch(struct bpf_map *map, }raw_spin_unlock_irqrestore (&b->lock, flags);kernel/bpf/hashtab.c __htab_map_lookup_and_delete_batch(struct bpf_map *map, · line 1398
Quoted 10 lines
kernel/bpf/hashtab.c +7
-0+ +while (node_to_free) { + l = node_to_free; + node_to_free = node_to_free->link; +bpf_lru_push_free (&htab->lru, &l->lru_node); + } +/* If we are not copying data, we can go to next bucket and avoid * unlocking the rcu. */ Reviewed-by Jakub Sitnicki <[email protected]>
Quoted 127 lines · Yonghong Song
Commit
057996380a42("bpf: Add batch ops to all htab bpf map") addedlookup_and_deletebatch operation for hash table. The current implementation hasbpf_lru_push_free()inside the bucket lock, which may cause a deadlock.syzbot reports:
-> #2 (&htab->buckets[i].lock#2){....}: __raw_spin_lock_irqsave include/linux/spinlock_api_smp.h:110 [inline] _raw_spin_lock_irqsave+0x95/0xcd kernel/locking/spinlock.c:159 htab_lru_map_delete_node+0xce/0x2f0 kernel/bpf/hashtab.c:593 __bpf_lru_list_shrink_inactive kernel/bpf/bpf_lru_list.c:220 [inline] __bpf_lru_list_shrink+0xf9/0x470 kernel/bpf/bpf_lru_list.c:266 bpf_lru_list_pop_free_to_local kernel/bpf/bpf_lru_list.c:340 [inline] bpf_common_lru_pop_free kernel/bpf/bpf_lru_list.c:447 [inline] bpf_lru_pop_free+0x87c/0x1670 kernel/bpf/bpf_lru_list.c:499 prealloc_lru_pop+0x2c/0xa0 kernel/bpf/hashtab.c:132 __htab_lru_percpu_map_update_elem+0x67e/0xa90 kernel/bpf/hashtab.c:1069 bpf_percpu_hash_update+0x16e/0x210 kernel/bpf/hashtab.c:1585 bpf_map_update_value.isra.0+0x2d7/0x8e0 kernel/bpf/syscall.c:181 generic_map_update_batch+0x41f/0x610 kernel/bpf/syscall.c:1319 bpf_map_do_batch+0x3f5/0x510 kernel/bpf/syscall.c:3348 __do_sys_bpf+0x9b7/0x41e0 kernel/bpf/syscall.c:3460 __se_sys_bpf kernel/bpf/syscall.c:3355 [inline] __x64_sys_bpf+0x73/0xb0 kernel/bpf/syscall.c:3355 do_syscall_64+0xfa/0x790 arch/x86/entry/common.c:294 entry_SYSCALL_64_after_hwframe+0x49/0xbe -> #0 (&loc_l->lock){....}: check_prev_add kernel/locking/lockdep.c:2475 [inline] check_prevs_add kernel/locking/lockdep.c:2580 [inline] validate_chain kernel/locking/lockdep.c:2970 [inline] __lock_acquire+0x2596/0x4a00 kernel/locking/lockdep.c:3954 lock_acquire+0x190/0x410 kernel/locking/lockdep.c:4484 __raw_spin_lock_irqsave include/linux/spinlock_api_smp.h:110 [inline] _raw_spin_lock_irqsave+0x95/0xcd kernel/locking/spinlock.c:159 bpf_common_lru_push_free kernel/bpf/bpf_lru_list.c:516 [inline] bpf_lru_push_free+0x250/0x5b0 kernel/bpf/bpf_lru_list.c:555 __htab_map_lookup_and_delete_batch+0x8d4/0x1540 kernel/bpf/hashtab.c:1374 htab_lru_map_lookup_and_delete_batch+0x34/0x40 kernel/bpf/hashtab.c:1491 bpf_map_do_batch+0x3f5/0x510 kernel/bpf/syscall.c:3348 __do_sys_bpf+0x1f7d/0x41e0 kernel/bpf/syscall.c:3456 __se_sys_bpf kernel/bpf/syscall.c:3355 [inline] __x64_sys_bpf+0x73/0xb0 kernel/bpf/syscall.c:3355 do_syscall_64+0xfa/0x790 arch/x86/entry/common.c:294 entry_SYSCALL_64_after_hwframe+0x49/0xbe Possible unsafe locking scenario: CPU0 CPU2 ---- ---- lock(&htab->buckets[i].lock#2); lock(&l->lock); lock(&htab->buckets[i].lock#2); lock(&loc_l->lock); *** DEADLOCK ***To fix the issue, for
htab_lru_map_lookup_and_delete_batch()in CPU0, let us dobpf_lru_push_free()out of the htab bucket lock. This can avoid the above deadlock scenario.Fixes 057996380a42 ("bpf: Add batch ops to all htab bpf map")Reported-by [email protected]Reported-by [email protected]Suggested-by Hillf Danton <[email protected]>Suggested-by Martin KaFai Lau <[email protected]>Cc Brian Vazquez <[email protected]>Signed-off-by Yonghong Song <[email protected]>--- kernel/bpf/hashtab.c | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-)kernel/bpf/hashtab.c +16
-4@@ -56,6 +56,7 @@ struct htab_elem {union {struct bpf_htab *htab;struct pcpu_freelist_node fnode; +struct htab_elem *link; }; }; };@@ -1255,6 +1256,7 @@ __htab_map_lookup_and_delete_batch(struct bpf_map *map,void __user *uvalues =u64_to_user_ptr (attr->batch.values);void __user *ukeys =u64_to_user_ptr (attr->batch.keys);void *ubatch =u64_to_user_ptr (attr->batch.in_batch); +struct htab_elem *node_to_free =NULL ;u32 batch, max_count, size, bucket_size;u64 elem_map_flags, map_flags;struct hlist_nulls_head *head;@@ -1370,9 +1372,13 @@ __htab_map_lookup_and_delete_batch(struct bpf_map *map, }if (do_delete) {hlist_nulls_del_rcu (&l->hash_node); -if (is_lru_map) -bpf_lru_push_free (&htab->lru, &l->lru_node); -else +if (is_lru_map) { +/* link to-be-freed elements together so +* they can freed outside bucket lock region. +*/ + l->link = node_to_free; + node_to_free = l; + }else free_htab_elem (htab, l); } dst_key += key_size;@@ -1380,6 +1386,13 @@ __htab_map_lookup_and_delete_batch(struct bpf_map *map, }raw_spin_unlock_irqrestore (&b->lock, flags); + +while (node_to_free) { + l = node_to_free; + node_to_free = node_to_free->link; +bpf_lru_push_free (&htab->lru, &l->lru_node); + } +/* If we are not copying data, we can go to next bucket and avoid * unlocking the rcu. */ --2.17.1