netfilter: ipset fixes, second batch
Subsystems (2)
cover netfilter: ipset fixes, second batch
Hi Pablo,
Here follows the second batch of the ipset fixes, waiting for the comments of sashiko.
Best regards,
Jozsef
Jozsef Kadlecsik (6):
netfilter: ipset: Don't use test_bit() in lockless RCU readers in hash
types
netfilter: ipset: Don't use test_bit() in lockless RCU readers in
bitmap types
netfilter: ipset: fix order of kfree_rcu() and rcu_assign_pointer()
netfilter: ipset: skip gc when resize is in progress
netfilter: ipset: fix potential torn read in reuse/forceadd cases
netfilter: ipset: add comment how cidr bookkeeping is working
net/netfilter/ipset/ip_set_bitmap_gen.h | 7 ++- net/netfilter/ipset/ip_set_core.c | 4 +- net/netfilter/ipset/ip_set_hash_gen.h | 63 +++++++++++++++++++------ 3 files changed, 56 insertions(+), 18 deletions(-)
Signature · 1 line
2.39.5
PATCH 1/6 netfilter: ipset: Don't use test_bit() in lockless RCU readers in hash types
Sashiko pointed out that there are a few lockless RCU readers using test_bit() which is a relaxed atomic operation and provides no memory barrier guarantees. Use test_bit_acquire() instead where the operation may run parallel with add/del/gc, i.e. is not one from the next cases
- protected by region lock
- in a set destroy phase
- in a new/temporary set creation phase
Also, add two missing smp_mb__after_atomic() operations.
--- net/netfilter/ipset/ip_set_hash_gen.h | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-)
net/netfilter/ipset/ip_set_hash_gen.h +6 -5
@@ -689,7 +689,7 @@ mtype_resize(struct ip_set *set, bool retried)continue ; pos =smp_load_acquire (&n->pos);for (j =0 ; j < pos; j++) { -if (!test_bit (j, n->used)) +if (!test_bit_acquire (j, n->used))continue ; data =ahash_data (n, j, dsize);if (SET_ELEM_EXPIRED (set, data))
@@ -826,7 +826,7 @@ mtype_ext_size(struct ip_set *set, u32 *elements, size_t *ext_size)continue ; pos =smp_load_acquire (&n->pos);for (j =0 ; j < pos; j++) { -if (!test_bit (j, n->used)) +if (!test_bit_acquire (j, n->used))continue ; data =ahash_data (n, j, set->dsize);if (!SET_ELEM_EXPIRED (set, data))
@@ -995,6 +995,7 @@ mtype_add(struct ip_set *set, void *value, const struct ip_set_ext *ext,/* Ensure all data writes are visible before updating position */ smp_store_release (&n->pos, npos);set_bit (j, n->used); +smp_mb__after_atomic ();if (old !=ERR_PTR (-ENOENT )) {rcu_assign_pointer (hbucket (t, key), n);if (old)
@@ -1201,7 +1202,7 @@ mtype_test_cidrs(struct ip_set *set, struct mtype_elem *d,continue ; pos =smp_load_acquire (&n->pos);for (i =0 ; i < pos; i++) { -if (!test_bit (i, n->used)) +if (!test_bit_acquire (i, n->used))continue ; data =ahash_data (n, i, set->dsize);if (!mtype_data_equal (data, d, &multi))
@@ -1259,7 +1260,7 @@ mtype_test(struct ip_set *set, void *value, const struct ip_set_ext *ext, } pos =smp_load_acquire (&n->pos);for (i =0 ; i < pos; i++) { -if (!test_bit (i, n->used)) +if (!test_bit_acquire (i, n->used))continue ; data =ahash_data (n, i, set->dsize);if (!mtype_data_equal (data, d, &multi))
@@ -1396,7 +1397,7 @@ mtype_list(const struct ip_set *set,continue ; pos =smp_load_acquire (&n->pos);for (i =0 ; i < pos; i++) { -if (!test_bit (i, n->used)) +if (!test_bit_acquire (i, n->used))continue ; e =ahash_data (n, i, set->dsize);if (SET_ELEM_EXPIRED (set, e))
Signature · 1 line
2.39.5
PATCH 2/6 netfilter: ipset: Don't use test_bit() in lockless RCU readers in bitmap types
In reply to Jozsef KadlecsikThe pair of the patch "netfilter: ipset: Don't use test_bit() in lockless RCU readers in hash types" for the bitmap types.
--- net/netfilter/ipset/ip_set_bitmap_gen.h | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-)
net/netfilter/ipset/ip_set_bitmap_gen.h +5 -2
@@ -51,7 +51,7 @@ mtype_ext_cleanup(struct ip_set *set)u32 id;for (id =0 ; id < map->elements; id++) -if (test_bit (id, map->members)) +if (test_bit_acquire (id, map->members))ip_set_ext_destroy (set,get_ext (set, map, id)); }
@@ -142,6 +142,7 @@ mtype_add(struct ip_set *set, void *value, const struct ip_set_ext *ext, ret =0 ; }else if (!(flags &IPSET_FLAG_EXIST )) {set_bit (e->id, map->members); +smp_mb__after_atomic ();return -IPSET_ERR_EXIST ; }/* Element is re-added, cleanup extensions */
@@ -166,6 +167,7 @@ mtype_add(struct ip_set *set, void *value, const struct ip_set_ext *ext,/* Activate element */ set_bit (e->id, map->members); +smp_mb__after_atomic (); set->elements++;return 0 ;
@@ -219,7 +221,7 @@ mtype_list(const struct ip_set *set,cond_resched_rcu (); id = cb->args[IPSET_CB_ARG0 ]; x =get_ext (set, map, id); - if (!test_bit(id, map->members) || + if (!test_bit_acquire(id, map->members) || (SET_WITH_TIMEOUT(set) && #ifdef IP_SET_BITMAP_STORED_TIMEOUT mtype_is_filled(x) &&
@@ -278,6 +280,7 @@ mtype_gc(struct timer_list *t) x =get_ext (set, map, id);if (ip_set_timeout_expired (ext_timeout (x, set))) {clear_bit (id, map->members); +smp_mb__after_atomic ();ip_set_ext_destroy (set, x); set->elements--; }
Signature · 1 line
2.39.5
PATCH 3/6 netfilter: ipset: fix order of kfree_rcu() and rcu_assign_pointer()
In reply to Jozsef KadlecsikSashiko pointed out that kfree_rcu() was called before rcu_assign_pointer() in handling the comment extension. Fix the order so that rcu_assign_pointer() called first.
--- net/netfilter/ipset/ip_set_core.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
net/netfilter/ipset/ip_set_core.c +2 -2
@@ -351,8 +351,8 @@ ip_set_init_comment(struct ip_set *set, struct ip_set_comment *comment,if (unlikely (c)) { set->ext_size -=sizeof (*c) +strlen (c->str) +1 ; -kfree_rcu (c, rcu);rcu_assign_pointer (comment->c,NULL ); +kfree_rcu (c, rcu); }if (!len)return ;
@@ -393,8 +393,8 @@ ip_set_comment_free(struct ip_set *set, void *ptr)if (unlikely (!c))return ; set->ext_size -=sizeof (*c) +strlen (c->str) +1 ; -kfree_rcu (c, rcu);rcu_assign_pointer (comment->c,NULL ); +kfree_rcu (c, rcu); }typedef void (*destroyer )(struct ip_set *,void *);
Signature · 1 line
2.39.5
PATCH 4/6 netfilter: ipset: skip gc when resize is in progress
In reply to Jozsef KadlecsikZhengchuan Liang reported that because resize does not copy the comment extension into the resized set but uses it's pointer, ongoing gc can free the extension in the original set which then results stale pointer in the resized one. The proposed patch was to recreate the extensions for every element in the resized set. It is both expensive and wastes memory, so better skip gc when resizing in progress detected: resizing will destroy the original set anyway, so doing gc on it is unnecessary.
Reported by: Zhengchuan Liang <[email protected]> Reported by: Eulgyu Kim <[email protected]>
--- net/netfilter/ipset/ip_set_hash_gen.h | 32 ++++++++++++++++++++------- 1 file changed, 24 insertions(+), 8 deletions(-)
net/netfilter/ipset/ip_set_hash_gen.h +24 -8
@@ -75,7 +75,9 @@ struct hbucket {struct htable_gc {struct delayed_work dwork;struct ip_set *set;/* Set the gc belongs to */ +spinlock_t lock;/* Lock to exclude gc and resize */ u32 region;/* Last gc run position */ +bool resizing;/* Signal resize in progress */ };/* The hash table: the table size stored here in order to make resizing easy */
@@ -573,16 +575,21 @@ mtype_gc(struct work_struct *work) t =ipset_dereference_set (h->table, set);atomic_inc (&t->uref); numof_locks =ahash_numof_locks (t->htable_bits); - r = gc->region++; -if (r >= numof_locks) { - r = gc->region =0 ; - } next_run = (IPSET_GC_PERIOD (set->timeout) *HZ ) / numof_locks;if (next_run <HZ /10 ) next_run =HZ /10 ;spin_unlock_bh (&set->lock); +spin_lock_bh (&gc->lock); +if (gc->resizing) + goto skip_gc; + r = gc->region++; +if (r >= numof_locks) { + r = gc->region =0 ; + }mtype_gc_do (set, h, t, r); +skip_gc: +spin_unlock_bh (&gc->lock);if (atomic_dec_and_test (&t->uref) &&atomic_read (&t->ref)) {pr_debug ("Table destroy after resize by expire: %p\n" , t);
@@ -590,7 +597,6 @@ mtype_gc(struct work_struct *work) }queue_delayed_work (system_power_efficient_wq, &gc->dwork, next_run); - } static void
@@ -646,6 +652,9 @@ mtype_resize(struct ip_set *set, bool retried)#endif orig =ipset_dereference_bh_nfnl (h->table); htable_bits = orig->htable_bits; +spin_lock_bh (&h->gc.lock); + h->gc.resizing =1 ; +spin_unlock_bh (&h->gc.lock); retry: ret =0 ;
@@ -672,7 +681,11 @@ mtype_resize(struct ip_set *set, bool retried)spin_lock_init (&t->hregion[i].lock);/* There can't be another parallel resizing, -* but dumping, gc, kernel side add/del are possible +* but dumping, kernel side add/del are possible. +* +* Parallel gc is explicitly excluded because +* resize destroys the old set and its extensions +* which can interfere with an ongoing gc. */ orig =ipset_dereference_bh_nfnl (h->table);atomic_set (&orig->ref,1 );
@@ -692,8 +705,7 @@ mtype_resize(struct ip_set *set, bool retried)if (!test_bit_acquire (j, n->used))continue ; data =ahash_data (n, j, dsize); -if (SET_ELEM_EXPIRED (set, data)) -continue ; +/* Expired elements copied as well */ #ifdef IP_SET_HASH_WITH_NETS /* We have readers running parallel with us, * so the live data cannot be modified.
@@ -785,6 +797,9 @@ mtype_resize(struct ip_set *set, bool retried) } out: +spin_lock_bh (&h->gc.lock); + h->gc.resizing =0 ; +spin_unlock_bh (&h->gc.lock);#ifdef IP_SET_HASH_WITH_NETS kfree (tmp);#endif
@@ -1594,6 +1609,7 @@ IPSET_TOKEN(HTYPE, _create)(struct net *net, struct ip_set *set,return -ENOMEM ; } h->gc.set = set; +spin_lock_init (&h->gc.lock);for (i =0 ; i <ahash_numof_locks (hbits); i++)spin_lock_init (&t->hregion[i].lock); h->maxelem = maxelem;
Signature · 1 line
2.39.5
PATCH 5/6 netfilter: ipset: fix potential torn read in reuse/forceadd cases
In reply to Jozsef KadlecsikSashiko pointed out that due to using memcpy() to overwrite already existing entry in reuse/forceadd cases, it can lead to torn reads for concurrent lockless RCU readers. Set the element explicitly to unused before trying to reuse it.
--- net/netfilter/ipset/ip_set_hash_gen.h | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-)
net/netfilter/ipset/ip_set_hash_gen.h +8 -1
@@ -868,7 +868,7 @@ mtype_add(struct ip_set *set, void *value, const struct ip_set_ext *ext,bool flag_exist = flags &IPSET_FLAG_EXIST ;bool deleted = false, forceadd = false, reuse = false;u32 r, key, multi =0 , elements, maxelem; -u8 npos =0 ; +u8 npos =0 , retried =0 ;rcu_read_lock_bh (); t =rcu_dereference_bh (h->table);
@@ -893,6 +893,7 @@ mtype_add(struct ip_set *set, void *value, const struct ip_set_ext *ext, }rcu_read_unlock_bh (); +retry:spin_lock_bh (&t->hregion[r].lock); n =rcu_dereference_bh (hbucket (t, key));if (!n) {
@@ -941,6 +942,10 @@ mtype_add(struct ip_set *set, void *value, const struct ip_set_ext *ext, j =0 ; data =ahash_data (n, j, set->dsize);if (!deleted) { +if (retried++) + goto set_full; +clear_bit (j, n->used); +smp_mb__after_atomic ();#ifdef IP_SET_HASH_WITH_NETS for (i =0 ; i <IPSET_NET_COUNT ; i++) mtype_del_cidr(set, h,
@@ -949,6 +954,8 @@ mtype_add(struct ip_set *set, void *value, const struct ip_set_ext *ext,#endif ip_set_ext_destroy (set, data); t->hregion[r].elements--; +spin_unlock_bh (&t->hregion[r].lock); + goto retry; } goto copy_data; }
Signature · 1 line
2.39.5
PATCH 6/6 netfilter: ipset: add comment how cidr bookkeeping is working
In reply to Jozsef KadlecsikSashiko thinks that cidr bookkeeping might be unsafe because the concurrent RCU reader in mtype_test_cidrs() uses the data without sequence locks or read-side barriers. However every right shift (add new entry) and left shift (delete entry) is performed by duplicating the entry just shifted. Therefore concurrent reader will just duplicate a test with the same value twice thus existing entries cannot be skipped.
--- net/netfilter/ipset/ip_set_hash_gen.h | 11 +++++++++++ 1 file changed, 11 insertions(+)
net/netfilter/ipset/ip_set_hash_gen.h +11 -0
@@ -342,6 +342,12 @@ mtype_add_cidr(struct ip_set *set, struct htype *h, u8 cidr, u8 n) } }if (j !=-1 ) { +/* We shift the cidr values to the right +* by duplicating the entries one by one, +* starting from the end. +* It means the same test can be repeated twice +* by a concurrent mtype_test_cidrs() reader. +*/ for (; i > j; i--) h->nets[i].cidr[n] = h->nets[i -1 ].cidr[n]; }
@@ -363,6 +369,11 @@ mtype_del_cidr(struct ip_set *set, struct htype *h, u8 cidr, u8 n) h->nets[CIDR_POS (cidr)].nets[n]--;if (h->nets[CIDR_POS (cidr)].nets[n] >0 ) goto unlock; +/* We shift the cidr values to the left +* by duplicating the remaining entries one by one. +* It means the same test can be repeated twice +* by a concurrent mtype_test_cidrs() reader. +*/ for (j = i; j < net_end && h->nets[j].cidr[n]; j++) h->nets[j].cidr[n] = h->nets[j +1 ].cidr[n]; h->nets[j].cidr[n] =0 ;
Signature · 1 line
2.39.5