Skip to contentSkip to conversation

netfilter: ipset: gc, backlog and cidr patches

PATCH New Jozsef Kadlecsik 1 participant 6 messages Jul 2

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

cover netfilter: ipset: gc, backlog and cidr patches

Hi Pablo,

Here follows three patches which addresses the gc-resize clashing in the comment extension, the add/del backlog cleanup in the error path and a fairly large rewriting of the cidr bookkeeping. The two other patches could be squeezed into the next ones on the list but I think they are a bit independent cleanups and clarifications.

The batch does not address the issue Florian pointed out about sashiko's complains about possible missing rcu locking and the possible torn reads, also from sashiko reports.

Best regards,
Jozsef

Jozsef Kadlecsik (5):
netfilter: ipset: mark the rcu locked areas properly
netfilter: ipset: exlude gc when resize is in progress
netfilter: ipset: cleanup the add/del backlog when resize failed
netfilter: ipset: allocate the proper memory for the generic hash
structure
netfilter: ipset: rework cidr bookkeeping

 net/netfilter/ipset/ip_set_hash_gen.h        | 311 ++++++++++++-------
 net/netfilter/ipset/ip_set_hash_ipportnet.c  |   4 +-
 net/netfilter/ipset/ip_set_hash_net.c        |   4 +-
 net/netfilter/ipset/ip_set_hash_netiface.c   |   4 +-
 net/netfilter/ipset/ip_set_hash_netnet.c     |  12 +-
 net/netfilter/ipset/ip_set_hash_netport.c    |   4 +-
 net/netfilter/ipset/ip_set_hash_netportnet.c |  12 +-
 7 files changed, 230 insertions(+), 121 deletions(-)
Signature · 1 line
2.39.5
#1 lore raw

PATCH 1/5 netfilter: ipset: mark the rcu locked areas properly

When we bump the uref counter, there's no need to keep the rcu lock because the referred hash table can't disappear. Also, from the same reason in mtype_gc we need the rcu lock and not a spinlock.

---
 net/netfilter/ipset/ip_set_hash_gen.h | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)
net/netfilter/ipset/ip_set_hash_gen.h +5 -8
index dedf59b661dd..c9a071766243 100644
@@ -569,9 +569,10 @@ mtype_gc(struct work_struct *work)
 	set = gc->set;
 	h = set->data;
 
-	spin_lock_bh(&set->lock);
-	t = ipset_dereference_set(h->table, set);
+	rcu_read_lock_bh();
+	t = rcu_dereference_bh(h->table);
 	atomic_inc(&t->uref);
+	rcu_read_unlock_bh();
 	numof_locks = ahash_numof_locks(t->htable_bits);
 	r = gc->region++;
 	if (r >= numof_locks) {
@@ -580,7 +581,6 @@ mtype_gc(struct work_struct *work)
 	next_run = (IPSET_GC_PERIOD(set->timeout) * HZ) / numof_locks;
 	if (next_run < HZ/10)
 		next_run = HZ/10;
-	spin_unlock_bh(&set->lock);
 
 	mtype_gc_do(set, h, t, r);
 
@@ -860,15 +860,13 @@ mtype_add(struct ip_set *set, void *value, const struct ip_set_ext *ext,
 	key = HKEY(value, h->initval, t->htable_bits);
 	r = ahash_region(key);
 	atomic_inc(&t->uref);
+	rcu_read_unlock_bh();
 	elements = t->hregion[r].elements;
 	maxelem = t->maxelem;
 	if (elements >= maxelem) {
 		u32 e;
-		if (SET_WITH_TIMEOUT(set)) {
-			rcu_read_unlock_bh();
+		if (SET_WITH_TIMEOUT(set))
 			mtype_gc_do(set, h, t, r);
-			rcu_read_lock_bh();
-		}
 		maxelem = h->maxelem;
 		elements = 0;
 		for (e = 0; e < ahash_numof_locks(t->htable_bits); e++)
@@ -876,7 +874,6 @@ mtype_add(struct ip_set *set, void *value, const struct ip_set_ext *ext,
 		if (elements >= maxelem && SET_WITH_FORCEADD(set))
 			forceadd = true;
 	}
-	rcu_read_unlock_bh();
 
 	spin_lock_bh(&t->hregion[r].lock);
 	n = rcu_dereference_bh(hbucket(t, key));
Signature · 1 line
2.39.5
#2 lore raw

PATCH 2/5 netfilter: ipset: exlude gc when resize is in progress

In reply to Jozsef Kadlecsik

Zhengchuan Liang and Eulgyu Kim reported that because resize does not copy the comment extension into the resized set but uses it's pointer, ongoing gc can free the extension in the original set which then results stale pointer in the resized one. The proposed patch was to recreate the extensions for every element in the resized set. It is both expensive and wastes memory, so better exclude gc when resizing in progress
detected: resizing will destroy the original set anyway, so doing gc on it is unnecessary.

Introduce a new spinlock to exclude parallel gc and resize. Because we just set and check a bool value, there's no need for the parameter to be atomic_t and rename it for better readability.

Reported by: Zhengchuan Liang <[email protected]> Reported by: Eulgyu Kim <[email protected]>

---
 net/netfilter/ipset/ip_set_hash_gen.h | 31 +++++++++++++++++----------
 1 file changed, 20 insertions(+), 11 deletions(-)
net/netfilter/ipset/ip_set_hash_gen.h +20 -11
index c9a071766243..8104dbac02fa 100644
@@ -75,12 +75,13 @@ struct hbucket {
 struct htable_gc {
 	struct delayed_work dwork;
 	struct ip_set *set;	/* Set the gc belongs to */
+	spinlock_t lock;	/* Lock to exclude gc and resize */
 	u32 region;		/* Last gc run position */
 };
 
 /* The hash table: the table size stored here in order to make resizing easy */
 struct htable {
-	atomic_t ref;		/* References for resizing */
+	bool resizing;		/* Mark ongoing resize */
 	atomic_t uref;		/* References for dumping and gc */
 	u8 htable_bits;		/* size of hash table == 2^htable_bits */
 	u32 maxelem;		/* Maxelem per region */
@@ -582,9 +583,12 @@ mtype_gc(struct work_struct *work)
 	if (next_run < HZ/10)
 		next_run = HZ/10;
 
-	mtype_gc_do(set, h, t, r);
+	spin_lock_bh(&gc->lock);
+	if (!t->resizing)
+		mtype_gc_do(set, h, t, r);
+	spin_unlock_bh(&gc->lock);
 
-	if (atomic_dec_and_test(&t->uref) && atomic_read(&t->ref)) {
+	if (atomic_dec_and_test(&t->uref) && t->resizing) {
 		pr_debug("Table destroy after resize by expire: %p\n", t);
 		mtype_ahash_destroy(set, t, false);
 	}
@@ -672,11 +676,13 @@ mtype_resize(struct ip_set *set, bool retried)
 		spin_lock_init(&t->hregion[i].lock);
 
 	/* There can't be another parallel resizing,
-	 * but dumping, gc, kernel side add/del are possible
+	 * but dumping and kernel side add/del are possible
 	 */
 	orig = ipset_dereference_bh_nfnl(h->table);
-	atomic_set(&orig->ref, 1);
 	atomic_inc(&orig->uref);
+	spin_lock_bh(&h->gc.lock);
+	orig->resizing = true;
+	spin_unlock_bh(&h->gc.lock);
 	pr_debug("attempt to resize set %s from %u to %u, t %p\n",
 		 set->name, orig->htable_bits, htable_bits, orig);
 	for (r = 0; r < ahash_numof_locks(orig->htable_bits); r++) {
@@ -792,7 +798,9 @@ mtype_resize(struct ip_set *set, bool retried)
 
 cleanup:
 	rcu_read_unlock_bh();
-	atomic_set(&orig->ref, 0);
+	spin_lock_bh(&h->gc.lock);
+	orig->resizing = false;
+	spin_unlock_bh(&h->gc.lock);
 	atomic_dec(&orig->uref);
 	mtype_ahash_destroy(set, t, false);
 	if (ret == -EAGAIN)
@@ -1000,7 +1008,7 @@ mtype_add(struct ip_set *set, void *value, const struct ip_set_ext *ext,
 	ret = 0;
 resize:
 	spin_unlock_bh(&t->hregion[r].lock);
-	if (atomic_read(&t->ref) && ext->target) {
+	if (t->resizing && ext && ext->target) {
 		/* Resize is in process and kernel side add, save values */
 		struct mtype_resize_ad *x;
 
@@ -1027,7 +1035,7 @@ mtype_add(struct ip_set *set, void *value, const struct ip_set_ext *ext,
 unlock:
 	spin_unlock_bh(&t->hregion[r].lock);
 out:
-	if (atomic_dec_and_test(&t->uref) && atomic_read(&t->ref)) {
+	if (atomic_dec_and_test(&t->uref) && t->resizing) {
 		pr_debug("Table destroy after resize by add: %p\n", t);
 		mtype_ahash_destroy(set, t, false);
 	}
@@ -1090,7 +1098,7 @@ mtype_del(struct ip_set *set, void *value, const struct ip_set_ext *ext,
 #endif
 		ip_set_ext_destroy(set, data);
 
-		if (atomic_read(&t->ref) && ext->target) {
+		if (t->resizing && ext && ext->target) {
 			/* Resize is in process and kernel side del,
 			 * save values
 			 */
@@ -1141,7 +1149,7 @@ mtype_del(struct ip_set *set, void *value, const struct ip_set_ext *ext,
 		list_add(&x->list, &h->ad);
 		spin_unlock_bh(&set->lock);
 	}
-	if (atomic_dec_and_test(&t->uref) && atomic_read(&t->ref)) {
+	if (atomic_dec_and_test(&t->uref) && t->resizing) {
 		pr_debug("Table destroy after resize by del: %p\n", t);
 		mtype_ahash_destroy(set, t, false);
 	}
@@ -1350,7 +1358,7 @@ mtype_uref(struct ip_set *set, struct netlink_callback *cb, bool start)
 		rcu_read_unlock_bh();
 	} else if (cb->args[IPSET_CB_PRIVATE]) {
 		t = (struct htable *)cb->args[IPSET_CB_PRIVATE];
-		if (atomic_dec_and_test(&t->uref) && atomic_read(&t->ref)) {
+		if (atomic_dec_and_test(&t->uref) && t->resizing) {
 			pr_debug("Table destroy after resize "
 				 " by dump: %p\n", t);
 			mtype_ahash_destroy(set, t, false);
@@ -1590,6 +1598,7 @@ IPSET_TOKEN(HTYPE, _create)(struct net *net, struct ip_set *set,
 		return -ENOMEM;
 	}
 	h->gc.set = set;
+	spin_lock_init(&h->gc.lock);
 	for (i = 0; i < ahash_numof_locks(hbits); i++)
 		spin_lock_init(&t->hregion[i].lock);
 	h->maxelem = maxelem;
Signature · 1 line
2.39.5
#3 lore raw

PATCH 3/5 netfilter: ipset: cleanup the add/del backlog when resize failed

In reply to Jozsef Kadlecsik

Sashiko pointed out that the add/del backlog was not cleaned up when resize failed. Fix it in the corresponding error path. Also, make sure that the add/del backlog is htable-specific so when resize creates a new htable, old/new backlog can't be mixed up.

---
 net/netfilter/ipset/ip_set_hash_gen.h | 28 +++++++++++++++++++--------
 1 file changed, 20 insertions(+), 8 deletions(-)
net/netfilter/ipset/ip_set_hash_gen.h +20 -8
index 8104dbac02fa..c0132d0f4cc0 100644
@@ -85,6 +85,7 @@ struct htable {
 	atomic_t uref;		/* References for dumping and gc */
 	u8 htable_bits;		/* size of hash table == 2^htable_bits */
 	u32 maxelem;		/* Maxelem per region */
+	struct list_head ad;	/* Resize add|del backlist */
 	struct ip_set_region *hregion;	/* Region locks and ext sizes */
 	struct hbucket __rcu *bucket[]; /* hashtable buckets */
 };
@@ -302,7 +303,6 @@ struct htype {
 	u8 netmask;		/* netmask value for subnets to store */
 	union nf_inet_addr bitmask;	/* stores bitmask */
 #endif
-	struct list_head ad;	/* Resize add|del backlist */
 	struct mtype_elem next; /* temporary storage for uadd */
 #ifdef IP_SET_HASH_WITH_NETS
 	struct net_prefixes nets[NLEN]; /* book-keeping of prefixes */
@@ -452,13 +452,14 @@ static void
 mtype_destroy(struct ip_set *set)
 {
 	struct htype *h = set->data;
+	struct htable *t = (__force struct htable *)h->table;
 	struct list_head *l, *lt;
 
-	mtype_ahash_destroy(set, (__force struct htable *)h->table, true);
-	list_for_each_safe(l, lt, &h->ad) {
+	list_for_each_safe(l, lt, &t->ad) {
 		list_del(l);
 		kfree(l);
 	}
+	mtype_ahash_destroy(set, t, true);
 	kfree(h);
 
 	set->data = NULL;
@@ -672,6 +673,7 @@ mtype_resize(struct ip_set *set, bool retried)
 	}
 	t->htable_bits = htable_bits;
 	t->maxelem = h->maxelem / ahash_numof_locks(htable_bits);
+	INIT_LIST_HEAD(&t->ad);
 	for (i = 0; i < ahash_numof_locks(htable_bits); i++)
 		spin_lock_init(&t->hregion[i].lock);
 
@@ -774,7 +776,7 @@ mtype_resize(struct ip_set *set, bool retried)
 	 * Kernel-side add cannot trigger a resize and userspace actions
 	 * are serialized by the mutex.
 	 */
-	list_for_each_safe(l, lt, &h->ad) {
+	list_for_each_safe(l, lt, &orig->ad) {
 		x = list_entry(l, struct mtype_resize_ad, list);
 		if (x->ad == IPSET_ADD) {
 			mtype_add(set, &x->d, &x->ext, &x->mext, x->flags);
@@ -801,10 +803,21 @@ mtype_resize(struct ip_set *set, bool retried)
 	spin_lock_bh(&h->gc.lock);
 	orig->resizing = false;
 	spin_unlock_bh(&h->gc.lock);
+	/* Make sure parallel readers see that orig->resizing is false
+	 * before we decrement uref */
+	synchronize_rcu();
 	atomic_dec(&orig->uref);
 	mtype_ahash_destroy(set, t, false);
 	if (ret == -EAGAIN)
 		goto retry;
+
+	/* Cleanup the backlog of ADD/DEL elements */
+	spin_lock_bh(&set->lock);
+	list_for_each_safe(l, lt, &orig->ad) {
+		list_del(l);
+		kfree(l);
+	}
+	spin_unlock_bh(&set->lock);
 	goto out;
 
 hbwarn:
@@ -1022,7 +1035,7 @@ mtype_add(struct ip_set *set, void *value, const struct ip_set_ext *ext,
 		memcpy(&x->mext, mext, sizeof(struct ip_set_ext));
 		x->flags = flags;
 		spin_lock_bh(&set->lock);
-		list_add_tail(&x->list, &h->ad);
+		list_add_tail(&x->list, &t->ad);
 		spin_unlock_bh(&set->lock);
 	}
 	goto out;
@@ -1146,7 +1159,7 @@ mtype_del(struct ip_set *set, void *value, const struct ip_set_ext *ext,
 	spin_unlock_bh(&t->hregion[r].lock);
 	if (x) {
 		spin_lock_bh(&set->lock);
-		list_add(&x->list, &h->ad);
+		list_add(&x->list, &t->ad);
 		spin_unlock_bh(&set->lock);
 	}
 	if (atomic_dec_and_test(&t->uref) && t->resizing) {
@@ -1625,9 +1638,8 @@ IPSET_TOKEN(HTYPE, _create)(struct net *net, struct ip_set *set,
 	}
 	t->htable_bits = hbits;
 	t->maxelem = h->maxelem / ahash_numof_locks(hbits);
+	INIT_LIST_HEAD(&t->ad);
 	RCU_INIT_POINTER(h->table, t);
-
-	INIT_LIST_HEAD(&h->ad);
 	set->data = h;
 #ifndef IP_SET_PROTO_UNDEF
 	if (set->family == NFPROTO_IPV4) {
Signature · 1 line
2.39.5
#4 lore raw

PATCH 4/5 netfilter: ipset: allocate the proper memory for the generic hash structure

In reply to Jozsef Kadlecsik

Because a single create function is emitted for every hash type, from the IPv4 and IPv6 generic hash structure definitions the last one, i.e. the IPv6 was in effect for IPv4 too. Use the proper size when allocating the structure. Comment properly that because create() refers to elements of the generic hash structure, all referred ones must come before the IPv4/IPv6 dependent 'next' member.

---
 net/netfilter/ipset/ip_set_hash_gen.h | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)
net/netfilter/ipset/ip_set_hash_gen.h +11 -2
index c0132d0f4cc0..8231317b0f1f 100644
@@ -303,10 +303,13 @@ struct htype {
 	u8 netmask;		/* netmask value for subnets to store */
 	union nf_inet_addr bitmask;	/* stores bitmask */
 #endif
-	struct mtype_elem next; /* temporary storage for uadd */
 #ifdef IP_SET_HASH_WITH_NETS
 	struct net_prefixes nets[NLEN]; /* book-keeping of prefixes */
 #endif
+	/* Because 'next' is IPv4/IPv6 dependent, no elements of this
+	 * structure and referred in create() may come after 'next'.
+	 */
+	struct mtype_elem next; /* temporary storage for uadd */
 };
 
 /* ADD|DEL entries saved during resize */
@@ -1584,7 +1587,13 @@ IPSET_TOKEN(HTYPE, _create)(struct net *net, struct ip_set *set,
 	if (tb[IPSET_ATTR_MAXELEM])
 		maxelem = ip_set_get_h32(tb[IPSET_ATTR_MAXELEM]);
 
-	hsize = sizeof(*h);
+#ifdef IP_SET_PROTO_UNDEF
+	hsize = sizeof(struct htype);
+#else
+	hsize = set->family == NFPROTO_IPV6 ?
+		sizeof(struct IPSET_TOKEN(HTYPE, 6)) :
+		sizeof(struct IPSET_TOKEN(HTYPE, 4));
+#endif
 	h = kzalloc(hsize, GFP_KERNEL);
 	if (!h)
 		return -ENOMEM;
Signature · 1 line
2.39.5
#5 lore raw

PATCH 5/5 netfilter: ipset: rework cidr bookkeeping

In reply to Jozsef Kadlecsik

According to sashiko, the current bookkeeping of cidr values are unsafe on weakly-ordered architectures. Replace the in-place updating with an RCU based method: create the new bookeeping structure, update and replace the old one with the new. Downside that we need to allocate memory when deleting a cidr entry - in case of memory pressure fall back to leave holes which possibility is taken into account at evaluation time.

Thanks to Pablo (Pablo Neira Ayuso <[email protected]>) and Cyntia (Cynthia <[email protected]>) for helping me in debugging which resulted the patch "netfilter: ipset: allocate the proper memory for the generic hash structure" on which this very patch depends.

---
 net/netfilter/ipset/ip_set_hash_gen.h        | 228 ++++++++++++-------
 net/netfilter/ipset/ip_set_hash_ipportnet.c  |   4 +-
 net/netfilter/ipset/ip_set_hash_net.c        |   4 +-
 net/netfilter/ipset/ip_set_hash_netiface.c   |   4 +-
 net/netfilter/ipset/ip_set_hash_netnet.c     |  12 +-
 net/netfilter/ipset/ip_set_hash_netport.c    |   4 +-
 net/netfilter/ipset/ip_set_hash_netportnet.c |  12 +-
 7 files changed, 175 insertions(+), 93 deletions(-)
net/netfilter/ipset/ip_set_hash_gen.h +151 -77
index 8231317b0f1f..2b473acad50c 100644
@@ -99,9 +99,16 @@ struct htable {
 #endif
 
 /* Book-keeping of the prefixes added to the set */
+struct net_prefix {
+	u8 cidr;			/* the cidr value */
+	u32 count;			/* number of elements of this cidr */
+};
+
 struct net_prefixes {
-	u32 nets[IPSET_NET_COUNT]; /* number of elements for this cidr */
-	u8 cidr[IPSET_NET_COUNT];  /* the cidr value */
+	struct rcu_head rcu;
+	u8 len;
+	struct net_prefix nets[]
+		__aligned(__alignof__(u64));
 };
 
 /* Compute the hash table size */
@@ -127,11 +134,6 @@ htable_size(u8 hbits)
 #else
 #define __CIDR(cidr, i)		(cidr)
 #endif
-
-/* cidr + 1 is stored in net_prefixes to support /0 */
-#define NCIDR_PUT(cidr)		((cidr) + 1)
-#define NCIDR_GET(cidr)		((cidr) - 1)
-
 #ifdef IP_SET_HASH_WITH_NETS_PACKED
 /* When cidr is packed with nomatch, cidr - 1 is stored in the data entry */
 #define DCIDR_PUT(cidr)		((cidr) - 1)
@@ -141,21 +143,9 @@ htable_size(u8 hbits)
 #define DCIDR_GET(cidr, i)	__CIDR(cidr, i)
 #endif
 
-#define INIT_CIDR(cidr, host_mask)	\
-	DCIDR_PUT(((cidr) ? NCIDR_GET(cidr) : host_mask))
-
-#ifdef IP_SET_HASH_WITH_NET0
-/* cidr from 0 to HOST_MASK value and c = cidr + 1 */
-#define NLEN			(HOST_MASK + 1)
-#define CIDR_POS(c)		((c) - 1)
-#else
-/* cidr from 1 to HOST_MASK value and c = cidr + 1 */
-#define NLEN			HOST_MASK
-#define CIDR_POS(c)		((c) - 2)
-#endif
+#define INIT_CIDR(n, host_mask)	\
+	DCIDR_PUT((n)->len ? (n)->nets[0].cidr : host_mask)
 
-#else
-#define NLEN			0
 #endif /* IP_SET_HASH_WITH_NETS */
 
 #define SET_ELEM_EXPIRED(set, d)	\
@@ -292,6 +282,7 @@ static const union nf_inet_addr zeromask = {};
 /* The generic hash structure */
 struct htype {
 	struct htable __rcu *table; /* the hash table */
+	struct net_prefixes __rcu *rnets[IPSET_NET_COUNT]; /* cidr prefixes */
 	struct htable_gc gc;	/* gc workqueue */
 	u32 maxelem;		/* max elements in the hash */
 	u32 initval;		/* random jhash init value */
@@ -302,9 +293,6 @@ struct htype {
 #if defined(IP_SET_HASH_WITH_NETMASK) || defined(IP_SET_HASH_WITH_BITMASK)
 	u8 netmask;		/* netmask value for subnets to store */
 	union nf_inet_addr bitmask;	/* stores bitmask */
-#endif
-#ifdef IP_SET_HASH_WITH_NETS
-	struct net_prefixes nets[NLEN]; /* book-keeping of prefixes */
 #endif
 	/* Because 'next' is IPv4/IPv6 dependent, no elements of this
 	 * structure and referred in create() may come after 'next'.
@@ -326,50 +314,88 @@ struct mtype_resize_ad {
 /* Network cidr size book keeping when the hash stores different
  * sized networks. cidr == real cidr + 1 to support /0.
  */
-static void
+static int
 mtype_add_cidr(struct ip_set *set, struct htype *h, u8 cidr, u8 n)
 {
-	int i, j;
+	struct net_prefixes *nets, *tmp;
+	int i, j, found, len = 0, ret = 0;
 
 	spin_lock_bh(&set->lock);
+	nets = __ipset_dereference(h->rnets[n]);
 	/* Add in increasing prefix order, so larger cidr first */
-	for (i = 0, j = -1; i < NLEN && h->nets[i].cidr[n]; i++) {
-		if (j != -1) {
+	for (i = 0, found = -1; i < nets->len; i++) {
+		if (nets->nets[i].count)
+			len++;
+		if (found != -1) {
 			continue;
-		} else if (h->nets[i].cidr[n] < cidr) {
-			j = i;
-		} else if (h->nets[i].cidr[n] == cidr) {
-			h->nets[CIDR_POS(cidr)].nets[n]++;
+		} else if (nets->nets[i].cidr < cidr) {
+			found = i;
+		} else if (nets->nets[i].cidr == cidr) {
+			nets->nets[i].count++;
 			goto unlock;
 		}
 	}
-	if (j != -1) {
-		for (; i > j; i--)
-			h->nets[i].cidr[n] = h->nets[i - 1].cidr[n];
+	len++;
+	tmp = kzalloc(sizeof(struct net_prefixes) +
+		      len * sizeof(struct net_prefix), GFP_ATOMIC);
+	if (!tmp)
+		return -ENOMEM;
+	tmp->len = len;
+	for (i = 0, j = 0; i < nets->len; i++) {
+		if (!nets->nets[i].count)
+			continue;
+		if (i == found) {
+			tmp->nets[j].cidr = cidr;
+			tmp->nets[j++].count = 1;
+		}
+		tmp->nets[j].cidr = nets->nets[i].cidr;
+		tmp->nets[j++].count = nets->nets[i].count;
+	}
+	if (found == -1) {
+		tmp->nets[j].cidr = cidr;
+		tmp->nets[j].count = 1;
 	}
-	h->nets[i].cidr[n] = cidr;
-	h->nets[CIDR_POS(cidr)].nets[n] = 1;
+	rcu_assign_pointer(h->rnets[n], tmp);
+	kfree_rcu(nets, rcu);
 unlock:
 	spin_unlock_bh(&set->lock);
+	return ret;
 }
 
 static void
 mtype_del_cidr(struct ip_set *set, struct htype *h, u8 cidr, u8 n)
 {
-	u8 i, j, net_end = NLEN - 1;
+	struct net_prefixes *nets, *tmp;
+	u8 i, j, found, len = 0;
 
 	spin_lock_bh(&set->lock);
-	for (i = 0; i < NLEN; i++) {
-		if (h->nets[i].cidr[n] != cidr)
-			continue;
-		h->nets[CIDR_POS(cidr)].nets[n]--;
-		if (h->nets[CIDR_POS(cidr)].nets[n] > 0)
-			goto unlock;
-		for (j = i; j < net_end && h->nets[j].cidr[n]; j++)
-			h->nets[j].cidr[n] = h->nets[j + 1].cidr[n];
-		h->nets[j].cidr[n] = 0;
+	nets = __ipset_dereference(h->rnets[n]);
+	for (i = 0, found = -1; i < nets->len; i++) {
+		if (nets->nets[i].count)
+			len++;
+		if (nets->nets[i].cidr == cidr)
+			found = i;
+	}
+	if (unlikely(found == -1))
+		return;
+	nets->nets[found].count--;
+	if (nets->nets[found].count)
 		goto unlock;
+	len--;
+	tmp = kzalloc(sizeof(struct net_prefixes) +
+		      len * sizeof(struct net_prefix), GFP_ATOMIC);
+	if (!tmp)
+		/* Leave a hole */
+		return;
+	tmp->len = len;
+	for (i = 0, j = 0; i < nets->len; i++) {
+		if (!nets->nets[i].count || i == found)
+			continue;
+		tmp->nets[j].cidr = nets->nets[i].cidr;
+		tmp->nets[j++].count = nets->nets[i].count;
 	}
+	rcu_assign_pointer(h->rnets[n], tmp);
+	kfree_rcu(nets, rcu);
 unlock:
 	spin_unlock_bh(&set->lock);
 }
@@ -402,6 +428,9 @@ static void
 mtype_flush(struct ip_set *set)
 {
 	struct htype *h = set->data;
+#ifdef IP_SET_HASH_WITH_NETS
+	struct net_prefixes *nets, *tmp;
+#endif
 	struct htable *t;
 	struct hbucket *n;
 	u32 r, i;
@@ -425,7 +454,17 @@ mtype_flush(struct ip_set *set)
 		spin_unlock_bh(&t->hregion[r].lock);
 	}
 #ifdef IP_SET_HASH_WITH_NETS
-	memset(h->nets, 0, sizeof(h->nets));
+	for (i = 0; i < IPSET_NET_COUNT; i++) {
+		nets = ipset_dereference_nfnl(h->rnets[i]);
+		tmp = kzalloc(sizeof(struct net_prefixes), GFP_ATOMIC);
+		if (!tmp) {
+			for (i = 0; i < nets->len; i++)
+				nets->nets[i].count = 0;
+		} else {
+			rcu_assign_pointer(h->rnets[i], tmp);
+			kfree_rcu(nets, rcu);
+		}
+	}
 #endif
 }
 
@@ -433,6 +472,9 @@ mtype_flush(struct ip_set *set)
 static void
 mtype_ahash_destroy(struct ip_set *set, struct htable *t, bool ext_destroy)
 {
+#ifdef IP_SET_HASH_WITH_NETS
+	struct htype *h = set->data;
+#endif
 	struct hbucket *n;
 	u32 i;
 
@@ -446,6 +488,11 @@ mtype_ahash_destroy(struct ip_set *set, struct htable *t, bool ext_destroy)
 		kfree(n);
 	}
 
+#ifdef IP_SET_HASH_WITH_NETS
+	if (ext_destroy)
+		for (i = 0; i < IPSET_NET_COUNT; i++)
+			kfree(h->rnets[i]);
+#endif
 	ip_set_free(t->hregion);
 	ip_set_free(t);
 }
@@ -519,8 +566,7 @@ mtype_gc_do(struct ip_set *set, struct htype *h, struct htable *t, u32 r)
 #ifdef IP_SET_HASH_WITH_NETS
 			for (k = 0; k < IPSET_NET_COUNT; k++)
 				mtype_del_cidr(set, h,
-					NCIDR_PUT(DCIDR_GET(data->cidr, k)),
-					k);
+					DCIDR_GET(data->cidr, k), k);
 #endif
 			t->hregion[r].elements--;
 			ip_set_ext_destroy(set, data);
@@ -950,8 +996,7 @@ mtype_add(struct ip_set *set, void *value, const struct ip_set_ext *ext,
 #ifdef IP_SET_HASH_WITH_NETS
 			for (i = 0; i < IPSET_NET_COUNT; i++)
 				mtype_del_cidr(set, h,
-					NCIDR_PUT(DCIDR_GET(data->cidr, i)),
-					i);
+					DCIDR_GET(data->cidr, i), i);
 #endif
 			ip_set_ext_destroy(set, data);
 			t->hregion[r].elements--;
@@ -996,7 +1041,7 @@ mtype_add(struct ip_set *set, void *value, const struct ip_set_ext *ext,
 	t->hregion[r].elements++;
 #ifdef IP_SET_HASH_WITH_NETS
 	for (i = 0; i < IPSET_NET_COUNT; i++)
-		mtype_add_cidr(set, h, NCIDR_PUT(DCIDR_GET(d->cidr, i)), i);
+		mtype_add_cidr(set, h, DCIDR_GET(d->cidr, i), i);
 #endif
 	memcpy(data, d, sizeof(struct mtype_elem));
 overwrite_extensions:
@@ -1110,7 +1155,7 @@ mtype_del(struct ip_set *set, void *value, const struct ip_set_ext *ext,
 #ifdef IP_SET_HASH_WITH_NETS
 		for (j = 0; j < IPSET_NET_COUNT; j++)
 			mtype_del_cidr(set, h,
-				       NCIDR_PUT(DCIDR_GET(d->cidr, j)), j);
+				DCIDR_GET(d->cidr, j), j);
 #endif
 		ip_set_ext_destroy(set, data);
 
@@ -1193,28 +1238,37 @@ mtype_test_cidrs(struct ip_set *set, struct mtype_elem *d,
 {
 	struct htype *h = set->data;
 	struct htable *t = rcu_dereference_bh(h->table);
+	struct net_prefixes *nets0;
 	struct hbucket *n;
 	struct mtype_elem *data;
 #if IPSET_NET_COUNT == 2
+	struct net_prefixes *nets1;
 	struct mtype_elem orig = *d;
-	int ret, i, j = 0, k;
+	int ret, i, j, k;
 #else
-	int ret, i, j = 0;
+	int ret, i, j;
 #endif
 	u32 key, multi = 0;
 	u8 pos;
 
 	pr_debug("test by nets\n");
-	for (; j < NLEN && h->nets[j].cidr[0] && !multi; j++) {
+	rcu_read_lock_bh();
+	nets0 = rcu_dereference_bh(h->rnets[0]);
+#if IPSET_NET_COUNT == 2
+	nets1 = rcu_dereference_bh(h->rnets[1]);
+#endif
+	for (j = 0; j < nets0->len && !multi; j++) {
+		if (!nets0->nets[j].count)
+			continue;
 #if IPSET_NET_COUNT == 2
 		mtype_data_reset_elem(d, &orig);
-		mtype_data_netmask(d, NCIDR_GET(h->nets[j].cidr[0]), false);
-		for (k = 0; k < NLEN && h->nets[k].cidr[1] && !multi;
-		     k++) {
-			mtype_data_netmask(d, NCIDR_GET(h->nets[k].cidr[1]),
-					   true);
+		mtype_data_netmask(d, nets0->nets[j].cidr, false);
+		for (k = 0; k < nets1->len && !multi; k++) {
+			if (!nets1->nets[k].count)
+				continue;
+			mtype_data_netmask(d, nets1->nets[k].cidr, true);
 #else
-		mtype_data_netmask(d, NCIDR_GET(h->nets[j].cidr[0]));
+		mtype_data_netmask(d, nets0->nets[j].cidr);
 #endif
 		key = HKEY(d, h->initval, t->htable_bits);
 		n = rcu_dereference_bh(hbucket(t, key));
@@ -1229,7 +1283,7 @@ mtype_test_cidrs(struct ip_set *set, struct mtype_elem *d,
 				continue;
 			ret = mtype_data_match(data, ext, mext, set, flags);
 			if (ret != 0)
-				return ret;
+				goto unlock;
 #ifdef IP_SET_HASH_WITH_MULTI
 			/* No match, reset multiple match flag */
 			multi = 0;
@@ -1239,7 +1293,10 @@ mtype_test_cidrs(struct ip_set *set, struct mtype_elem *d,
 		}
 #endif
 	}
-	return 0;
+	ret = 0;
+unlock:
+	rcu_read_unlock_bh();
+	return ret;
 }
 #endif
 
@@ -1504,6 +1561,9 @@ IPSET_TOKEN(HTYPE, _create)(struct net *net, struct ip_set *set,
 	int ret __attribute__((unused)) = 0;
 	u8 netmask = set->family == NFPROTO_IPV4 ? 32 : 128;
 	union nf_inet_addr bitmask = onesmask;
+#endif
+#ifdef IP_SET_HASH_WITH_NETS
+	struct net_prefixes *nets;
 #endif
 	size_t hsize;
 	struct htype *h;
@@ -1604,21 +1664,25 @@ IPSET_TOKEN(HTYPE, _create)(struct net *net, struct ip_set *set,
 	 */
 	hbits = fls(hashsize - 1);
 	hsize = htable_size(hbits);
-	if (hsize == 0) {
-		kfree(h);
-		return -ENOMEM;
-	}
+	if (hsize == 0)
+		goto free_h;
 	t = ip_set_alloc(hsize);
-	if (!t) {
-		kfree(h);
-		return -ENOMEM;
-	}
+	if (!t)
+		goto free_h;
 	t->hregion = ip_set_alloc(ahash_sizeof_regions(hbits));
-	if (!t->hregion) {
-		ip_set_free(t);
-		kfree(h);
-		return -ENOMEM;
+	if (!t->hregion)
+		goto free_t;
+#ifdef IP_SET_HASH_WITH_NETS
+	for (i = 0; i < IPSET_NET_COUNT; i++) {
+		nets = kzalloc(sizeof(struct net_prefixes), GFP_KERNEL);
+		if (!nets) {
+			if (i != 0)
+				kfree(h->rnets[0]);
+			goto free_hregion;
+		}
+		RCU_INIT_POINTER(h->rnets[i], nets);
 	}
+#endif
 	h->gc.set = set;
 	spin_lock_init(&h->gc.lock);
 	for (i = 0; i < ahash_numof_locks(hbits); i++)
@@ -1682,6 +1746,16 @@ IPSET_TOKEN(HTYPE, _create)(struct net *net, struct ip_set *set,
 		 t->htable_bits, h->maxelem, set->data, t);
 
 	return 0;
+
+#ifdef IP_SET_HASH_WITH_NETS
+free_hregion:
+	ip_set_free(t->hregion);
+#endif
+free_t:
+	ip_set_free(t);
+free_h:
+	kfree(h);
+	return -ENOMEM;
 }
 #endif /* IP_SET_EMIT_CREATE */
 
net/netfilter/ipset/ip_set_hash_ipportnet.c +2 -2
index 2d6652d43199..195853a25b06 100644
@@ -138,7 +138,7 @@ hash_ipportnet4_kadt(struct ip_set *set, const struct sk_buff *skb,
 	const struct hash_ipportnet4 *h = set->data;
 	ipset_adtfn adtfn = set->variant->adt[adt];
 	struct hash_ipportnet4_elem e = {
-		.cidr = INIT_CIDR(h->nets[0].cidr[0], HOST_MASK),
+		.cidr = INIT_CIDR(h->rnets[0], HOST_MASK),
 	};
 	struct ip_set_ext ext = IP_SET_INIT_KEXT(skb, opt, set);
 
@@ -398,7 +398,7 @@ hash_ipportnet6_kadt(struct ip_set *set, const struct sk_buff *skb,
 	const struct hash_ipportnet6 *h = set->data;
 	ipset_adtfn adtfn = set->variant->adt[adt];
 	struct hash_ipportnet6_elem e = {
-		.cidr = INIT_CIDR(h->nets[0].cidr[0], HOST_MASK),
+		.cidr = INIT_CIDR(h->rnets[0], HOST_MASK),
 	};
 	struct ip_set_ext ext = IP_SET_INIT_KEXT(skb, opt, set);
 
net/netfilter/ipset/ip_set_hash_net.c +2 -2
index ce0a9ce5a91f..092f3c9281b8 100644
@@ -117,7 +117,7 @@ hash_net4_kadt(struct ip_set *set, const struct sk_buff *skb,
 	const struct hash_net4 *h = set->data;
 	ipset_adtfn adtfn = set->variant->adt[adt];
 	struct hash_net4_elem e = {
-		.cidr = INIT_CIDR(h->nets[0].cidr[0], HOST_MASK),
+		.cidr = INIT_CIDR(h->rnets[0], HOST_MASK),
 	};
 	struct ip_set_ext ext = IP_SET_INIT_KEXT(skb, opt, set);
 
@@ -291,7 +291,7 @@ hash_net6_kadt(struct ip_set *set, const struct sk_buff *skb,
 	const struct hash_net6 *h = set->data;
 	ipset_adtfn adtfn = set->variant->adt[adt];
 	struct hash_net6_elem e = {
-		.cidr = INIT_CIDR(h->nets[0].cidr[0], HOST_MASK),
+		.cidr = INIT_CIDR(h->rnets[0], HOST_MASK),
 	};
 	struct ip_set_ext ext = IP_SET_INIT_KEXT(skb, opt, set);
 
net/netfilter/ipset/ip_set_hash_netiface.c +2 -2
index 30a655e5c4fd..b44b95f766b7 100644
@@ -161,7 +161,7 @@ hash_netiface4_kadt(struct ip_set *set, const struct sk_buff *skb,
 	struct hash_netiface4 *h = set->data;
 	ipset_adtfn adtfn = set->variant->adt[adt];
 	struct hash_netiface4_elem e = {
-		.cidr = INIT_CIDR(h->nets[0].cidr[0], HOST_MASK),
+		.cidr = INIT_CIDR(h->rnets[0], HOST_MASK),
 		.elem = 1,
 	};
 	struct ip_set_ext ext = IP_SET_INIT_KEXT(skb, opt, set);
@@ -382,7 +382,7 @@ hash_netiface6_kadt(struct ip_set *set, const struct sk_buff *skb,
 	struct hash_netiface6 *h = set->data;
 	ipset_adtfn adtfn = set->variant->adt[adt];
 	struct hash_netiface6_elem e = {
-		.cidr = INIT_CIDR(h->nets[0].cidr[0], HOST_MASK),
+		.cidr = INIT_CIDR(h->rnets[0], HOST_MASK),
 		.elem = 1,
 	};
 	struct ip_set_ext ext = IP_SET_INIT_KEXT(skb, opt, set);
net/netfilter/ipset/ip_set_hash_netnet.c +8 -4
index 8fbe649c9dd3..f7c8a1cc30fc 100644
@@ -149,8 +149,10 @@ hash_netnet4_kadt(struct ip_set *set, const struct sk_buff *skb,
 	struct hash_netnet4_elem e = { };
 	struct ip_set_ext ext = IP_SET_INIT_KEXT(skb, opt, set);
 
-	e.cidr[0] = INIT_CIDR(h->nets[0].cidr[0], HOST_MASK);
-	e.cidr[1] = INIT_CIDR(h->nets[0].cidr[1], HOST_MASK);
+	rcu_read_lock_bh();
+	e.cidr[0] = INIT_CIDR(h->rnets[0], HOST_MASK);
+	e.cidr[1] = INIT_CIDR(h->rnets[1], HOST_MASK);
+	rcu_read_unlock_bh();
 	if (adt == IPSET_TEST)
 		e.ccmp = (HOST_MASK << (sizeof(e.cidr[0]) * 8)) | HOST_MASK;
 
@@ -388,8 +390,10 @@ hash_netnet6_kadt(struct ip_set *set, const struct sk_buff *skb,
 	struct hash_netnet6_elem e = { };
 	struct ip_set_ext ext = IP_SET_INIT_KEXT(skb, opt, set);
 
-	e.cidr[0] = INIT_CIDR(h->nets[0].cidr[0], HOST_MASK);
-	e.cidr[1] = INIT_CIDR(h->nets[0].cidr[1], HOST_MASK);
+	rcu_read_lock_bh();
+	e.cidr[0] = INIT_CIDR(h->rnets[0], HOST_MASK);
+	e.cidr[1] = INIT_CIDR(h->rnets[1], HOST_MASK);
+	rcu_read_unlock_bh();
 	if (adt == IPSET_TEST)
 		e.ccmp = (HOST_MASK << (sizeof(u8) * 8)) | HOST_MASK;
 
net/netfilter/ipset/ip_set_hash_netport.c +2 -2
index d1a0628df4ef..5de4b511de76 100644
@@ -133,7 +133,7 @@ hash_netport4_kadt(struct ip_set *set, const struct sk_buff *skb,
 	const struct hash_netport4 *h = set->data;
 	ipset_adtfn adtfn = set->variant->adt[adt];
 	struct hash_netport4_elem e = {
-		.cidr = INIT_CIDR(h->nets[0].cidr[0], HOST_MASK),
+		.cidr = INIT_CIDR(h->rnets[0], HOST_MASK),
 	};
 	struct ip_set_ext ext = IP_SET_INIT_KEXT(skb, opt, set);
 
@@ -353,7 +353,7 @@ hash_netport6_kadt(struct ip_set *set, const struct sk_buff *skb,
 	const struct hash_netport6 *h = set->data;
 	ipset_adtfn adtfn = set->variant->adt[adt];
 	struct hash_netport6_elem e = {
-		.cidr = INIT_CIDR(h->nets[0].cidr[0], HOST_MASK),
+		.cidr = INIT_CIDR(h->rnets[0], HOST_MASK),
 	};
 	struct ip_set_ext ext = IP_SET_INIT_KEXT(skb, opt, set);
 
net/netfilter/ipset/ip_set_hash_netportnet.c +8 -4
index bf4f91b78e1d..6291532be7a5 100644
@@ -157,8 +157,10 @@ hash_netportnet4_kadt(struct ip_set *set, const struct sk_buff *skb,
 	struct hash_netportnet4_elem e = { };
 	struct ip_set_ext ext = IP_SET_INIT_KEXT(skb, opt, set);
 
-	e.cidr[0] = INIT_CIDR(h->nets[0].cidr[0], HOST_MASK);
-	e.cidr[1] = INIT_CIDR(h->nets[0].cidr[1], HOST_MASK);
+	rcu_read_lock_bh();
+	e.cidr[0] = INIT_CIDR(h->rnets[0], HOST_MASK);
+	e.cidr[1] = INIT_CIDR(h->rnets[1], HOST_MASK);
+	rcu_read_unlock_bh();
 	if (adt == IPSET_TEST)
 		e.ccmp = (HOST_MASK << (sizeof(e.cidr[0]) * 8)) | HOST_MASK;
 
@@ -452,8 +454,10 @@ hash_netportnet6_kadt(struct ip_set *set, const struct sk_buff *skb,
 	struct hash_netportnet6_elem e = { };
 	struct ip_set_ext ext = IP_SET_INIT_KEXT(skb, opt, set);
 
-	e.cidr[0] = INIT_CIDR(h->nets[0].cidr[0], HOST_MASK);
-	e.cidr[1] = INIT_CIDR(h->nets[0].cidr[1], HOST_MASK);
+	rcu_read_lock_bh();
+	e.cidr[0] = INIT_CIDR(h->rnets[0], HOST_MASK);
+	e.cidr[1] = INIT_CIDR(h->rnets[1], HOST_MASK);
+	rcu_read_unlock_bh();
 	if (adt == IPSET_TEST)
 		e.ccmp = (HOST_MASK << (sizeof(u8) * 8)) | HOST_MASK;
 
Signature · 1 line
2.39.5