Skip to contentSkip to conversation

netfilter: flowtable: add and use NF_FLOW_CONFIRMED bit to skip GC

PATCH New Pablo Neira Ayuso 1 participant 4 messages Aug 17

Subsystems (3)
lore
Share & tools
lore

Pablo Neira Ayuso

author maintainer
#0 lore raw

PATCH v3 1/4 netfilter: flowtable: add and use NF_FLOW_CONFIRMED bit to skip GC

This new NF_FLOW_CONFIRMED bit is set on once the flow entry has fully been set up, so it can be exposed to the GC.

Apparently, there is a narrow window where GC might walk over a flow entry while it is in a partial state, ie. flow entry has been inserted in the hashes but the NF_FLOW_HW has not been set up yet. While packets are flowing in the software datapath that could teardown the flow.

This reverts commit 2014ac62df9d ("netfilter: flowtable: publish GC-visible tuple last") since reversing the tuple order insertion only mitigates the issue for one scenario.

Note that the flowtable iterators can still find reach flows in partial state. This is fine because they are used for cleanup scenario (ie. netdevice is gone) where the teardown bit is set on. On flowtable removal, no flow is seen in partial state because the flow_offload action already does not see packets (ie. flowtable in detached from rule).

The datapath lookup function still can find the flow in partial state, this is addressed by a follow up patch which adds a check to skip expired flows.

Fixes 2014ac62df9d ("netfilter: flowtable: publish GC-visible tuple last")

---
v3: - remove check from lookup path
- add smp_mb__before_atomic() before set_bit(NF_FLOW_CONFIRMED, ...

 include/net/netfilter/nf_flow_table.h |  1 +
 net/netfilter/nf_flow_table_core.c    | 16 +++++++++++-----
 2 files changed, 12 insertions(+), 5 deletions(-)
include/net/netfilter/nf_flow_table.h +1 -0
index ce414118962f..618d26e49fdf 100644
@@ -189,6 +189,7 @@ enum nf_flow_flags {
 	NF_FLOW_HW_PENDING,
 	NF_FLOW_HW_BIDIRECTIONAL,
 	NF_FLOW_HW_ESTABLISHED,
+	NF_FLOW_CONFIRMED,
 };
 
 enum flow_offload_type {
net/netfilter/nf_flow_table_core.c +11 -5
index 8b1165f2b5a4..cbb56ca94717 100644
@@ -332,18 +332,17 @@ int flow_offload_add(struct nf_flowtable *flow_table, struct flow_offload *flow)
 	flow->timeout = nf_flowtable_time_stamp + flow_offload_get_timeout(flow);
 
 	err = rhashtable_insert_fast(&flow_table->rhashtable,
-				     &flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].node,
+				     &flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].node,
 				     nf_flow_offload_rhash_params);
 	if (err < 0)
 		return err;
 
-	/* GC only iterates original-direction entries; publish original last. */
 	err = rhashtable_insert_fast(&flow_table->rhashtable,
-				     &flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].node,
+				     &flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].node,
 				     nf_flow_offload_rhash_params);
 	if (err < 0) {
 		rhashtable_remove_fast(&flow_table->rhashtable,
-				       &flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].node,
+				       &flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].node,
 				       nf_flow_offload_rhash_params);
 		return err;
 	}
@@ -353,6 +352,9 @@ int flow_offload_add(struct nf_flowtable *flow_table, struct flow_offload *flow)
 	if (nf_flowtable_hw_offload(flow_table))
 		nf_flow_offload_add(flow_table, flow);
 
+	smp_mb__before_atomic();
+	set_bit(NF_FLOW_CONFIRMED, &flow->flags);
+
 	return 0;
 }
 EXPORT_SYMBOL_GPL(flow_offload_add);
@@ -568,8 +570,12 @@ static void nf_flow_table_extend_ct_timeout(struct nf_conn *ct)
 static void nf_flow_offload_gc_step(struct nf_flowtable *flow_table,
 				    struct flow_offload *flow, void *data)
 {
-	bool teardown = test_bit(NF_FLOW_TEARDOWN, &flow->flags);
+	bool teardown;
+
+	if (!test_bit(NF_FLOW_CONFIRMED, &flow->flags))
+		return;
 
+	teardown = test_bit(NF_FLOW_TEARDOWN, &flow->flags);
 	if (nf_flow_has_expired(flow) ||
 	    nf_ct_is_dying(flow->ct) ||
 	    nf_flow_custom_gc(flow_table, flow)) {
Signature · 1 line
2.47.3
author maintainer
#1 lore raw

PATCH v3 2/4 netfilter: flowtable: do not expose expired flows via lookup function

The lookup function currently relies on the teardown bit to skip flow entries that will be removed soon by the GC. However, datapath can still reach a flow that has expired without the teardown bit set on. This includes a flow that is being set up at this time whose timeout is still not fully initially.

Although the new NF_FLOW_CONFIRMED bit could be used to skip this flow entry in lookups, skipping a expired flow in the lookup is convenient. While it is safe to access such expired flow entry from datapath because of the rcu semantics, the flow->ct pointer for a released flow via rcu might point to a different conntrack object due to the typesafe rcu semantics. Note that nf_ct_put() called from flow_offload_free() releases a conntrack inmmediately.

This patch adds the missing READ_ONCE/WRITE_ONCE to update/access the flow entry timeout. Then, skip expired flow entries from the flow lookup function so they become stale inmediately even though the GC did not set on the teardown bit yet.

Fixes ac2a66665e23 ("netfilter: add generic flow table infrastructure")

---
v3: new in this series

 net/netfilter/nf_flow_table_core.c | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)
net/netfilter/nf_flow_table_core.c +6 -6
index cbb56ca94717..675d470ea581 100644
@@ -172,7 +172,7 @@ EXPORT_SYMBOL_GPL(flow_offload_route_init);
 
 static inline bool nf_flow_has_expired(const struct flow_offload *flow)
 {
-	return nf_flow_timeout_delta(flow->timeout) <= 0;
+	return nf_flow_timeout_delta(READ_ONCE(flow->timeout)) <= 0;
 }
 
 static void flow_offload_fixup_tcp(struct nf_conn *ct, u8 tcp_state)
@@ -329,7 +329,7 @@ int flow_offload_add(struct nf_flowtable *flow_table, struct flow_offload *flow)
 {
 	int err;
 
-	flow->timeout = nf_flowtable_time_stamp + flow_offload_get_timeout(flow);
+	WRITE_ONCE(flow->timeout, nf_flowtable_time_stamp);
 
 	err = rhashtable_insert_fast(&flow_table->rhashtable,
 				     &flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].node,
@@ -352,6 +352,7 @@ int flow_offload_add(struct nf_flowtable *flow_table, struct flow_offload *flow)
 	if (nf_flowtable_hw_offload(flow_table))
 		nf_flow_offload_add(flow_table, flow);
 
+	WRITE_ONCE(flow->timeout, flow->timeout + flow_offload_get_timeout(flow));
 	smp_mb__before_atomic();
 	set_bit(NF_FLOW_CONFIRMED, &flow->flags);
 
@@ -414,10 +415,9 @@ flow_offload_lookup(struct nf_flowtable *flow_table,
 
 	dir = tuplehash->tuple.dir;
 	flow = container_of(tuplehash, struct flow_offload, tuplehash[dir]);
-	if (test_bit(NF_FLOW_TEARDOWN, &flow->flags))
-		return NULL;
-
-	if (unlikely(nf_ct_is_dying(flow->ct)))
+	if (nf_flow_has_expired(flow) ||
+	    test_bit(NF_FLOW_TEARDOWN, &flow->flags) ||
+	    unlikely(nf_ct_is_dying(flow->ct)))
 		return NULL;
 
 	return tuplehash;
Signature · 1 line
2.47.3
author maintainer
#2 lore raw

PATCH v3 3/4 netfilter: flowtable: serialize IPS_OFFLOAD bit clearing

In reply to Pablo Neira Ayuso

Clear the IPS_OFFLOAD bit iif the teardown bit is unset to address a race between GC, the netdevice event handler (which also results in a lockless flowtable iteration to set on the teardown bit) and the datapath (which can teardown the flow too). Otherwise, it is possible to clear the offload bit of a different ct object that is recycling the same memory area due to typesafe rcu semantics.

After this patch, the flow_offload_teardown() function is turned into noop if the teardown bit has been already set on.

Fixes d2d31ea8cd80 ("netfilter: conntrack: fix erronous removal of offload bit")

---
v3: new in this series

 net/netfilter/nf_flow_table_core.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)
net/netfilter/nf_flow_table_core.c +3 -2
index 675d470ea581..8c84f02c4dd3 100644
@@ -394,9 +394,10 @@ static void flow_offload_del(struct nf_flowtable *flow_table,
 
 void flow_offload_teardown(struct flow_offload *flow)
 {
-	clear_bit(IPS_OFFLOAD_BIT, &flow->ct->status);
-	if (!test_and_set_bit(NF_FLOW_TEARDOWN, &flow->flags))
+	if (!test_and_set_bit(NF_FLOW_TEARDOWN, &flow->flags)) {
+		clear_bit(IPS_OFFLOAD_BIT, &flow->ct->status);
 		flow_offload_fixup_ct(flow);
+	}
 }
 EXPORT_SYMBOL_GPL(flow_offload_teardown);
 
Signature · 1 line
2.47.3
author maintainer
#3 lore raw

PATCH v3 4/4 netfilter: flowtable: let GC handle partially set up flow

In reply to Pablo Neira Ayuso

If the reply tuple cannot be inserted into the rhashtable, let GC remove the tuple in the original direction. Otherwise, the flowtable lockless rcu iterator might be already walking over such flow entry and access different recycled ct due to the typesafe rcu semantics.

The rhashtable_remove_fast() call in flow_offload_del() will just report ENOENT for the reply tuple.

Let the GC clear the offload bit when setting up the

While at it, remove WARN_ON_ONCE() which is reachable in case of OOM in tc act_ct.

Fixes ac2a66665e23 ("netfilter: add generic flow table infrastructure")

---
v3: new in this series

 net/netfilter/nf_flow_table_core.c |  8 +++++---
 net/netfilter/nft_flow_offload.c   |  8 +-------
 net/sched/act_ct.c                 | 12 ++----------
 3 files changed, 8 insertions(+), 20 deletions(-)
net/netfilter/nf_flow_table_core.c +5 -3
index 8c84f02c4dd3..27cbabc092d0 100644
@@ -341,9 +341,11 @@ int flow_offload_add(struct nf_flowtable *flow_table, struct flow_offload *flow)
 				     &flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].node,
 				     nf_flow_offload_rhash_params);
 	if (err < 0) {
-		rhashtable_remove_fast(&flow_table->rhashtable,
-				       &flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].node,
-				       nf_flow_offload_rhash_params);
+		/* Let GC handle this flow in partial state, confirm it so it
+		 * is handled as an expired flow. The rhashtable_remove_fast()
+		 * call will just report ENOENT for the reply tuple.
+		 */
+		set_bit(NF_FLOW_CONFIRMED, &flow->flags);
 		return err;
 	}
 
net/netfilter/nft_flow_offload.c +1 -7
index 32b4281038dd..54671006ce98 100644
@@ -59,7 +59,6 @@ static void nft_flow_offload_eval(const struct nft_expr *expr,
 	struct flow_offload *flow;
 	enum ip_conntrack_dir dir;
 	struct nf_conn *ct;
-	int ret;
 
 	if (nft_flow_offload_skip(pkt->skb, nft_pf(pkt)))
 		goto out;
@@ -118,14 +117,9 @@ static void nft_flow_offload_eval(const struct nft_expr *expr,
 		flow_offload_ct_tcp(ct);
 
 	__set_bit(NF_FLOW_HW_BIDIRECTIONAL, &flow->flags);
-	ret = flow_offload_add(flowtable, flow);
-	if (ret < 0)
-		goto err_flow_add;
-
+	flow_offload_add(flowtable, flow);
 	return;
 
-err_flow_add:
-	flow_offload_free(flow);
 err_flow_alloc:
 	dst_release(route.tuple[dir].dst);
 	dst_release(route.tuple[!dir].dst);
net/sched/act_ct.c +2 -10
index e250969c84ac..31df7f9bce86 100644
@@ -427,16 +427,13 @@ static void tcf_ct_flow_table_add(struct tcf_ct_flow_table *ct_ft,
 {
 	struct nf_conn_act_ct_ext *act_ct_ext;
 	struct flow_offload *entry;
-	int err;
 
 	if (test_and_set_bit(IPS_OFFLOAD_BIT, &ct->status))
 		return;
 
 	entry = flow_offload_alloc(ct);
-	if (!entry) {
-		WARN_ON_ONCE(1);
+	if (!entry)
 		goto err_alloc;
-	}
 
 	if (tcp) {
 		ct->proto.tcp.seen[0].flags |= IP_CT_TCP_FLAG_BE_LIBERAL;
@@ -451,14 +448,9 @@ static void tcf_ct_flow_table_add(struct tcf_ct_flow_table *ct_ft,
 		tcf_ct_flow_tc_ifidx(entry, act_ct_ext, FLOW_OFFLOAD_DIR_REPLY);
 	}
 
-	err = flow_offload_add(&ct_ft->nf_ft, entry);
-	if (err)
-		goto err_add;
-
+	flow_offload_add(&ct_ft->nf_ft, entry);
 	return;
 
-err_add:
-	flow_offload_free(entry);
 err_alloc:
 	clear_bit(IPS_OFFLOAD_BIT, &ct->status);
 }
Signature · 1 line
2.47.3