netfilter: conntrack: remove percpu lists
Subsystems (2)
Florian Westphal
maintainerv4 cover netfilter: conntrack: remove percpu lists
This series removes the unconfirmed and dying percpu lists.
Dying list is replaced by pernet list, only used when reliable event delivery mode was requested.
Unconfirmed list is replaced by a generation id for the conntrack extesions, to detect when pointers to external objects (timeout policy, helper, ...) has gone stale.
An alternative to the genid would be to always take references on such external objects, let me know if that is the preferred solution.
Changes in v4:
- drop patch to move to global event_cb in netlink
- this allows to drop the preceding nfnl patch
- drop cttimeout rcu patch, its not required
- amend commit message in patch 3 to mention need to move IPS_CONFIRMED bit setting.
Changes in v3:
- fix build bugs reported by kbuild robot
- add patch #16
Florian Westphal (10):
netfilter: ecache: use dedicated list for event redelivery
netfilter: conntrack: include ecache dying list in dumps
netfilter: conntrack: remove the percpu dying list
netfilter: cttimeout: decouple unlink and free on netns destruction
netfilter: remove nf_ct_unconfirmed_destroy helper
netfilter: extensions: introduce extension genid count
netfilter: cttimeout: decouple unlink and free on netns destruction
netfilter: conntrack: remove __nf_ct_unconfirmed_destroy
netfilter: conntrack: remove unconfirmed list
netfilter: conntrack: avoid unconditional local_bh_disable
include/net/netfilter/nf_conntrack.h | 7 +- include/net/netfilter/nf_conntrack_ecache.h | 4 +- include/net/netfilter/nf_conntrack_extend.h | 31 +-- include/net/netfilter/nf_conntrack_labels.h | 10 +- include/net/netfilter/nf_conntrack_timeout.h | 8 - include/net/netns/conntrack.h | 7 - net/netfilter/nf_conntrack_core.c | 230 ++++++++----------- net/netfilter/nf_conntrack_ecache.c | 127 +++++----- net/netfilter/nf_conntrack_extend.c | 32 ++- net/netfilter/nf_conntrack_helper.c | 5 - net/netfilter/nf_conntrack_netlink.c | 76 +++--- net/netfilter/nfnetlink_cttimeout.c | 47 +++- 12 files changed, 298 insertions(+), 286 deletions(-)
Signature · 1 line
2.35.1
Florian Westphal
maintainerPATCH v4 1/10 netfilter: ecache: use dedicated list for event redelivery
This disentangles event redelivery and the percpu dying list.
Because entries are now stored on a dedicated list, all entries are in NFCT_ECACHE_DESTROY_FAIL state and all entries still have confirmed bit set -- the reference count is at least 1.
The 'struct net' back-pointer can be removed as well.
The pcpu dying list will be removed eventually, it has no functionality.
--- include/net/netfilter/nf_conntrack.h | 3 +- include/net/netfilter/nf_conntrack_ecache.h | 2 - net/netfilter/nf_conntrack_core.c | 33 +++++- net/netfilter/nf_conntrack_ecache.c | 118 +++++++++----------- 4 files changed, 83 insertions(+), 73 deletions(-)
include/net/netfilter/nf_conntrack.h +2 -1
@@ -45,7 +45,8 @@ union nf_conntrack_expect_proto {struct nf_conntrack_net_ecache {struct delayed_work dwork; -struct netns_ct *ct_net; +spinlock_t dying_lock; +struct hlist_nulls_head dying_list; };struct nf_conntrack_net {
include/net/netfilter/nf_conntrack_ecache.h +0 -2
@@ -14,7 +14,6 @@#include <net/netfilter/nf_conntrack_extend.h> enum nf_ct_ecache_state { -NFCT_ECACHE_UNKNOWN ,/* destroy event not sent */ NFCT_ECACHE_DESTROY_FAIL ,/* tried but failed to send destroy event */ NFCT_ECACHE_DESTROY_SENT ,/* sent destroy event after failure */ };
@@ -23,7 +22,6 @@ struct nf_conntrack_ecache {unsigned long cache;/* bitops want long */ u16 ctmask;/* bitmask of ct events to be delivered */ u16 expmask;/* bitmask of expect events to be delivered */ - enum nf_ct_ecache_state state:8;/* ecache state */u32 missed;/* missed events */ u32 portid;/* netlink portid of destroyer */ };
net/netfilter/nf_conntrack_core.c +28 -5
@@ -660,15 +660,12 @@ void nf_ct_destroy(struct nf_conntrack *nfct) }EXPORT_SYMBOL (nf_ct_destroy); -static void nf_ct_delete_from_lists(struct nf_conn *ct) +static void __nf_ct_delete_from_lists(struct nf_conn *ct) { struct net *net = nf_ct_net(ct); unsigned int hash, reply_hash; unsigned int sequence; - nf_ct_helper_destroy(ct); - - local_bh_disable(); do { sequence = read_seqcount_begin(&nf_conntrack_generation); hash = hash_conntrack(net,
@@ -681,12 +678,33 @@ static void nf_ct_delete_from_lists(struct nf_conn *ct)clean_from_lists (ct);nf_conntrack_double_unlock (hash, reply_hash); +} +static void nf_ct_delete_from_lists (struct nf_conn *ct) +{ +nf_ct_helper_destroy (ct); +local_bh_disable (); + +__nf_ct_delete_from_lists (ct);nf_ct_add_to_dying_list (ct);local_bh_enable (); } +static void nf_ct_add_to_ecache_list (struct nf_conn *ct) +{ +#ifdef CONFIG_NF_CONNTRACK_EVENTS +struct nf_conntrack_net *cnet =nf_ct_pernet (nf_ct_net (ct)); + +spin_lock (&cnet->ecache.dying_lock); +hlist_nulls_add_head_rcu (&ct->tuplehash[IP_CT_DIR_ORIGINAL ].hnnode, + &cnet->ecache.dying_list); +spin_unlock (&cnet->ecache.dying_lock); +#else +nf_ct_add_to_dying_list (ct); +#endif +} +bool nf_ct_delete (struct nf_conn *ct,u32 portid,int report) {struct nf_conn_tstamp *tstamp;
@@ -709,7 +727,12 @@ bool nf_ct_delete(struct nf_conn *ct, u32 portid, int report)/* destroy event was not delivered. nf_ct_put will * be done by event cache worker on redelivery. */ -nf_ct_delete_from_lists (ct); +nf_ct_helper_destroy (ct); +local_bh_disable (); +__nf_ct_delete_from_lists (ct); +nf_ct_add_to_ecache_list (ct); +local_bh_enable (); +nf_conntrack_ecache_work (nf_ct_net (ct),NFCT_ECACHE_DESTROY_FAIL );return false; }
net/netfilter/nf_conntrack_ecache.c +53 -65
184 lines not on this page · raw message
Signature · 1 line
2.35.1
Florian Westphal
maintainerPATCH v4 2/10 netfilter: conntrack: include ecache dying list in dumps
In reply to Florian WestphalThe new pernet dying list includes conntrack entries that await delivery of the 'destroy' event via ctnetlink.
The old percpu dying list will be removed soon.
--- include/net/netfilter/nf_conntrack_ecache.h | 2 + net/netfilter/nf_conntrack_ecache.c | 10 +++++ net/netfilter/nf_conntrack_netlink.c | 43 +++++++++++++++++++++ 3 files changed, 55 insertions(+)
include/net/netfilter/nf_conntrack_ecache.h +2 -0
@@ -164,6 +164,8 @@ void nf_conntrack_ecache_work(struct net *net, enum nf_ct_ecache_state state);void nf_conntrack_ecache_pernet_init (struct net *net);void nf_conntrack_ecache_pernet_fini (struct net *net); +struct nf_conntrack_net_ecache *nf_conn_pernet_ecache (const struct net *net); +static inline bool nf_conntrack_ecache_dwork_pending (const struct net *net) {return net->ct.ecache_dwork_pending;
net/netfilter/nf_conntrack_ecache.c +10 -0
@@ -38,6 +38,16 @@ enum retry_state { STATE_DONE, }; +struct nf_conntrack_net_ecache *nf_conn_pernet_ecache (const struct net *net) +{ +struct nf_conntrack_net *cnet =nf_ct_pernet (net); + +return &cnet->ecache; +} +#if IS_MODULE (CONFIG_NF_CT_NETLINK ) +EXPORT_SYMBOL_GPL (nf_conn_pernet_ecache); +#endif +static enum retry_state ecache_work_evict_list (struct nf_conntrack_net *cnet) {unsigned long stop = jiffies +ECACHE_MAX_JIFFIES ;
net/netfilter/nf_conntrack_netlink.c +43 -0
55 lines not on this page · raw message
Signature · 1 line
2.35.1
Florian Westphal
maintainerPATCH v4 3/10 netfilter: conntrack: remove the percpu dying list
In reply to Florian WestphalIts no longer needed. Entries that need event redelivery are placed on the new pernet dying list.
The advantage is that there is no need to take additional spinlock on conntrack removal unless event redelivery failed or the conntrack entry was never added to the table in the first place (confirmed bit not set).
The IPS_CONFIRMED bit now needs to be set as soon as the entry has been unlinked from the unconfirmed list, else the destroy function may attempt to unlink it a second time.
--- include/net/netns/conntrack.h | 1 - net/netfilter/nf_conntrack_core.c | 35 +++++----------------------- net/netfilter/nf_conntrack_ecache.c | 1 - net/netfilter/nf_conntrack_netlink.c | 23 ++++++------------ 4 files changed, 13 insertions(+), 47 deletions(-)
include/net/netns/conntrack.h +0 -1
@@ -96,7 +96,6 @@ struct nf_ip_net {struct ct_pcpu {spinlock_t lock;struct hlist_nulls_head unconfirmed; -struct hlist_nulls_head dying; };struct netns_ct {
net/netfilter/nf_conntrack_core.c +6 -29
@@ -525,21 +525,6 @@ clean_from_lists(struct nf_conn *ct)nf_ct_remove_expectations (ct); } -/* must be called with local_bh_disable */ -static void nf_ct_add_to_dying_list (struct nf_conn *ct) -{ -struct ct_pcpu *pcpu; - -/* add this conntrack to the (per cpu) dying list */ - ct->cpu =smp_processor_id (); - pcpu =per_cpu_ptr (nf_ct_net (ct)->ct.pcpu_lists, ct->cpu); - -spin_lock (&pcpu->lock); -hlist_nulls_add_head (&ct->tuplehash[IP_CT_DIR_ORIGINAL ].hnnode, - &pcpu->dying); -spin_unlock (&pcpu->lock); -} -/* must be called with local_bh_disable */ static void nf_ct_add_to_unconfirmed_list (struct nf_conn *ct) {
@@ -556,11 +541,11 @@ static void nf_ct_add_to_unconfirmed_list(struct nf_conn *ct) }/* must be called with local_bh_disable */ -static void nf_ct_del_from_dying_or_unconfirmed_list (struct nf_conn *ct) +static void nf_ct_del_from_unconfirmed_list (struct nf_conn *ct) {struct ct_pcpu *pcpu; -/* We overload first tuple to link into unconfirmed or dying list.*/ +/* We overload first tuple to link into unconfirmed list.*/ pcpu =per_cpu_ptr (nf_ct_net (ct)->ct.pcpu_lists, ct->cpu);spin_lock (&pcpu->lock);
@@ -648,7 +633,8 @@ void nf_ct_destroy(struct nf_conntrack *nfct)*/ nf_ct_remove_expectations (ct); -nf_ct_del_from_dying_or_unconfirmed_list (ct); +if (unlikely (!nf_ct_is_confirmed (ct))) +nf_ct_del_from_unconfirmed_list (ct);local_bh_enable ();
@@ -686,7 +672,6 @@ static void nf_ct_delete_from_lists(struct nf_conn *ct)local_bh_disable ();__nf_ct_delete_from_lists (ct); -nf_ct_add_to_dying_list (ct);local_bh_enable (); }
@@ -700,8 +685,6 @@ static void nf_ct_add_to_ecache_list(struct nf_conn *ct)hlist_nulls_add_head_rcu (&ct->tuplehash[IP_CT_DIR_ORIGINAL ].hnnode, &cnet->ecache.dying_list);spin_unlock (&cnet->ecache.dying_lock); -#else -nf_ct_add_to_dying_list (ct);#endif }
@@ -995,7 +978,6 @@ static void __nf_conntrack_insert_prepare(struct nf_conn *ct)struct nf_conn_tstamp *tstamp;refcount_inc (&ct->ct_general.use); - ct->status |=IPS_CONFIRMED ;/* set conntrack timestamp, if enabled. */ tstamp =nf_conn_tstamp_find (ct);
@@ -1024,7 +1006,6 @@ static int __nf_ct_resolve_clash(struct sk_buff *skb,nf_conntrack_get (&ct->ct_general);nf_ct_acct_merge (ct, ctinfo, loser_ct); -nf_ct_add_to_dying_list (loser_ct);nf_ct_put (loser_ct);nf_ct_set (skb, ct, ctinfo);
@@ -1157,7 +1138,6 @@ nf_ct_resolve_clash(struct sk_buff *skb, struct nf_conntrack_tuple_hash *h,return ret; drop: -nf_ct_add_to_dying_list (loser_ct);NF_CT_STAT_INC (net, drop);NF_CT_STAT_INC (net, insert_failed);return NF_DROP ;
@@ -1224,10 +1204,10 @@ __nf_conntrack_confirm(struct sk_buff *skb)* user context, else we insert an already 'dead' hash, blocking * further use of that particular connection -JM. */ -nf_ct_del_from_dying_or_unconfirmed_list (ct); +nf_ct_del_from_unconfirmed_list (ct); + ct->status |=IPS_CONFIRMED ;if (unlikely (nf_ct_is_dying (ct))) { -nf_ct_add_to_dying_list (ct);NF_CT_STAT_INC (net, insert_failed); goto dying; }
@@ -1251,7 +1231,6 @@ __nf_conntrack_confirm(struct sk_buff *skb) goto out;if (chainlen++ > max_chainlen) { chaintoolong: -nf_ct_add_to_dying_list (ct);NF_CT_STAT_INC (net, chaintoolong);NF_CT_STAT_INC (net, insert_failed); ret =NF_DROP ;
@@ -2800,7 +2779,6 @@ void nf_conntrack_init_end(void)* We need to use special "null" values, not used in hash table */ #define UNCONFIRMED_NULLS_VAL ((1<<30)+0) -#define DYING_NULLS_VAL ((1<<30)+1)int nf_conntrack_init_net (struct net *net) {
@@ -2821,7 +2799,6 @@ int nf_conntrack_init_net(struct net *net)spin_lock_init (&pcpu->lock);INIT_HLIST_NULLS_HEAD (&pcpu->unconfirmed,UNCONFIRMED_NULLS_VAL ); -INIT_HLIST_NULLS_HEAD (&pcpu->dying,DYING_NULLS_VAL ); } net->ct.stat = alloc_percpu(struct ip_conntrack_stat);
net/netfilter/nf_conntrack_ecache.c +0 -1
@@ -95,7 +95,6 @@ static enum retry_state ecache_work_evict_list(struct nf_conntrack_net *cnet)hlist_nulls_for_each_entry_safe (h, n, &evicted_list, hnnode) {struct nf_conn *ct =nf_ct_tuplehash_to_ctrack (h); -hlist_nulls_add_fake (&ct->tuplehash[IP_CT_DIR_ORIGINAL ].hnnode);hlist_nulls_del_rcu (&ct->tuplehash[IP_CT_DIR_REPLY ].hnnode);nf_ct_put (ct);
net/netfilter/nf_conntrack_netlink.c +7 -16
@@ -62,7 +62,6 @@ struct ctnetlink_list_dump_ctx {struct nf_conn *last;unsigned int cpu;bool done; -bool retrans_done; }; static int ctnetlink_dump_tuples_proto(struct sk_buff *skb,
@@ -1751,13 +1750,12 @@ static int ctnetlink_dump_one_entry(struct sk_buff *skb, }static int -ctnetlink_dump_list (struct sk_buff *skb,struct netlink_callback *cb,bool dying) +ctnetlink_dump_unconfirmed (struct sk_buff *skb,struct netlink_callback *cb) {struct ctnetlink_list_dump_ctx *ctx = (void *)cb->ctx;struct nf_conn *ct, *last;struct nf_conntrack_tuple_hash *h;struct hlist_nulls_node *n; -struct hlist_nulls_head *list;struct net *net =sock_net (skb->sk);int res, cpu;
@@ -1774,12 +1772,11 @@ ctnetlink_dump_list(struct sk_buff *skb, struct netlink_callback *cb, bool dying pcpu =per_cpu_ptr (net->ct.pcpu_lists, cpu);spin_lock_bh (&pcpu->lock); - list = dying ? &pcpu->dying : &pcpu->unconfirmed; restart: -hlist_nulls_for_each_entry (h, n, list, hnnode) { +hlist_nulls_for_each_entry (h, n, &pcpu->unconfirmed, hnnode) { ct =nf_ct_tuplehash_to_ctrack (h); - res =ctnetlink_dump_one_entry (skb, cb, ct, dying); + res =ctnetlink_dump_one_entry (skb, cb, ct, false);if (res <0 ) { ctx->cpu = cpu;spin_unlock_bh (&pcpu->lock);
@@ -1812,8 +1809,8 @@ ctnetlink_dump_dying(struct sk_buff *skb, struct netlink_callback *cb)struct hlist_nulls_node *n;#endif -if (ctx->retrans_done) -return ctnetlink_dump_list (skb, cb, true); +if (ctx->done) +return 0 ; ctx->last =NULL ;
@@ -1842,10 +1839,10 @@ ctnetlink_dump_dying(struct sk_buff *skb, struct netlink_callback *cb)spin_unlock_bh (&ecache_net->dying_lock);#endif + ctx->done = true;nf_ct_put (last); - ctx->retrans_done = true; -return ctnetlink_dump_list (skb, cb, true); +return skb->len; } static int ctnetlink_get_ct_dying(struct sk_buff *skb,
@@ -1863,12 +1860,6 @@ static int ctnetlink_get_ct_dying(struct sk_buff *skb,return -EOPNOTSUPP ; } -static int -ctnetlink_dump_unconfirmed (struct sk_buff *skb,struct netlink_callback *cb) -{ -return ctnetlink_dump_list (skb, cb, false); -} -static int ctnetlink_get_ct_unconfirmed (struct sk_buff *skb,const struct nfnl_info *info,const struct nlattr *const cda[])
Signature · 1 line
2.35.1
Florian Westphal
maintainerPATCH v4 4/10 netfilter: cttimeout: decouple unlink and free on netns destruction
In reply to Florian WestphalMake it so netns pre_exit unlinks the objects from the pernet list, so they cannot be found anymore.
netns core issues a synchronize_rcu() before calling the exit hooks so any the time the exit hooks run unconfirmed nf_conn entries have been free'd or they have been committed to the hashtable.
The exit hook still tags unconfirmed entries as dying, this can now be removed in a followup change.
--- include/net/netfilter/nf_conntrack_timeout.h | 8 ------ net/netfilter/nfnetlink_cttimeout.c | 30 ++++++++++++++++++-- 2 files changed, 28 insertions(+), 10 deletions(-)
include/net/netfilter/nf_conntrack_timeout.h +0 -8
@@ -17,14 +17,6 @@ struct nf_ct_timeout {char data[]; }; -struct ctnl_timeout { -struct list_head head; -struct rcu_head rcu_head; -refcount_t refcnt; -char name[CTNL_TIMEOUT_NAME_MAX ]; -struct nf_ct_timeout timeout; -}; -struct nf_conn_timeout {struct nf_ct_timeout __rcu *timeout; };
net/netfilter/nfnetlink_cttimeout.c +28 -2
@@ -33,8 +33,19 @@static unsigned int nfct_timeout_id__read_mostly ; +struct ctnl_timeout { +struct list_head head; +struct rcu_head rcu_head; +refcount_t refcnt; +char name[CTNL_TIMEOUT_NAME_MAX ]; +struct nf_ct_timeout timeout; + +struct list_head free_head; +}; +struct nfct_timeout_pernet {struct list_head nfct_timeout_list; +struct list_head nfct_timeout_freelist; };MODULE_LICENSE ("GPL" );
@@ -574,10 +585,24 @@ static int __net_init cttimeout_net_init(struct net *net)struct nfct_timeout_pernet *pernet =nfct_timeout_pernet (net);INIT_LIST_HEAD (&pernet->nfct_timeout_list); +INIT_LIST_HEAD (&pernet->nfct_timeout_freelist);return 0 ; } +static void __net_exit cttimeout_net_pre_exit(struct net *net) +{ +struct nfct_timeout_pernet *pernet =nfct_timeout_pernet (net); +struct ctnl_timeout *cur, *tmp; + +list_for_each_entry_safe (cur, tmp, &pernet->nfct_timeout_list, head) { +list_del_rcu (&cur->head); +list_add (&cur->free_head, &pernet->nfct_timeout_freelist); + } + +/* core calls synchronize_rcu() after this */ +} + static void __net_exit cttimeout_net_exit(struct net *net) {struct nfct_timeout_pernet *pernet =nfct_timeout_pernet (net);
@@ -586,8 +611,8 @@ static void __net_exit cttimeout_net_exit(struct net *net)nf_ct_unconfirmed_destroy (net);nf_ct_untimeout (net,NULL ); -list_for_each_entry_safe (cur, tmp, &pernet->nfct_timeout_list, head) { -list_del_rcu (&cur->head); +list_for_each_entry_safe (cur, tmp, &pernet->nfct_timeout_freelist, head) { +list_del (&cur->free_head);if (refcount_dec_and_test (&cur->refcnt))kfree_rcu (cur, rcu_head);
@@ -596,6 +621,7 @@ static void __net_exit cttimeout_net_exit(struct net *net)static struct pernet_operations cttimeout_ops = { .init = cttimeout_net_init, + .pre_exit = cttimeout_net_pre_exit, .exit = cttimeout_net_exit, .id = &nfct_timeout_id, .size =sizeof (struct nfct_timeout_pernet ),
Signature · 1 line
2.35.1
Florian Westphal
maintainerPATCH v4 5/10 netfilter: remove nf_ct_unconfirmed_destroy helper
In reply to Florian WestphalThis helper tags connetions not yet in the conntrack table as dying. These nf_conn entries will be dropped instead when the core attempts to insert them from the input or postrouting 'confirm' hook.
After the previous change, the entries get unlinked from the list earlier, so that by the time the actual exit hook runs, new connections no longer have a timeout policy assigned.
Its enough to walk the hashtable instead.
--- include/net/netfilter/nf_conntrack.h | 3 --- net/netfilter/nf_conntrack_core.c | 14 -------------- net/netfilter/nfnetlink_cttimeout.c | 4 +++- 3 files changed, 3 insertions(+), 18 deletions(-)
include/net/netfilter/nf_conntrack.h +0 -3
@@ -237,9 +237,6 @@ static inline bool nf_ct_kill(struct nf_conn *ct)return nf_ct_delete (ct,0 ,0 ); } -/* Set all unconfirmed conntrack as dying */ -void nf_ct_unconfirmed_destroy (struct net *); -/* Iterate over all conntracks: if iter returns true, it's deleted. */ void nf_ct_iterate_cleanup_net(struct net *net, int (*iter)(struct nf_conn *i, void *data),
net/netfilter/nf_conntrack_core.c +0 -14
@@ -2431,20 +2431,6 @@ __nf_ct_unconfirmed_destroy(struct net *net) } } -void nf_ct_unconfirmed_destroy (struct net *net) -{ -struct nf_conntrack_net *cnet =nf_ct_pernet (net); - -might_sleep (); - -if (atomic_read (&cnet->count) >0 ) { -__nf_ct_unconfirmed_destroy (net); -nf_queue_nf_hook_drop (net); -synchronize_net (); - } -} -EXPORT_SYMBOL_GPL (nf_ct_unconfirmed_destroy); -void nf_ct_iterate_cleanup_net (struct net *net,int (*iter)(struct nf_conn *i,void *data),void *data,u32 portid,int report)
net/netfilter/nfnetlink_cttimeout.c +3 -1
@@ -608,7 +608,9 @@ static void __net_exit cttimeout_net_exit(struct net *net)struct nfct_timeout_pernet *pernet =nfct_timeout_pernet (net);struct ctnl_timeout *cur, *tmp; -nf_ct_unconfirmed_destroy (net); +if (list_empty (&pernet->nfct_timeout_freelist)) +return ; +nf_ct_untimeout (net,NULL );list_for_each_entry_safe (cur, tmp, &pernet->nfct_timeout_freelist, head) {
Signature · 1 line
2.35.1
Florian Westphal
maintainerPATCH v4 6/10 netfilter: extensions: introduce extension genid count
In reply to Florian WestphalMultiple netfilter extensions store pointers to external data in their extension area struct.
Examples:
1. Timeout policies
2. Connection tracking helpers.
No references are taken for these.
When a helper or timeout policy is removed, the conntrack table gets traversed and affected extensions are cleared.
Conntrack entries not yet in the hashtable are referenced via a special list, the unconfirmed list.
On removal of a policy or connection tracking helper, the unconfirmed list gets traversed an all entries are marked as dying, this prevents them from getting committed to the table at insertion time: core checks for dying bit, if set, the conntrack entry gets destroyed at confirm time.
The disadvantage is that each new conntrack has to be added to the percpu unconfirmed list, and each insertion needs to remove it from this list. The list is only ever needed when a policy or helper is removed -- a rare occurrence.
Add a generation ID count: Instead of adding to the list and then traversing that list on policy/helper removal, increment a counter that is stored in the extension area.
For unconfirmed conntracks, the extension has the genid valid at ct allocation time.
Removal of a helper/policy etc. increments the counter. At confirmation time, validate that ext->genid == global_id.
If the stored number is not the same, do not allow the conntrack insertion, just like as if a confirmed-list traversal would have flagged the entry as dying.
After insertion, the genid is no longer relevant (conntrack entries are now reachable via the conntrack table iterators and is set to 0.
This allows removal of the percpu unconfirmed list.
--- include/net/netfilter/nf_conntrack_extend.h | 31 ++++++------ include/net/netfilter/nf_conntrack_labels.h | 10 +++- net/netfilter/nf_conntrack_core.c | 55 +++++++++++++++++++++ net/netfilter/nf_conntrack_extend.c | 32 +++++++++++- 4 files changed, 111 insertions(+), 17 deletions(-)
include/net/netfilter/nf_conntrack_extend.h +16 -15
@@ -34,21 +34,11 @@ enum nf_ct_ext_id { NF_CT_EXT_NUM, }; -#define NF_CT_EXT_HELPER_TYPE struct nf_conn_help -#define NF_CT_EXT_NAT_TYPE struct nf_conn_nat -#define NF_CT_EXT_SEQADJ_TYPE struct nf_conn_seqadj -#define NF_CT_EXT_ACCT_TYPE struct nf_conn_acct -#define NF_CT_EXT_ECACHE_TYPE struct nf_conntrack_ecache -#define NF_CT_EXT_TSTAMP_TYPE struct nf_conn_tstamp -#define NF_CT_EXT_TIMEOUT_TYPE struct nf_conn_timeout -#define NF_CT_EXT_LABELS_TYPE struct nf_conn_labels -#define NF_CT_EXT_SYNPROXY_TYPE struct nf_conn_synproxy -#define NF_CT_EXT_ACT_CT_TYPE struct nf_conn_act_ct_ext -/* Extensions: optional stuff which isn't permanently in struct. */ struct nf_ct_ext {u8 offset[NF_CT_EXT_NUM ];u8 len; +unsigned int gen_id;char data[]__aligned(8) ; };
@@ -62,17 +52,28 @@ static inline bool nf_ct_ext_exist(const struct nf_conn *ct, u8 id) return (ct->ext && __nf_ct_ext_exist(ct->ext, id)); } -static inline void *__nf_ct_ext_find (const struct nf_conn *ct,u8 id) +void *__nf_ct_ext_find (const struct nf_ct_ext *ext,u8 id); + +static inline void *nf_ct_ext_find (const struct nf_conn *ct,u8 id) { -if (!nf_ct_ext_exist (ct, id)) +struct nf_ct_ext *ext = ct->ext; + +if (!ext || !__nf_ct_ext_exist (ext, id))return NULL ; +if (unlikely (ext->gen_id)) +return __nf_ct_ext_find (ext, id); +return (void *)ct->ext + ct->ext->offset[id]; } -#define nf_ct_ext_find (ext, id) \ - ((id##_TYPE *)__nf_ct_ext_find((ext), (id)))/* Add this type, returns pointer to data or NULL. */ void *nf_ct_ext_add (struct nf_conn *ct,enum nf_ct_ext_id id,gfp_t gfp); +/* ext genid. if ext->id != ext_genid, extensions cannot be used +* anymore unless conntrack has CONFIRMED bit set. +*/ +extern atomic_t nf_conntrack_ext_genid; +void nf_ct_ext_bump_genid (void ); +#endif /* _NF_CONNTRACK_EXTEND_H */
include/net/netfilter/nf_conntrack_labels.h +9 -1
@@ -17,10 +17,18 @@ struct nf_conn_labels {unsigned long bits[NF_CT_LABELS_MAX_SIZE /sizeof (long )]; }; +/* Can't use nf_ct_ext_find(), flow dissector cannot use symbols +* exported by nf_conntrack module. +*/ static inline struct nf_conn_labels *nf_ct_labels_find (const struct nf_conn *ct) {#ifdef CONFIG_NF_CONNTRACK_LABELS -return nf_ct_ext_find (ct,NF_CT_EXT_LABELS ); +struct nf_ct_ext *ext = ct->ext; + +if (!ext || !__nf_ct_ext_exist (ext,NF_CT_EXT_LABELS )) +return NULL ; + +return (void *)ct->ext + ct->ext->offset[NF_CT_EXT_LABELS ];#else return NULL ;#endif
net/netfilter/nf_conntrack_core.c +55 -0
91 lines not on this page · raw message
net/netfilter/nf_conntrack_extend.c +31 -1
@@ -27,6 +27,8 @@#define NF_CT_EXT_PREALLOC 128u/* conntrack events are on by default */ +atomic_t nf_conntrack_ext_genid__read_mostly =ATOMIC_INIT (1 ); +static const u8 nf_ct_ext_type_len[NF_CT_EXT_NUM ] = { [NF_CT_EXT_HELPER ] =sizeof (struct nf_conn_help ), #if IS_ENABLED(CONFIG_NF_NAT)
@@ -116,8 +118,10 @@ void *nf_ct_ext_add(struct nf_conn *ct, enum nf_ct_ext_id id, gfp_t gfp)if (!new)return NULL ; -if (!ct->ext) +if (!ct->ext) {memset (new->offset,0 ,sizeof (new->offset)); + new->gen_id =atomic_read (&nf_conntrack_ext_genid); + } new->offset[id] = newoff; new->len = newlen;
@@ -127,3 +131,29 @@ void *nf_ct_ext_add(struct nf_conn *ct, enum nf_ct_ext_id id, gfp_t gfp)return (void *)new + newoff; }EXPORT_SYMBOL (nf_ct_ext_add); + +/* Use nf_ct_ext_find wrapper. This is only useful for unconfirmed entries. */ +void *__nf_ct_ext_find (const struct nf_ct_ext *ext,u8 id) +{ +unsigned int gen_id =atomic_read (&nf_conntrack_ext_genid); +unsigned int this_id =READ_ONCE (ext->gen_id); + +if (!__nf_ct_ext_exist (ext, id)) +return NULL ; + +if (this_id ==0 || ext->gen_id == gen_id) +return (void *)ext + ext->offset[id]; + +return NULL ; +} +EXPORT_SYMBOL (__nf_ct_ext_find); + +void nf_ct_ext_bump_genid (void ) +{ +unsigned int value =atomic_inc_return (&nf_conntrack_ext_genid); + +if (value ==UINT_MAX ) +atomic_set (&nf_conntrack_ext_genid,1 ); + +msleep (HZ ); +}
Signature · 1 line
2.35.1
Florian Westphal
maintainerPATCH v4 7/10 netfilter: cttimeout: decouple unlink and free on netns destruction
In reply to Florian WestphalIncrement the extid on module removal; this makes sure that even in extreme cases any old uncofirmed entry that happened to be kept e.g. on nfnetlink_queue list will not trip over a stale timeout reference.
--- net/netfilter/nfnetlink_cttimeout.c | 12 ++++++++++++ 1 file changed, 12 insertions(+)
net/netfilter/nfnetlink_cttimeout.c +12 -0
@@ -656,12 +656,24 @@ static int __init cttimeout_init(void)return ret; } +static int untimeout (struct nf_conn *ct,void *timeout) +{ +struct nf_conn_timeout *timeout_ext =nf_ct_timeout_find (ct); + +if (timeout_ext) +RCU_INIT_POINTER (timeout_ext->timeout,NULL ); + +return 0 ; +} + static void __exit cttimeout_exit(void) {nfnetlink_subsys_unregister (&cttimeout_subsys);unregister_pernet_subsys (&cttimeout_ops);RCU_INIT_POINTER (nf_ct_timeout_hook,NULL ); + +nf_ct_iterate_destroy (untimeout,NULL );synchronize_rcu (); }
Signature · 1 line
2.35.1
Florian Westphal
maintainerPATCH v4 8/10 netfilter: conntrack: remove __nf_ct_unconfirmed_destroy
In reply to Florian WestphalIts not needed anymore:
A. If entry is totally new, then the rcu-protected resource must already have been removed from global visibility before call to nf_ct_iterate_destroy.
B. If entry was allocated before, but is not yet in the hash table
(uncofirmed case), genid gets incremented and synchronize_rcu() call makes sure access has completed.
C. Next attempt to peek at extension area will fail for unconfirmed
conntracks, because ext->genid != genid.
D. Conntracks in the hash are iterated as before.
--- net/netfilter/nf_conntrack_core.c | 46 ++++++++--------------------- net/netfilter/nf_conntrack_helper.c | 5 ---- net/netfilter/nfnetlink_cttimeout.c | 1 - 3 files changed, 13 insertions(+), 39 deletions(-)
net/netfilter/nf_conntrack_core.c +13 -33
@@ -2457,34 +2457,6 @@ static int iter_net_only(struct nf_conn *i, void *data)return d->iter (i, d->data); } -static void -__nf_ct_unconfirmed_destroy (struct net *net) -{ -int cpu; - -for_each_possible_cpu (cpu) { -struct nf_conntrack_tuple_hash *h; -struct hlist_nulls_node *n; -struct ct_pcpu *pcpu; - - pcpu =per_cpu_ptr (net->ct.pcpu_lists, cpu); - -spin_lock_bh (&pcpu->lock); -hlist_nulls_for_each_entry (h, n, &pcpu->unconfirmed, hnnode) { -struct nf_conn *ct; - - ct =nf_ct_tuplehash_to_ctrack (h); - -/* we cannot call iter() on unconfirmed list, the -* owning cpu can reallocate ct->ext at any time. -*/ -set_bit (IPS_DYING_BIT , &ct->status); - } -spin_unlock_bh (&pcpu->lock); -cond_resched (); - } -} -void nf_ct_iterate_cleanup_net (struct net *net,int (*iter)(struct nf_conn *i,void *data),void *data,u32 portid,int report)
@@ -2527,26 +2499,34 @@ nf_ct_iterate_destroy(int (*iter)(struct nf_conn *i, void *data), void *data)if (atomic_read (&cnet->count) ==0 )continue ; -__nf_ct_unconfirmed_destroy (net);nf_queue_nf_hook_drop (net); }up_read (&net_rwsem);/* Need to wait for netns cleanup worker to finish, if its * running -- it might have deleted a net namespace from -* the global list, so our __nf_ct_unconfirmed_destroy() might -* not have affected all namespaces. +* the global list, so hook drop above might not have +* affected all namespaces. */ net_ns_barrier (); -/* a conntrack could have been unlinked from unconfirmed list -* before we grabbed pcpu lock in __nf_ct_unconfirmed_destroy(). +/* a skb w. unconfirmed conntrack could have been reinjected just +* before we called nf_queue_nf_hook_drop(). +* * This makes sure its inserted into conntrack table. */ synchronize_net ();nf_ct_ext_bump_genid ();nf_ct_iterate_cleanup (iter, data,0 ,0 ); + +/* Another cpu might be in a rcu read section with +* rcu protected pointer cleared in iter callback +* or hidden via nf_ct_ext_bump_genid() above. +* +* Wait until those are done. +*/ +synchronize_rcu (); }EXPORT_SYMBOL_GPL (nf_ct_iterate_destroy);
net/netfilter/nf_conntrack_helper.c +0 -5
@@ -468,11 +468,6 @@ void nf_conntrack_helper_unregister(struct nf_conntrack_helper *me)nf_ct_expect_iterate_destroy (expect_iter_me,NULL );nf_ct_iterate_destroy (unhelp, me); - -/* Maybe someone has gotten the helper already when unhelp above. -* So need to wait it. -*/ -synchronize_rcu (); }EXPORT_SYMBOL_GPL (nf_conntrack_helper_unregister);
net/netfilter/nfnetlink_cttimeout.c +0 -1
@@ -674,7 +674,6 @@ static void __exit cttimeout_exit(void)RCU_INIT_POINTER (nf_ct_timeout_hook,NULL );nf_ct_iterate_destroy (untimeout,NULL ); -synchronize_rcu (); }module_init (cttimeout_init);
Signature · 1 line
2.35.1
Florian Westphal
maintainerPATCH v4 9/10 netfilter: conntrack: remove unconfirmed list
In reply to Florian WestphalIt has no function anymore and can be removed.
--- include/net/netfilter/nf_conntrack.h | 1 - include/net/netns/conntrack.h | 6 ---- net/netfilter/nf_conntrack_core.c | 54 ++-------------------------- net/netfilter/nf_conntrack_netlink.c | 44 +---------------------- 4 files changed, 3 insertions(+), 102 deletions(-)
include/net/netfilter/nf_conntrack.h +0 -1
@@ -101,7 +101,6 @@ struct nf_conn {/* Have we seen traffic both ways yet? (bitset) */ unsigned long status; -u16 cpu;possible_net_t ct_net;#if IS_ENABLED (CONFIG_NF_NAT )
include/net/netns/conntrack.h +0 -6
@@ -93,11 +93,6 @@ struct nf_ip_net {#endif }; -struct ct_pcpu { -spinlock_t lock; -struct hlist_nulls_head unconfirmed; -}; -struct netns_ct {#ifdef CONFIG_NF_CONNTRACK_EVENTS bool ecache_dwork_pending;
@@ -109,7 +104,6 @@ struct netns_ct {u8 sysctl_tstamp;u8 sysctl_checksum; -struct ct_pcpu __percpu *pcpu_lists;struct ip_conntrack_stat __percpu *stat;struct nf_ct_event_notifier __rcu *nf_conntrack_event_cb;struct nf_ip_net nf_ct_proto;
net/netfilter/nf_conntrack_core.c +2 -52
103 lines not on this page · raw message
net/netfilter/nf_conntrack_netlink.c +1 -43
50 lines not on this page · raw message
Signature · 1 line
2.35.1
Florian Westphal
maintainerPATCH v4 10/10 netfilter: conntrack: avoid unconditional local_bh_disable
In reply to Florian WestphalNow that the conntrack entry isn't placed on the pcpu list anymore the bh only needs to be disabled in the 'expectation present' case.
--- net/netfilter/nf_conntrack_core.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-)
net/netfilter/nf_conntrack_core.c +2 -5
@@ -1739,10 +1739,9 @@ init_conntrack(struct net *net, struct nf_conn *tmpl, ecache ? ecache->expmask :0 , GFP_ATOMIC); -local_bh_disable (); cnet =nf_ct_pernet (net);if (cnet->expect_count) { -spin_lock (&nf_conntrack_expect_lock); +spin_lock_bh (&nf_conntrack_expect_lock); exp =nf_ct_find_expectation (net, zone, tuple);if (exp) { pr_debug("expectation arrives ct=%p exp=%p\n",
@@ -1765,7 +1764,7 @@ init_conntrack(struct net *net, struct nf_conn *tmpl,#endif NF_CT_STAT_INC (net, expect_new); } -spin_unlock (&nf_conntrack_expect_lock); +spin_unlock_bh (&nf_conntrack_expect_lock); }if (!exp)__nf_ct_try_assign_helper (ct, tmpl,GFP_ATOMIC );
@@ -1773,8 +1772,6 @@ init_conntrack(struct net *net, struct nf_conn *tmpl,/* Now it is going to be associated with an sk_buff, set refcount to 1. */ refcount_set (&ct->ct_general.use,1 ); -local_bh_enable (); -if (exp) {if (exp->expectfn) exp->expectfn (ct, exp);
Signature · 1 line
2.35.1
Pablo Neira Ayuso
maintainerQuoted 11 lines · Florian Westphal
This series removes the unconfirmed and dying percpu lists.
Dying list is replaced by pernet list, only used when reliable event delivery mode was requested.
Unconfirmed list is replaced by a generation id for the conntrack extesions, to detect when pointers to external objects (timeout policy, helper, ...) has gone stale.
An alternative to the genid would be to always take references on such external objects, let me know if that is the preferred solution.
Pushed it out to nf-next, thanks