netfilter: nat: simplify & convert bysrc hash to rhashtable
Subsystems (2)
Florian Westphal
maintainercover netfilter: nat: simplify & convert bysrc hash to rhashtable
As discussed during NFWS in Amsterdam this gets rid of the 'ct' backpointer in the nat extension structure.
We place nf_conn objects in the bysource table, then convert it to rhashtable to get parallel insert and automated sizing of the table.
After these patches, we have one additional pointer in nf_conn (same size as before due to alignment), nat extension area is reduced to 8 byte instead of 32.
Florian Westphal
maintainerPATCH 1/2 netfilter: move nat hlist_head to nf_conn
The nat extension structure is 32bytes in size on x86_64:
struct nf_conn_nat {
struct hlist_node bysource; /* 0 16 */
struct nf_conn * ct; /* 16 8 */
union nf_conntrack_nat_help help; /* 24 4 */
int masq_index; /* 28 4 */
/* size: 32, cachelines: 1, members: 4 */
/* last cacheline: 32 bytes */
};The hlist is needed to quickly check for possible tuple collisions when installing a new nat binding. Storing this in the extension area has two drawbacks:
1. We need ct backpointer to get the conntrack struct from the extension.
2. When reallocation of extension area occurs we need to fixup the bysource hash head via hlist_replace_rcu.
We can avoid both by placing the hlist_head in nf_conn and place nf_conn in the bysource hash rather than the extenstion.
We can also remove the ->move support; no other extension needs it.
Moving the entire nat extension into nf_conn would be possible as well but then we have to add yet another callback for deletion from the bysource hash table rather than just using nat extension ->destroy hook for this.
nf_conn size doesn't increase due to aligment, followup patch replaces hlist_node with single pointer.
--- include/net/netfilter/nf_conntrack.h | 3 +++ include/net/netfilter/nf_conntrack_extend.h | 3 --- include/net/netfilter/nf_nat.h | 2 -- net/netfilter/nf_conntrack_extend.c | 15 ++----------- net/netfilter/nf_nat_core.c | 33 ++++++----------------------- 5 files changed, 12 insertions(+), 44 deletions(-)
include/net/netfilter/nf_conntrack.h +3 -0
@@ -117,6 +117,9 @@ struct nf_conn {/* Extensions */ struct nf_ct_ext *ext; +#if IS_ENABLED (CONFIG_NF_NAT ) +struct hlist_node nat_bysource; +#endif /* Storage reserved for other modules, must be the last member */ union nf_conntrack_proto proto; };
include/net/netfilter/nf_conntrack_extend.h +0 -3
@@ -99,9 +99,6 @@ void *__nf_ct_ext_add_length(struct nf_conn *ct, enum nf_ct_ext_id id,struct nf_ct_ext_type {/* Destroys relationships (can be NULL). */ void (*destroy)(struct nf_conn *ct); -/* Called when realloacted (can be NULL). -Contents has already been moved. */ -void (*move)(void *new,void *old);enum nf_ct_ext_id id;
include/net/netfilter/nf_nat.h +0 -2
@@ -29,8 +29,6 @@ struct nf_conn;/* The structure embedded in the conntrack structure. */ struct nf_conn_nat { -struct hlist_node bysource; -struct nf_conn *ct;union nf_conntrack_nat_help help;#if IS_ENABLED (CONFIG_NF_NAT_MASQUERADE_IPV4 ) || \IS_ENABLED (CONFIG_NF_NAT_MASQUERADE_IPV6 )
net/netfilter/nf_conntrack_extend.c +2 -13
@@ -73,7 +73,7 @@ void *__nf_ct_ext_add_length(struct nf_conn *ct, enum nf_ct_ext_id id, size_t var_alloc_len, gfp_t gfp) {struct nf_ct_ext *old, *new; -int i, newlen, newoff; +int newlen, newoff;struct nf_ct_ext_type *t;/* Conntrack must not be confirmed to avoid races on reallocation. */
@@ -99,19 +99,8 @@ void *__nf_ct_ext_add_length(struct nf_conn *ct, enum nf_ct_ext_id id,return NULL ;if (new != old) { -for (i =0 ; i <NF_CT_EXT_NUM ; i++) { -if (!__nf_ct_ext_exist (old, i)) -continue ; - -rcu_read_lock (); - t =rcu_dereference (nf_ct_ext_types[i]); -if (t && t->move) - t->move ((void *)new + new->offset[i], - (void *)old + old->offset[i]); -rcu_read_unlock (); - }kfree_rcu (old, rcu); - ct->ext = new; +rcu_assign_pointer (ct->ext, new); } new->offset[id] = newoff;
net/netfilter/nf_nat_core.c +7 -26
@@ -198,11 +198,9 @@ find_appropriate_src(struct net *net, const struct nf_nat_range *range) {unsigned int h =hash_by_src (net, tuple); -const struct nf_conn_nat *nat;const struct nf_conn *ct; -hlist_for_each_entry_rcu (nat, &nf_nat_bysource[h], bysource) { - ct = nat->ct; +hlist_for_each_entry_rcu (ct, &nf_nat_bysource[h], nat_bysource) {if (same_src (ct, tuple) &&net_eq (net,nf_ct_net (ct)) &&nf_ct_zone_equal (ct, zone,IP_CT_DIR_ORIGINAL )) {
@@ -435,8 +433,7 @@ nf_nat_setup_info(struct nf_conn *ct,spin_lock_bh (&nf_nat_lock);/* nf_conntrack_alter_reply might re-allocate extension aera */ nat =nfct_nat (ct); - nat->ct = ct; -hlist_add_head_rcu (&nat->bysource, +hlist_add_head_rcu (&ct->nat_bysource, &nf_nat_bysource[srchash]);spin_unlock_bh (&nf_nat_lock); }
@@ -543,7 +540,7 @@ static int nf_nat_proto_clean(struct nf_conn *ct, void *data)if (nf_nat_proto_remove (ct, data))return 1 ; -if (!nat || !nat->ct) +if (!nat)return 0 ;/* This netns is being destroyed, and conntrack has nat null binding.
@@ -556,9 +553,8 @@ static int nf_nat_proto_clean(struct nf_conn *ct, void *data)return 1 ;spin_lock_bh (&nf_nat_lock); -hlist_del_rcu (&nat->bysource); +hlist_del_rcu (&ct->nat_bysource); ct->status &= ~IPS_NAT_DONE_MASK ; - nat->ct =NULL ;spin_unlock_bh (&nf_nat_lock);add_timer (&ct->timeout);
@@ -688,27 +684,13 @@ static void nf_nat_cleanup_conntrack(struct nf_conn *ct) {struct nf_conn_nat *nat =nf_ct_ext_find (ct,NF_CT_EXT_NAT ); -if (nat ==NULL || nat->ct ==NULL ) +if (!nat)return ; -NF_CT_ASSERT (nat->ct->status &IPS_SRC_NAT_DONE ); - -spin_lock_bh (&nf_nat_lock); -hlist_del_rcu (&nat->bysource); -spin_unlock_bh (&nf_nat_lock); -} - -static void nf_nat_move_storage (void *new,void *old) -{ -struct nf_conn_nat *new_nat = new; -struct nf_conn_nat *old_nat = old; -struct nf_conn *ct = old_nat->ct; - -if (!ct || !(ct->status &IPS_SRC_NAT_DONE )) -return ; +NF_CT_ASSERT (ct->status &IPS_SRC_NAT_DONE );spin_lock_bh (&nf_nat_lock); -hlist_replace_rcu (&old_nat->bysource, &new_nat->bysource); +hlist_del_rcu (&ct->nat_bysource);spin_unlock_bh (&nf_nat_lock); }
@@ -716,7 +698,6 @@ static struct nf_ct_ext_type nat_extend __read_mostly = { .len = sizeof(struct nf_conn_nat), .align = __alignof__(struct nf_conn_nat), .destroy = nf_nat_cleanup_conntrack, - .move = nf_nat_move_storage, .id = NF_CT_EXT_NAT, .flags = NF_CT_EXT_F_PREALLOC, };
Signature · 1 line
2.7.3
Re: [PATCH nf-next 1/2] netfilter: move nat hlist_head to nf_conn
Quoted 22 lines · Florian Westphal
The nat extension structure is 32bytes in size on
x86_64:struct
nf_conn_nat{struct hlist_node bysource; /* 0 16 */ struct nf_conn * ct; /* 16 8 */ union nf_conntrack_nat_help help; /* 24 4 */ int masq_index; /* 28 4 */ /* size: 32, cachelines: 1, members: 4 */ /* last cacheline: 32 bytes */ };The hlist is needed to quickly check for possible tuple collisions when installing a new nat binding. Storing this in the extension area has two drawbacks:
1. We need ct backpointer to get the conntrack struct from the extension.
2. When reallocation of extension area occurs we need to fixup the bysource hash head viahlist_replace_rcu.We can avoid both by placing the
hlist_headinnf_connand placenf_connin the bysource hash rather than the extenstion.
Quoted 8 lines
We can also remove the ->move support; no other extension needs it.
Moving the entire nat extension into
nf_connwould be possible as well but then we have to add yet another callback for deletion from the bysource hash table rather than just using nat extension ->destroy hook for this.
nf_connsize doesn't increase due to aligment, followup patch replaceshlist_nodewith single pointer.
Applied, thanks.
Florian Westphal
maintainerPATCH 2/6 netfilter: nat: convert nat bysrc hash to rhashtable
In reply to Florian WestphalIt did use a fixed-size bucket list plus single lock to protect add/del.
Unlike the main conntrack table we only need to add and remove keys. Convert it to rhashtable to get table autosizing and per-bucket locking.
The maximum number of entries is -- as before -- tied to the number of conntracks so we do not need another upperlimit.
The change does not handle rhashtable_remove_fast error, only possible "error" is -ENOENT, and that is something that can happen legitimetely, e.g. because nat module was inserted at a later time and no src manip took place yet.
Tested with http-client-benchmark + httpterm with DNAT and SNAT rules in place.
--- include/net/netfilter/nf_conntrack.h | 3 +- include/net/netfilter/nf_nat.h | 1 + net/netfilter/nf_nat_core.c | 127 +++++++++++++++++++---------------- 3 files changed, 72 insertions(+), 59 deletions(-)
include/net/netfilter/nf_conntrack.h +2 -1
@@ -17,6 +17,7 @@#include <linux/bitops.h> #include <linux/compiler.h> #include <linux/atomic.h> +#include <linux/rhashtable.h> #include <linux/netfilter/nf_conntrack_tcp.h> #include <linux/netfilter/nf_conntrack_dccp.h>
@@ -118,7 +119,7 @@ struct nf_conn {struct nf_ct_ext *ext;#if IS_ENABLED (CONFIG_NF_NAT ) -struct hlist_node nat_bysource; +struct rhash_head nat_bysource;#endif /* Storage reserved for other modules, must be the last member */ union nf_conntrack_proto proto;
include/net/netfilter/nf_nat.h +1 -0
@@ -1,5 +1,6 @@#ifndef _NF_NAT_H#define _NF_NAT_H +#include <linux/rhashtable.h> #include <linux/netfilter_ipv4.h> #include <linux/netfilter/nf_nat.h> #include <net/netfilter/nf_conntrack_tuple.h>
net/netfilter/nf_nat_core.c +69 -58
215 lines not on this page · raw message
Signature · 1 line
2.7.3
Re: [PATCH nf-next 2/6] netfilter: nat: convert nat bysrc hash to rhashtable
Quoted 8 lines · Florian Westphal
It did use a fixed-size bucket list plus single lock to protect add/del.
Unlike the main conntrack table we only need to add and remove keys. Convert it to rhashtable to get table autosizing and per-bucket locking.
The maximum number of entries is -- as before -- tied to the number of conntracks so we do not need another upperlimit.
Quoted 7 lines
The change does not handle
rhashtable_remove_fasterror, only possible "error" is-ENOENT, and that is something that can happen legitimetely, e.g. because nat module was inserted at a later time and no src manip took place yet.Tested with http-client-benchmark + httpterm with DNAT and SNAT rules in place.
Applied, thanks.
I'm fixing this minor glitch here.
CC [M] net/netfilter/nf_nat_core.onet/netfilter/nf_nat_core.c: In function ‘nf_nat_proto_clean’:net/netfilter/nf_nat_core.c:555:6: warning: unused variable ‘err’ [-Wunused-variable]
int err;
^No need to resend.
--
To unsubscribe from this list: send the line "unsubscribe netfilter-devel" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html