netfilter: nf_tables: extend retpoline workarounds
Subsystems (2)
Florian Westphal
maintainercover netfilter: nf_tables: extend retpoline workarounds
- Skip the retpoline if-else cascade if the cpu is recent enough to not need the retpoline thunk.
- add objref and 'ct state' to the builtin-call list.
This means that 'ct state established,related accept' works without an indirect call.
Florian Westphal (3):
netfilter: nf_tables: add static key to skip retpoline workarounds
netfilter: nf_tables: avoid retpoline overhead for objref calls
netfilter: nf_tables: avoid retpoline overhead for some ct expression
calls
include/net/netfilter/nf_tables_core.h | 16 ++++++++ net/netfilter/Makefile | 6 +++ net/netfilter/nf_tables_core.c | 35 +++++++++++++++- net/netfilter/nft_ct.c | 39 ++++++++++++------ net/netfilter/nft_ct_fast.c | 56 ++++++++++++++++++++++++++ net/netfilter/nft_objref.c | 12 +++--- 6 files changed, 145 insertions(+), 19 deletions(-) create mode 100644 net/netfilter/nft_ct_fast.c
Signature · 1 line
2.38.2
Florian Westphal
maintainerPATCH 1/3 netfilter: nf_tables: add static key to skip retpoline workarounds
If CONFIG_RETPOLINE is enabled nf_tables avoids indirect calls for builtin expressions.
On newer cpus indirect calls do not go through the retpoline thunk anymore, even for RETPOLINE=y builds.
Just like with the new tc retpoline wrappers:
Add a static key to skip the if / else if cascade if the cpu does not require retpolines.
--- net/netfilter/nf_tables_core.c | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-)
net/netfilter/nf_tables_core.c +29 -1
@@ -21,6 +21,26 @@#include <net/netfilter/nf_log.h> #include <net/netfilter/nft_meta.h> +#if defined(CONFIG_RETPOLINE ) && defined(CONFIG_X86 ) + +static struct static_key_false nf_tables_skip_direct_calls; + +static bool nf_skip_indirect_calls (void ) +{ +return static_branch_likely (&nf_tables_skip_direct_calls); +} + +static void __init nf_skip_indirect_calls_enable (void ) +{ +if (!cpu_feature_enabled (X86_FEATURE_RETPOLINE )) +static_branch_enable (&nf_tables_skip_direct_calls); +} +#else +static inline bool nf_skip_indirect_calls (void ) {return false; } + +static inline void nf_skip_indirect_calls_enable (void ) { } +#endif + static noinline void __nft_trace_packet(struct nft_traceinfo *info,const struct nft_chain *chain,enum nft_trace_types type)
@@ -193,7 +213,12 @@ static void expr_call_ops_eval(const struct nft_expr *expr, struct nft_pktinfo *pkt) {#ifdef CONFIG_RETPOLINE -unsigned long e = (unsigned long )expr->ops->eval; +unsigned long e; + +if (nf_skip_indirect_calls ()) + goto indirect_call; + + e = (unsigned long )expr->ops->eval;#define X (e, fun) \ do { if ((e) == (unsigned long)(fun)) \ return fun(expr, regs, pkt); } while (0)
@@ -210,6 +235,7 @@ static void expr_call_ops_eval(const struct nft_expr *expr,X (e, nft_rt_get_eval);X (e, nft_bitwise_eval);#undef X +indirect_call:#endif /* CONFIG_RETPOLINE */ expr->ops->eval (expr, regs, pkt); }
@@ -369,6 +395,8 @@ int __init nf_tables_core_module_init(void) goto err; } +nf_skip_indirect_calls_enable (); +return 0 ; err:
Signature · 1 line
2.38.2
Florian Westphal
maintainerPATCH 2/3 netfilter: nf_tables: avoid retpoline overhead for objref calls
In reply to Florian Westphalobjref expression is builtin, so avoid calls to it for RETOLINE=y builds.
--- include/net/netfilter/nf_tables_core.h | 4 ++++ net/netfilter/nf_tables_core.c | 2 ++ net/netfilter/nft_objref.c | 12 ++++++------ 3 files changed, 12 insertions(+), 6 deletions(-)
include/net/netfilter/nf_tables_core.h +4 -0
@@ -164,4 +164,8 @@ void nft_payload_inner_eval(const struct nft_expr *expr, struct nft_regs *regs,const struct nft_pktinfo *pkt, struct nft_inner_tun_ctx *ctx); +void nft_objref_eval (const struct nft_expr *expr,struct nft_regs *regs, +const struct nft_pktinfo *pkt); +void nft_objref_map_eval (const struct nft_expr *expr,struct nft_regs *regs, +const struct nft_pktinfo *pkt);#endif /* _NET_NF_TABLES_CORE_H */
net/netfilter/nf_tables_core.c +2 -0
@@ -234,6 +234,8 @@ static void expr_call_ops_eval(const struct nft_expr *expr,X (e, nft_dynset_eval);X (e, nft_rt_get_eval);X (e, nft_bitwise_eval); +X (e, nft_objref_eval); +X (e, nft_objref_map_eval);#undef X indirect_call:#endif /* CONFIG_RETPOLINE */
net/netfilter/nft_objref.c +6 -6
@@ -13,9 +13,9 @@#define nft_objref_priv (expr) *((struct nft_object **)nft_expr_priv(expr)) -static void nft_objref_eval (const struct nft_expr *expr, -struct nft_regs *regs, -const struct nft_pktinfo *pkt) +void nft_objref_eval (const struct nft_expr *expr, +struct nft_regs *regs, +const struct nft_pktinfo *pkt) {struct nft_object *obj =nft_objref_priv (expr);
@@ -100,9 +100,9 @@ struct nft_objref_map {struct nft_set_binding binding; }; -static void nft_objref_map_eval (const struct nft_expr *expr, -struct nft_regs *regs, -const struct nft_pktinfo *pkt) +void nft_objref_map_eval (const struct nft_expr *expr, +struct nft_regs *regs, +const struct nft_pktinfo *pkt) {struct nft_objref_map *priv =nft_expr_priv (expr);const struct nft_set *set = priv->set;
Signature · 1 line
2.38.2
Florian Westphal
maintainerPATCH 3/3 netfilter: nf_tables: avoid retpoline overhead for some ct expression calls
In reply to Florian Westphalnft_ct expression cannot be made builtin to nf_tables without also forcing the conntrack itself to be builtin.
However, this can be avoided by splitting retrieval of a few selector keys that only need to access the nf_conn structure, i.e. no function calls to nf_conntrack code.
Many rulesets start with something like
"ct status established,related accept"
With this change, this no longer requires an indirect call.
--- include/net/netfilter/nf_tables_core.h | 12 ++++++ net/netfilter/Makefile | 6 +++ net/netfilter/nf_tables_core.c | 3 ++ net/netfilter/nft_ct.c | 39 ++++++++++++------ net/netfilter/nft_ct_fast.c | 56 ++++++++++++++++++++++++++ 5 files changed, 104 insertions(+), 12 deletions(-) create mode 100644 net/netfilter/nft_ct_fast.c
include/net/netfilter/nf_tables_core.h +12 -0
@@ -61,6 +61,16 @@ struct nft_immediate_expr {extern const struct nft_expr_ops nft_cmp_fast_ops;extern const struct nft_expr_ops nft_cmp16_fast_ops; +struct nft_ct { +enum nft_ct_keys key:8 ; +enum ip_conntrack_dir dir:8 ; +u8 len; +union { +u8 dreg; +u8 sreg; + }; +}; +struct nft_payload {enum nft_payload_bases base:8 ;u8 offset;
@@ -140,6 +150,8 @@ void nft_rt_get_eval(const struct nft_expr *expr, struct nft_regs *regs, const struct nft_pktinfo *pkt);void nft_counter_eval (const struct nft_expr *expr,struct nft_regs *regs,const struct nft_pktinfo *pkt); +void nft_ct_get_fast_eval (const struct nft_expr *expr, +struct nft_regs *regs,const struct nft_pktinfo *pkt);enum {NFT_PAYLOAD_CTX_INNER_TUN = (1 <<0 ),
net/netfilter/Makefile +6 -0
@@ -98,6 +98,12 @@ nf_tables-objs += nft_set_pipapo_avx2.o endif endif +ifdef CONFIG_NFT_CT +ifdef CONFIG_RETPOLINE +nf_tables-objs += nft_ct_fast.o +endif +endif + obj-$(CONFIG_NF_TABLES) += nf_tables.o obj-$(CONFIG_NFT_COMPAT) += nft_compat.o obj-$(CONFIG_NFT_CONNLIMIT) += nft_connlimit.o
net/netfilter/nf_tables_core.c +3 -0
@@ -228,6 +228,9 @@ static void expr_call_ops_eval(const struct nft_expr *expr,X (e, nft_counter_eval);X (e, nft_meta_get_eval);X (e, nft_lookup_eval); +#if IS_ENABLED (CONFIG_NFT_CT ) +X (e, nft_ct_get_fast_eval); +#endif X (e, nft_range_eval);X (e, nft_immediate_eval);X (e, nft_byteorder_eval);
net/netfilter/nft_ct.c +27 -12
@@ -12,7 +12,7 @@#include <linux/netlink.h> #include <linux/netfilter.h> #include <linux/netfilter/nf_tables.h> -#include <net/netfilter/nf_tables.h> +#include <net/netfilter/nf_tables_core.h> #include <net/netfilter/nf_conntrack.h> #include <net/netfilter/nf_conntrack_acct.h> #include <net/netfilter/nf_conntrack_tuple.h>
@@ -23,16 +23,6 @@#include <net/netfilter/nf_conntrack_l4proto.h> #include <net/netfilter/nf_conntrack_expect.h> -struct nft_ct { -enum nft_ct_keys key:8 ; -enum ip_conntrack_dir dir:8 ; -u8 len; -union { -u8 dreg; -u8 sreg; - }; -}; -struct nft_ct_helper_obj {struct nf_conntrack_helper *helper4;struct nf_conntrack_helper *helper6;
@@ -759,6 +749,18 @@ static bool nft_ct_set_reduce(struct nft_regs_track *track,return false; } +#ifdef CONFIG_RETPOLINE +static const struct nft_expr_ops nft_ct_get_fast_ops = { + .type = &nft_ct_type, + .size =NFT_EXPR_SIZE (sizeof (struct nft_ct )), + .eval = nft_ct_get_fast_eval, + .init = nft_ct_get_init, + .destroy = nft_ct_get_destroy, + .dump = nft_ct_get_dump, + .reduce = nft_ct_set_reduce, +}; +#endif +static const struct nft_expr_ops nft_ct_set_ops = { .type = &nft_ct_type, .size =NFT_EXPR_SIZE (sizeof (struct nft_ct )),
@@ -791,8 +793,21 @@ nft_ct_select_ops(const struct nft_ctx *ctx,if (tb[NFTA_CT_DREG ] && tb[NFTA_CT_SREG ])return ERR_PTR (-EINVAL ); -if (tb[NFTA_CT_DREG ]) +if (tb[NFTA_CT_DREG ]) { +#ifdef CONFIG_RETPOLINE +u32 k =ntohl (nla_get_be32 (tb[NFTA_CT_KEY ])); + +switch (k) { +case NFT_CT_STATE : +case NFT_CT_DIRECTION : +case NFT_CT_STATUS : +case NFT_CT_MARK : +case NFT_CT_SECMARK : +return &nft_ct_get_fast_ops; + } +#endif return &nft_ct_get_ops; + }if (tb[NFTA_CT_SREG ]) {#ifdef CONFIG_NF_CONNTRACK_ZONES
net/netfilter/nft_ct_fast.c new file +56 -0
56 lines not on this page · raw message
Signature · 1 line
2.38.2