indirect_call_wrapper: extend indirect wrapper to support up to 4 calls
Subsystems (3)
PATCH v2 1/2 indirect_call_wrapper: extend indirect wrapper to support up to 4 calls
There are many places where 2 annotations are not enough. This patch adds INDIRECT_CALL_3 and INDIRECT_CALL_4 to cover such cases.
--- include/linux/indirect_call_wrapper.h | 12 ++++++++++++ 1 file changed, 12 insertions(+)
include/linux/indirect_call_wrapper.h +12 -0
@@ -23,6 +23,16 @@likely (f == f2) ?f2 (__VA_ARGS__) : \INDIRECT_CALL_1 (f, f1, __VA_ARGS__); \ }) +#define INDIRECT_CALL_3 (f, f3, f2, f1, ...) \ + ({ \ + likely(f == f3) ? f3(__VA_ARGS__) : \ + INDIRECT_CALL_2(f, f2, f1, __VA_ARGS__); \ + }) +#define INDIRECT_CALL_4 (f, f4, f3, f2, f1, ...) \ + ({ \ + likely(f == f4) ? f4(__VA_ARGS__) : \ + INDIRECT_CALL_3(f, f3, f2, f1, __VA_ARGS__); \ + })#define INDIRECT_CALLABLE_DECLARE (f) f#define INDIRECT_CALLABLE_SCOPE
@@ -30,6 +40,8 @@#else #define INDIRECT_CALL_1 (f, f1, ...) f(__VA_ARGS__)#define INDIRECT_CALL_2 (f, f2, f1, ...) f(__VA_ARGS__) +#define INDIRECT_CALL_3 (f, f3, f2, f1, ...) f(__VA_ARGS__) +#define INDIRECT_CALL_4 (f, f4, f3, f2, f1, ...) f(__VA_ARGS__)#define INDIRECT_CALLABLE_DECLARE (f)#define INDIRECT_CALLABLE_SCOPE static#endif
Signature · 1 line
2.27.0.111.gc72c7da667-goog
PATCH v2 2/2 ipv6: fib6: avoid indirect calls from fib6_rule_lookup
It was reported that a considerable amount of cycles were spent on the expensive indirect calls on fib6_rule_lookup. This patch introduces an inline helper called pol_route_func that uses the indirect_call_wrappers to avoid the indirect calls.
This patch saves around 50ns per call.
Performance was measured on the receiver by checking the amount of syncookies that server was able to generate under a synflood load.
Traffic was generated using trafgen[1] which was pushing around 1Mpps on a single queue. Receiver was using only one rx queue which help to create a bottle neck and make the experiment rx-bounded.
These are the syncookies generated over 10s from the different runs:
Whithout the patch:
TcpExtSyncookiesSent 3553749 0.0 TcpExtSyncookiesSent 3550895 0.0 TcpExtSyncookiesSent 3553845 0.0 TcpExtSyncookiesSent 3541050 0.0 TcpExtSyncookiesSent 3539921 0.0 TcpExtSyncookiesSent 3557659 0.0 TcpExtSyncookiesSent 3526812 0.0 TcpExtSyncookiesSent 3536121 0.0 TcpExtSyncookiesSent 3529963 0.0 TcpExtSyncookiesSent 3536319 0.0
With the patch:
TcpExtSyncookiesSent 3611786 0.0 TcpExtSyncookiesSent 3596682 0.0 TcpExtSyncookiesSent 3606878 0.0 TcpExtSyncookiesSent 3599564 0.0 TcpExtSyncookiesSent 3601304 0.0 TcpExtSyncookiesSent 3609249 0.0 TcpExtSyncookiesSent 3617437 0.0 TcpExtSyncookiesSent 3608765 0.0 TcpExtSyncookiesSent 3620205 0.0 TcpExtSyncookiesSent 3601895 0.0
Without the patch the average is 354263 pkt/s or 2822 ns/pkt and with the patch the average is 360738 pkt/s or 2772 ns/pkt which gives an estimate of 50 ns per packet.
Changelog since v1:
- Change ordering in the ICW (Paolo Abeni)
--- include/net/ip6_fib.h | 36 ++++++++++++++++++++++++++++++++++++ net/ipv6/fib6_rules.c | 9 ++++++--- net/ipv6/ip6_fib.c | 3 ++- net/ipv6/route.c | 8 ++++---- 4 files changed, 48 insertions(+), 8 deletions(-)
include/net/ip6_fib.h +36 -0
@@ -19,6 +19,7 @@#include <net/netlink.h> #include <net/inetpeer.h> #include <net/fib_notifier.h> +#include <linux/indirect_call_wrapper.h> #ifdef CONFIG_IPV6_MULTIPLE_TABLES #define FIB6_TABLE_HASHSZ 256
@@ -552,6 +553,41 @@ struct bpf_iter__ipv6_route { };#endif +INDIRECT_CALLABLE_DECLARE(struct rt6_info *ip6_pol_route_output(struct net *net, +struct fib6_table *table, +struct flowi6 *fl6, +const struct sk_buff *skb, +int flags)); +INDIRECT_CALLABLE_DECLARE(struct rt6_info *ip6_pol_route_input(struct net *net, +struct fib6_table *table, +struct flowi6 *fl6, +const struct sk_buff *skb, +int flags)); +INDIRECT_CALLABLE_DECLARE(struct rt6_info *__ip6_route_redirect(struct net *net, +struct fib6_table *table, +struct flowi6 *fl6, +const struct sk_buff *skb, +int flags)); +INDIRECT_CALLABLE_DECLARE(struct rt6_info *ip6_pol_route_lookup(struct net *net, +struct fib6_table *table, +struct flowi6 *fl6, +const struct sk_buff *skb, +int flags)); +static inline struct rt6_info *pol_lookup_func (pol_lookup_t lookup, +struct net *net, +struct fib6_table *table, +struct flowi6 *fl6, +const struct sk_buff *skb, +int flags) +{ +return INDIRECT_CALL_4 (lookup, + ip6_pol_route_output, + ip6_pol_route_input, + ip6_pol_route_lookup, + __ip6_route_redirect, + net, table, fl6, skb, flags); +} +#ifdef CONFIG_IPV6_MULTIPLE_TABLES static inline bool fib6_has_custom_rules (const struct net *net) {
net/ipv6/fib6_rules.c +6 -3
@@ -111,11 +111,13 @@ struct dst_entry *fib6_rule_lookup(struct net *net, struct flowi6 *fl6, }else {struct rt6_info *rt; - rt =lookup (net, net->ipv6.fib6_local_tbl, fl6, skb, flags); + rt =pol_lookup_func (lookup, + net, net->ipv6.fib6_local_tbl, fl6, skb, flags);if (rt != net->ipv6.ip6_null_entry && rt->dst.error != -EAGAIN )return &rt->dst;ip6_rt_put_flags (rt, flags); - rt =lookup (net, net->ipv6.fib6_main_tbl, fl6, skb, flags); + rt =pol_lookup_func (lookup, + net, net->ipv6.fib6_main_tbl, fl6, skb, flags);if (rt->dst.error != -EAGAIN )return &rt->dst;ip6_rt_put_flags (rt, flags);
@@ -226,7 +228,8 @@ static int __fib6_rule_action(struct fib_rule *rule, struct flowi *flp, goto out; } - rt =lookup (net, table, flp6, arg->lookup_data, flags); + rt =pol_lookup_func (lookup, + net, table, flp6, arg->lookup_data, flags);if (rt != net->ipv6.ip6_null_entry) { err =fib6_rule_saddr (net, rule, flags, flp6,ip6_dst_idev (&rt->dst)->dev);
net/ipv6/ip6_fib.c +2 -1
@@ -314,7 +314,8 @@ struct dst_entry *fib6_rule_lookup(struct net *net, struct flowi6 *fl6, {struct rt6_info *rt; - rt =lookup (net, net->ipv6.fib6_main_tbl, fl6, skb, flags); + rt =pol_lookup_func (lookup, + net, net->ipv6.fib6_main_tbl, fl6, skb, flags);if (rt->dst.error == -EAGAIN ) {ip6_rt_put_flags (rt, flags); rt = net->ipv6.ip6_null_entry;
net/ipv6/route.c +4 -4
@@ -1207,7 +1207,7 @@ static struct rt6_info *ip6_create_rt_rcu(const struct fib6_result *res)return nrt; } -static struct rt6_info *ip6_pol_route_lookup(struct net *net, +INDIRECT_CALLABLE_SCOPE struct rt6_info *ip6_pol_route_lookup(struct net *net, struct fib6_table *table, struct flowi6 *fl6, const struct sk_buff *skb,
@@ -2274,7 +2274,7 @@ struct rt6_info *ip6_pol_route(struct net *net, struct fib6_table *table, }EXPORT_SYMBOL_GPL (ip6_pol_route); -static struct rt6_info *ip6_pol_route_input(struct net *net, +INDIRECT_CALLABLE_SCOPE struct rt6_info *ip6_pol_route_input(struct net *net, struct fib6_table *table, struct flowi6 *fl6, const struct sk_buff *skb,
@@ -2465,7 +2465,7 @@ void ip6_route_input(struct sk_buff *skb) &fl6, skb, flags)); } -static struct rt6_info *ip6_pol_route_output(struct net *net, +INDIRECT_CALLABLE_SCOPE struct rt6_info *ip6_pol_route_output(struct net *net, struct fib6_table *table, struct flowi6 *fl6, const struct sk_buff *skb,
@@ -2912,7 +2912,7 @@ struct ip6rd_flowi {struct in6_addr gateway; }; -static struct rt6_info *__ip6_route_redirect(struct net *net, +INDIRECT_CALLABLE_SCOPE struct rt6_info *__ip6_route_redirect(struct net *net, struct fib6_table *table, struct flowi6 *fl6, const struct sk_buff *skb,
Signature · 1 line
2.27.0.111.gc72c7da667-goog
Quoted 30 lines · Brian Vazquez
There are many places where 2 annotations are not enough. This patch adds
INDIRECT_CALL_3andINDIRECT_CALL_4to cover such cases.Signed-off-by Brian Vazquez <[email protected]>--- include/linux/indirect_call_wrapper.h | 12 ++++++++++++ 1 file changed, 12 insertions(+)include/linux/indirect_call_wrapper.h +10
-0@@ -23,6 +23,16 @@likely (f == f2) ?f2 (__VA_ARGS__) : \INDIRECT_CALL_1 (f, f1, __VA_ARGS__); \ }) +#define INDIRECT_CALL_3 (f, f3, f2, f1, ...) \ + ({ \ + likely(f == f3) ? f3(__VA_ARGS__) : \ + INDIRECT_CALL_2(f, f2, f1, __VA_ARGS__); \ + }) +#define INDIRECT_CALL_4 (f, f4, f3, f2, f1, ...) \ + ({ \ + likely(f == f4) ? f4(__VA_ARGS__) : \ + INDIRECT_CALL_3(f, f3, f2, f1, __VA_ARGS__); \ + })#define INDIRECT_CALLABLE_DECLARE (f) f#define INDIRECT_CALLABLE_SCOPE
include/linux/indirect_call_wrapper.h line 47
Quoted 8 lines
include/linux/indirect_call_wrapper.h +2
-0@@ -30,6 +40,8 @@#else #define INDIRECT_CALL_1 (f, f1, ...) f(__VA_ARGS__)#define INDIRECT_CALL_2 (f, f2, f1, ...) f(__VA_ARGS__) +#define INDIRECT_CALL_3 (f, f3, f2, f1, ...) f(__VA_ARGS__) +#define INDIRECT_CALL_4 (f, f4, f3, f2, f1, ...) f(__VA_ARGS__)#define INDIRECT_CALLABLE_DECLARE (f)#define INDIRECT_CALLABLE_SCOPE static#endif
Re: [PATCH v2 net-next 2/2] ipv6: fib6: avoid indirect calls from fib6_rule_lookup
In reply to Brian VazquezQuoted 190 lines · Brian Vazquez
It was reported that a considerable amount of cycles were spent on the expensive indirect calls on
fib6_rule_lookup. This patch introduces an inline helper calledpol_route_functhat uses theindirect_call_wrappersto avoid the indirect calls.This patch saves around 50ns per call.
Performance was measured on the receiver by checking the amount of syncookies that server was able to generate under a synflood load.
Traffic was generated using trafgen[1] which was pushing around 1Mpps on a single queue. Receiver was using only one rx queue which help to create a bottle neck and make the experiment rx-bounded.
These are the syncookies generated over 10s from the different runs:
Whithout the patch:
TcpExtSyncookiesSent 3553749 0.0 TcpExtSyncookiesSent 3550895 0.0 TcpExtSyncookiesSent 3553845 0.0 TcpExtSyncookiesSent 3541050 0.0 TcpExtSyncookiesSent 3539921 0.0 TcpExtSyncookiesSent 3557659 0.0 TcpExtSyncookiesSent 3526812 0.0 TcpExtSyncookiesSent 3536121 0.0 TcpExtSyncookiesSent 3529963 0.0 TcpExtSyncookiesSent 3536319 0.0With the patch:
TcpExtSyncookiesSent 3611786 0.0 TcpExtSyncookiesSent 3596682 0.0 TcpExtSyncookiesSent 3606878 0.0 TcpExtSyncookiesSent 3599564 0.0 TcpExtSyncookiesSent 3601304 0.0 TcpExtSyncookiesSent 3609249 0.0 TcpExtSyncookiesSent 3617437 0.0 TcpExtSyncookiesSent 3608765 0.0 TcpExtSyncookiesSent 3620205 0.0 TcpExtSyncookiesSent 3601895 0.0Without the patch the average is 354263 pkt/s or 2822 ns/pkt and with the patch the average is 360738 pkt/s or 2772 ns/pkt which gives an estimate of 50 ns per packet.
Changelog since v1:
- Change ordering in the ICW (Paolo Abeni)Cc Luigi Rizzo <[email protected]>Cc Paolo Abeni <[email protected]>Reported-by Eric Dumazet <[email protected]>Signed-off-by Brian Vazquez <[email protected]>--- include/net/ip6_fib.h | 36 ++++++++++++++++++++++++++++++++++++ net/ipv6/fib6_rules.c | 9 ++++++--- net/ipv6/ip6_fib.c | 3 ++- net/ipv6/route.c | 8 ++++---- 4 files changed, 48 insertions(+), 8 deletions(-)include/net/ip6_fib.h +36
-0@@ -19,6 +19,7 @@#include <net/netlink.h> #include <net/inetpeer.h> #include <net/fib_notifier.h> +#include <linux/indirect_call_wrapper.h> #ifdef CONFIG_IPV6_MULTIPLE_TABLES #define FIB6_TABLE_HASHSZ 256@@ -552,6 +553,41 @@ struct bpf_iter__ipv6_route { };#endif +INDIRECT_CALLABLE_DECLARE(struct rt6_info *ip6_pol_route_output(struct net *net, +struct fib6_table *table, +struct flowi6 *fl6, +const struct sk_buff *skb, +int flags)); +INDIRECT_CALLABLE_DECLARE(struct rt6_info *ip6_pol_route_input(struct net *net, +struct fib6_table *table, +struct flowi6 *fl6, +const struct sk_buff *skb, +int flags)); +INDIRECT_CALLABLE_DECLARE(struct rt6_info *__ip6_route_redirect(struct net *net, +struct fib6_table *table, +struct flowi6 *fl6, +const struct sk_buff *skb, +int flags)); +INDIRECT_CALLABLE_DECLARE(struct rt6_info *ip6_pol_route_lookup(struct net *net, +struct fib6_table *table, +struct flowi6 *fl6, +const struct sk_buff *skb, +int flags)); +static inline struct rt6_info *pol_lookup_func (pol_lookup_t lookup, +struct net *net, +struct fib6_table *table, +struct flowi6 *fl6, +const struct sk_buff *skb, +int flags) +{ +return INDIRECT_CALL_4 (lookup, + ip6_pol_route_output, + ip6_pol_route_input, + ip6_pol_route_lookup, + __ip6_route_redirect, + net, table, fl6, skb, flags); +} +#ifdef CONFIG_IPV6_MULTIPLE_TABLES static inline bool fib6_has_custom_rules (const struct net *net) {net/ipv6/fib6_rules.c +6
-3@@ -111,11 +111,13 @@ struct dst_entry *fib6_rule_lookup(struct net *net, struct flowi6 *fl6, }else {struct rt6_info *rt; - rt =lookup (net, net->ipv6.fib6_local_tbl, fl6, skb, flags); + rt =pol_lookup_func (lookup, + net, net->ipv6.fib6_local_tbl, fl6, skb, flags);if (rt != net->ipv6.ip6_null_entry && rt->dst.error != -EAGAIN )return &rt->dst;ip6_rt_put_flags (rt, flags); - rt =lookup (net, net->ipv6.fib6_main_tbl, fl6, skb, flags); + rt =pol_lookup_func (lookup, + net, net->ipv6.fib6_main_tbl, fl6, skb, flags);if (rt->dst.error != -EAGAIN )return &rt->dst;ip6_rt_put_flags (rt, flags);@@ -226,7 +228,8 @@ static int __fib6_rule_action(struct fib_rule *rule, struct flowi *flp, goto out; } - rt =lookup (net, table, flp6, arg->lookup_data, flags); + rt =pol_lookup_func (lookup, + net, table, flp6, arg->lookup_data, flags);if (rt != net->ipv6.ip6_null_entry) { err =fib6_rule_saddr (net, rule, flags, flp6,ip6_dst_idev (&rt->dst)->dev);net/ipv6/ip6_fib.c +2
-1@@ -314,7 +314,8 @@ struct dst_entry *fib6_rule_lookup(struct net *net, struct flowi6 *fl6, {struct rt6_info *rt; - rt =lookup (net, net->ipv6.fib6_main_tbl, fl6, skb, flags); + rt =pol_lookup_func (lookup, + net, net->ipv6.fib6_main_tbl, fl6, skb, flags);if (rt->dst.error == -EAGAIN ) {ip6_rt_put_flags (rt, flags); rt = net->ipv6.ip6_null_entry;net/ipv6/route.c +3
-3@@ -1207,7 +1207,7 @@ static struct rt6_info *ip6_create_rt_rcu(const struct fib6_result *res)return nrt; } -static struct rt6_info *ip6_pol_route_lookup(struct net *net, +INDIRECT_CALLABLE_SCOPE struct rt6_info *ip6_pol_route_lookup(struct net *net, struct fib6_table *table, struct flowi6 *fl6, const struct sk_buff *skb,@@ -2274,7 +2274,7 @@ struct rt6_info *ip6_pol_route(struct net *net, struct fib6_table *table, }EXPORT_SYMBOL_GPL (ip6_pol_route); -static struct rt6_info *ip6_pol_route_input(struct net *net, +INDIRECT_CALLABLE_SCOPE struct rt6_info *ip6_pol_route_input(struct net *net, struct fib6_table *table, struct flowi6 *fl6, const struct sk_buff *skb,@@ -2465,7 +2465,7 @@ void ip6_route_input(struct sk_buff *skb) &fl6, skb, flags)); } -static struct rt6_info *ip6_pol_route_output(struct net *net, +INDIRECT_CALLABLE_SCOPE struct rt6_info *ip6_pol_route_output(struct net *net, struct fib6_table *table, struct flowi6 *fl6, const struct sk_buff *skb,
net/ipv6/route.c struct ip6rd_flowi { · line 2918
Quoted 8 lines
net/ipv6/route.c +1
-1@@ -2912,7 +2912,7 @@ struct ip6rd_flowi {struct in6_addr gateway; }; -static struct rt6_info *__ip6_route_redirect(struct net *net, +INDIRECT_CALLABLE_SCOPE struct rt6_info *__ip6_route_redirect(struct net *net, struct fib6_table *table, struct flowi6 *fl6, const struct sk_buff *skb,
From: Brian Vazquez <[email protected]>
Date: Tue, 23 Jun 2020 09:42:31 -0700
Quoted 4 lines
There are many places where 2 annotations are not enough. This patch adds
INDIRECT_CALL_3andINDIRECT_CALL_4to cover such cases.Signed-off-by Brian Vazquez <[email protected]>
Applied.
Re: [PATCH v2 net-next 2/2] ipv6: fib6: avoid indirect calls from fib6_rule_lookup
In reply to Brian VazquezFrom: Brian Vazquez <[email protected]>
Date: Tue, 23 Jun 2020 09:42:32 -0700
Quoted 17 lines
It was reported that a considerable amount of cycles were spent on the expensive indirect calls on
fib6_rule_lookup. This patch introduces an inline helper calledpol_route_functhat uses theindirect_call_wrappersto avoid the indirect calls.This patch saves around 50ns per call.
Performance was measured on the receiver by checking the amount of syncookies that server was able to generate under a synflood load.
Traffic was generated using trafgen[1] which was pushing around 1Mpps on a single queue. Receiver was using only one rx queue which help to create a bottle neck and make the experiment rx-bounded.
These are the syncookies generated over 10s from the different runs:
Whithout the patch:
...
Quoted 1 line
With the patch:
...
Quoted 6 lines
Without the patch the average is 354263 pkt/s or 2822 ns/pkt and with the patch the average is 360738 pkt/s or 2772 ns/pkt which gives an estimate of 50 ns per packet.
Quoted 7 lines
Changelog since v1:
- Change ordering in the ICW (Paolo Abeni)Cc Luigi Rizzo <[email protected]>Cc Paolo Abeni <[email protected]>Reported-by Eric Dumazet <[email protected]>Signed-off-by Brian Vazquez <[email protected]>
Applied.