Defer xfrm to be done post-GSO
Subsystems (7)
RFC PATCH 1/1 Defer xfrm to be done post-GSO
Experimental patch as promised in
http://marc.info/?l=linux-netdev&m=144899280626165&w=2
Disclaimer: this patch is only an experiment to see if xfrm can be done post-GSO to leverage from GSO benefits. I'm sharing to get some feedback on the general direction being pursued here.
At the moment, for single-stream iperf using esp-null and a 10G link, results in 3-3.5 Gbps vs the baseline of 1.8-2 Gbps.
While the 1.8 -> 3 Gbps is a step in the right direction, it still far away from the 6-9 Gbps that one can get with just GSO (6 Gbps if GRO is disabled), so input on other ways to improve this is invited.
Major things done in this patch:
- don't disable TSO in sk_setup_caps() if a dst->header_len is found
- in xfrm4_output, if GSO is applicable, bail out without esp header addition - that will get done after skb_segment()
- at the end of tcp_gso_segment() (when tcp segment is available), set things up for xfrm_output_one and trigger the esp_output
A 1-bit hole in sk_buff is used to track an skb that needs xfrm (might not need to burn that bit, but using it for now)
Signed-off-by Sowmini Varadhan <[email protected]>
--- include/linux/skbuff.h | 6 +++- include/net/xfrm.h | 1 + net/core/dev.c | 8 +++-- net/core/sock.c | 4 +++ net/ipv4/af_inet.c | 11 +++++++- net/ipv4/ip_output.c | 4 +++ net/ipv4/tcp_offload.c | 56 +++++++++++++++++++++++++++++++++++++++ net/ipv4/tcp_output.c | 1 + net/ipv4/xfrm4_mode_transport.c | 51 +++++++++++++++++++++++++++++++++++ net/ipv4/xfrm4_output.c | 9 ++++++ net/xfrm/xfrm_output.c | 3 +- 11 files changed, 147 insertions(+), 7 deletions(-)
include/linux/skbuff.h +4 -2
@@ -593,8 +593,8 @@ struct sk_buff { fclone:2, peeked:1, head_frag:1, - xmit_more:1; -/* one bit hole */ + xmit_more:1, + recirc:1; /* uses one bit hole */kmemcheck_bitfield_end (flags1);/* fields enclosed in headers_start/headers_end are copied
@@ -3577,5 +3577,7 @@ static inline unsigned int skb_gso_network_seglen(const struct sk_buff *skb)return hdr_len +skb_gso_transport_seglen (skb); } +#define XFRM_GSO 1/* use this for now to quickly toggle back to baseline */ +#endif /* __KERNEL__ */ #endif /* _LINUX_SKBUFF_H */
include/net/xfrm.h +1 -0
@@ -1508,6 +1508,7 @@ int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type);int xfrm_input_resume (struct sk_buff *skb,int nexthdr);int xfrm_output_resume (struct sk_buff *skb,int err);int xfrm_output (struct sock *sk,struct sk_buff *skb); +int xfrm_output_one (struct sk_buff *skb,int err);int xfrm_inner_extract_output (struct xfrm_state *x,struct sk_buff *skb);void xfrm_local_error (struct sk_buff *skb,int mtu);int xfrm4_extract_header (struct sk_buff *skb);
net/core/dev.c +5 -3
@@ -2804,7 +2804,7 @@ static struct sk_buff *validate_xmit_skb(struct sk_buff *skb, struct net_devicestruct sk_buff *validate_xmit_skb_list (struct sk_buff *skb,struct net_device *dev) { -struct sk_buff *next, *head =NULL , *tail; +struct sk_buff *next, *head =NULL , *tail =NULL ;for (; skb !=NULL ; skb = next) { next = skb->next;
@@ -3086,10 +3086,12 @@ static int __dev_queue_xmit(struct sk_buff *skb, void *accel_priv)/* If device/qdisc don't need skb->dst, release it right now while * its hot in this cpu cache. */ -if (dev->priv_flags &IFF_XMIT_DST_RELEASE ) +if ((dev->priv_flags &IFF_XMIT_DST_RELEASE ) && + !skb->recirc) {skb_dst_drop (skb); -else + }else {skb_dst_force (skb); + }#ifdef CONFIG_NET_SWITCHDEV /* Don't forward if offload device already forwarded */
net/core/sock.c +4 -0
@@ -1613,7 +1613,11 @@ void sk_setup_caps(struct sock *sk, struct dst_entry *dst) sk->sk_route_caps |=NETIF_F_GSO_SOFTWARE ; sk->sk_route_caps &= ~sk->sk_route_nocaps;if (sk_can_gso (sk)) { +#ifndef XFRM_GSO if (dst->header_len) { +#else +if (0 ) { +#endif sk->sk_route_caps &= ~NETIF_F_GSO_MASK ; }else { sk->sk_route_caps |=NETIF_F_SG |NETIF_F_HW_CSUM ;
net/ipv4/af_inet.c +10 -1
@@ -1203,6 +1203,7 @@ static struct sk_buff *inet_gso_segment(struct sk_buff *skb,int nhoff;int ihl;int id; +bool need_xfrm = skb->recirc; if (unlikely(skb_shinfo(skb)->gso_type & ~(SKB_GSO_TCPV4 |
@@ -1254,14 +1255,22 @@ static struct sk_buff *inet_gso_segment(struct sk_buff *skb, udpfrag = proto ==IPPROTO_UDP && !skb->encapsulation; ops =rcu_dereference (inet_offloads[proto]); -if (likely (ops && ops->callbacks.gso_segment)) +if (likely (ops && ops->callbacks.gso_segment)) { +/* tcp_gso_segment gets called here. It will add the +* XFRM by calling xfrm_output_one->esp_output. +* We will move things around to make space for the +* esp header in xfrm4_mode_transport.c (for transport +* mode- this is in xfrm4_transport_output_gso() +*/ segs = ops->callbacks.gso_segment (skb, features); + }if (IS_ERR_OR_NULL (segs)) goto out; skb = segs; do { + nhoff = skb_network_header(skb) - skb_mac_header(skb); iph = (struct iphdr *)(skb_mac_header(skb) + nhoff); if (udpfrag) { iph->id = htons(id);
net/ipv4/ip_output.c +4 -0
@@ -270,10 +270,14 @@ static int ip_finish_output(struct net *net, struct sock *sk, struct sk_buff *sk#if defined(CONFIG_NETFILTER ) && defined(CONFIG_XFRM )/* Policy lookup after SNAT yielded a new policy */ if (skb_dst (skb)->xfrm) { +if (sk_can_gso (sk) &&skb_is_gso (skb) && + sk->sk_gso_type ==SKB_GSO_TCPV4 ) + goto xfrm_gso;IPCB (skb)->flags |=IPSKB_REROUTED ;return dst_output (net, sk, skb); }#endif +xfrm_gso: mtu =ip_skb_dst_mtu (skb);if (skb_is_gso (skb))return ip_finish_output_gso (net, sk, skb, mtu);
net/ipv4/tcp_offload.c +56 -0
86 lines not on this page · raw message
net/ipv4/tcp_output.c +1 -0
@@ -951,6 +951,7 @@ static int tcp_transmit_skb(struct sock *sk, struct sk_buff *skb, int clone_it,skb_reset_transport_header (skb);skb_orphan (skb); + skb->recirc =0 ; skb->sk = sk; skb->destructor =skb_is_tcp_pure_ack (skb) ? sock_wfree : tcp_wfree;skb_set_hash_from_sk (skb, sk);
net/ipv4/xfrm4_mode_transport.c +51 -0
69 lines not on this page · raw message
net/ipv4/xfrm4_output.c +9 -0
@@ -98,6 +98,15 @@ static int __xfrm4_output(struct net *net, struct sock *sk, struct sk_buff *skb)int xfrm4_output (struct net *net,struct sock *sk,struct sk_buff *skb) { +#ifdef XFRM_GSO +if (sk_can_gso (sk) && sk->sk_gso_type ==SKB_GSO_TCPV4 && +skb_is_gso (skb)) { +BUG_ON (IPCB (skb)->flags &IPSKB_REROUTED ); + skb->recirc =1 ; +return (ip_output (net, sk, skb)); + } +#endif /* XFRM_GSO */ + return NF_HOOK_COND(NFPROTO_IPV4, NF_INET_POST_ROUTING, net, sk, skb, NULL, skb_dst(skb)->dev, __xfrm4_output,
net/xfrm/xfrm_output.c +2 -1
@@ -50,7 +50,7 @@ static struct dst_entry *skb_dst_pop(struct sk_buff *skb)return child; } -static int xfrm_output_one (struct sk_buff *skb,int err) +int xfrm_output_one (struct sk_buff *skb,int err) {struct dst_entry *dst =skb_dst (skb);struct xfrm_state *x = dst->xfrm;
@@ -128,6 +128,7 @@ static int xfrm_output_one(struct sk_buff *skb, int err) out:return err; } +EXPORT_SYMBOL_GPL (xfrm_output_one);int xfrm_output_resume (struct sk_buff *skb,int err) {
Signature · 1 line
1.7.1