Open vSwitch zerocopy upcall
Subsystems (4)
PATCH v5 1/3 net: Export skb_zerocopy() to zerocopy from one skb to another
In reply to Thomas GrafMake the skb zerocopy logic written for nfnetlink queue available for use by other modules.
--- include/linux/skbuff.h | 3 ++ net/core/skbuff.c | 85 ++++++++++++++++++++++++++++++++++++ net/netfilter/nfnetlink_queue_core.c | 59 ++----------------------- 3 files changed, 92 insertions(+), 55 deletions(-)
include/linux/skbuff.h +3 -0
@@ -2372,6 +2372,9 @@ int skb_splice_bits(struct sk_buff *skb, unsigned int offset, struct pipe_inode_info *pipe, unsigned int len, unsigned int flags);void skb_copy_and_csum_dev (const struct sk_buff *skb,u8 *to); +unsigned int skb_zerocopy_headlen (const struct sk_buff *from); +void skb_zerocopy (struct sk_buff *to,const struct sk_buff *from, +int len,int hlen);void skb_split (struct sk_buff *skb,struct sk_buff *skb1,const u32 len);int skb_shift (struct sk_buff *tgt,struct sk_buff *skb,int shiftlen);void skb_scrub_packet (struct sk_buff *skb,bool xnet);
net/core/skbuff.c +85 -0
91 lines not on this page · raw message
net/netfilter/nfnetlink_queue_core.c +4 -55
83 lines not on this page · raw message
Signature · 1 line
1.8.3.1
PATCH v5 2/3 openvswitch: Allow user space to announce ability to accept unaligned Netlink messages
In reply to Thomas Graf--- include/uapi/linux/openvswitch.h | 4 ++++ net/openvswitch/datapath.c | 4 ++++ net/openvswitch/datapath.h | 2 ++ 3 files changed, 10 insertions(+)
include/uapi/linux/openvswitch.h +4 -0
@@ -75,6 +75,7 @@ enum ovs_datapath_attr {OVS_DP_ATTR_UPCALL_PID ,/* Netlink PID to receive upcalls */ OVS_DP_ATTR_STATS ,/* struct ovs_dp_stats */ OVS_DP_ATTR_MEGAFLOW_STATS ,/* struct ovs_dp_megaflow_stats */ +OVS_DP_ATTR_USER_FEATURES ,/* OVS_DP_F_* */ __OVS_DP_ATTR_MAX };
@@ -106,6 +107,9 @@ struct ovs_vport_stats {__u64 tx_dropped;/* no space available in linux */ }; +/* Allow last Netlink attribute to be unaligned */ +#define OVS_DP_F_UNALIGNED (1 << 0) +/* Fixed logical ports. */ #define OVSP_LOCAL ((__u32)0)
net/openvswitch/datapath.c +4 -0
@@ -1061,6 +1061,7 @@ static struct genl_ops dp_flow_genl_ops[] = {static const struct nla_policy datapath_policy[OVS_DP_ATTR_MAX +1 ] = { [OVS_DP_ATTR_NAME ] = { .type =NLA_NUL_STRING , .len =IFNAMSIZ -1 }, [OVS_DP_ATTR_UPCALL_PID ] = { .type =NLA_U32 }, + [OVS_DP_ATTR_USER_FEATURES ] = { .type =NLA_U32 }, };static struct genl_family dp_datapath_genl_family = {
@@ -1217,6 +1218,9 @@ static int ovs_dp_cmd_new(struct sk_buff *skb, struct genl_info *info) parms.port_no =OVSP_LOCAL ; parms.upcall_portid =nla_get_u32 (a[OVS_DP_ATTR_UPCALL_PID ]); +if (a[OVS_DP_ATTR_USER_FEATURES ]) + dp->user_features =nla_get_u32 (a[OVS_DP_ATTR_USER_FEATURES ]); + vport =new_vport (&parms);if (IS_ERR (vport)) { err =PTR_ERR (vport);
net/openvswitch/datapath.h +2 -0
@@ -88,6 +88,8 @@ struct datapath {/* Network namespace ref. */ struct net *net;#endif + +u32 user_features; };/**
Signature · 1 line
1.8.3.1
v5 cover Open vSwitch zerocopy upcall
Respin of the zerocopy patches for the openvswitch upcall.
V5: - Removed padding requirement in user space
- Added OVS_DP_F_UNALIGNED flag let user space signal ability to receive unaligned Netlink messages, fall back to linear copy otherwise.
V4: - Daniel Borkmann pointed out that the style in skbuff.h has changed
in net-next, adapted to no longer using extern in headers.
V3: - Removed unneeded alignment of nlmsg_len after padding
V2: - Added skb_zerocopy_headlen() to calculate headroom of destination
buffer. This also takes care of the from->head_frag issue.
- Attribute alignment for frag_list case
- API documentation
- performance data for CHECKSUM_PARTIAL tx case
Thomas Graf (3):
net: Export skb_zerocopy() to zerocopy from one skb to another
openvswitch: Allow user space to announce ability to accept unaligned
Netlink messages
openvswitch: Use skb_zerocopy() for upcall
include/linux/skbuff.h | 3 ++ include/uapi/linux/openvswitch.h | 4 ++ net/core/skbuff.c | 85 ++++++++++++++++++++++++++++++++++++ net/netfilter/nfnetlink_queue_core.c | 59 ++----------------------- net/openvswitch/datapath.c | 58 ++++++++++++++++-------- net/openvswitch/datapath.h | 2 + 6 files changed, 139 insertions(+), 72 deletions(-)
Signature · 1 line
1.8.3.1
PATCH v5 3/3 openvswitch: Use skb_zerocopy() for upcall
Use of skb_zerocopy() avoids the expensive call to memcpy() when copying the packet data into the Netlink skb. Completes checksum through skb_checksum_help() if needed.
Cost of memcpy is significantly reduced from:
+ 7.48% vhost-8471 [k] memcpy + 5.57% ovs-vswitchd [k] memcpy + 2.81% vhost-8471 [k] csum_partial_copy_generic
to:
+ 5.72% ovs-vswitchd [k] memcpy + 3.32% vhost-5153 [k] memcpy + 0.68% vhost-5153 [k] skb_zerocopy
(megaflows disabled)
--- net/openvswitch/datapath.c | 54 +++++++++++++++++++++++++++++++--------------- 1 file changed, 37 insertions(+), 17 deletions(-)
net/openvswitch/datapath.c +37 -17
@@ -108,10 +108,10 @@ int lockdep_ovsl_is_held(void)#endif static struct vport *new_vport (const struct vport_parms *); -static int queue_gso_packets (struct net *,int dp_ifindex,struct sk_buff *, -const struct dp_upcall_info *); -static int queue_userspace_packet (struct net *,int dp_ifindex, -struct sk_buff *, +static int queue_gso_packets (struct datapath *,struct net *,int dp_ifindex, +struct sk_buff *,const struct dp_upcall_info *); +static int queue_userspace_packet (struct datapath *,struct net *, +int dp_ifindex,struct sk_buff *,const struct dp_upcall_info *);/* Must be called with rcu_read_lock or ovs_mutex. */
@@ -292,9 +292,9 @@ int ovs_dp_upcall(struct datapath *dp, struct sk_buff *skb, }if (!skb_is_gso (skb)) - err =queue_userspace_packet (ovs_dp_get_net (dp), dp_ifindex, skb, upcall_info); + err =queue_userspace_packet (dp,ovs_dp_get_net (dp), dp_ifindex, skb, upcall_info);else - err =queue_gso_packets (ovs_dp_get_net (dp), dp_ifindex, skb, upcall_info); + err =queue_gso_packets (dp,ovs_dp_get_net (dp), dp_ifindex, skb, upcall_info);if (err) goto err;
@@ -310,7 +310,7 @@ err:return err; } -static int queue_gso_packets (struct net *net,int dp_ifindex, +static int queue_gso_packets (struct datapath *dp,struct net *net,int dp_ifindex,struct sk_buff *skb,const struct dp_upcall_info *upcall_info) {
@@ -327,7 +327,7 @@ static int queue_gso_packets(struct net *net, int dp_ifindex,/* Queue all of the segments. */ skb = segs; do { - err = queue_userspace_packet(net, dp_ifindex, skb, upcall_info); + err = queue_userspace_packet(dp, net, dp_ifindex, skb, upcall_info); if (err) break;
@@ -381,10 +381,11 @@ static size_t key_attr_size(void) }static size_t upcall_msg_size (const struct sk_buff *skb, -const struct nlattr *userdata) +const struct nlattr *userdata, +unsigned int hdrlen) {size_t size =NLMSG_ALIGN (sizeof (struct ovs_header )) - +nla_total_size (skb->len)/* OVS_PACKET_ATTR_PACKET */ + +nla_total_size (hdrlen)/* OVS_PACKET_ATTR_PACKET */ +nla_total_size (key_attr_size ());/* OVS_PACKET_ATTR_KEY */ /* OVS_PACKET_ATTR_USERDATA */
@@ -394,14 +395,15 @@ static size_t upcall_msg_size(const struct sk_buff *skb,return size; } -static int queue_userspace_packet (struct net *net,int dp_ifindex, -struct sk_buff *skb, +static int queue_userspace_packet (struct datapath *dp,struct net *net, +int dp_ifindex,struct sk_buff *skb,const struct dp_upcall_info *upcall_info) {struct ovs_header *upcall;struct sk_buff *nskb =NULL ;struct sk_buff *user_skb;/* to be queued to userspace */ struct nlattr *nla; +unsigned int hlen;int err;if (vlan_tx_tag_present (skb)) {
@@ -422,7 +424,21 @@ static int queue_userspace_packet(struct net *net, int dp_ifindex, goto out; } - user_skb =genlmsg_new (upcall_msg_size (skb, upcall_info->userdata),GFP_ATOMIC ); +/* Complete checksum if needed */ +if (skb->ip_summed ==CHECKSUM_PARTIAL && + (err =skb_checksum_help (skb))) + goto out; + +/* Older versions of OVS user space enforce alignment of the last +* Netlink attribute to NLA_ALIGNTO which would require extensive +* padding logic. Only perform zerocopy if padding is not required. +*/ +if (dp->user_features &OVS_DP_F_UNALIGNED ) + hlen =skb_zerocopy_headlen (skb); +else + hlen = skb->len; + + user_skb =genlmsg_new (upcall_msg_size (skb, upcall_info->userdata, hlen),GFP_ATOMIC );if (!user_skb) { err = -ENOMEM ; goto out;
@@ -441,13 +457,17 @@ static int queue_userspace_packet(struct net *net, int dp_ifindex,nla_len (upcall_info->userdata), nla_data(upcall_info->userdata)); - nla =__nla_reserve (user_skb,OVS_PACKET_ATTR_PACKET , skb->len); +/* Only reserve room for attribute header, packet data is added +* in skb_zerocopy() */ +if (!(nla =nla_reserve (user_skb,OVS_PACKET_ATTR_PACKET ,0 ))) + goto out; + nla->nla_len =nla_attr_size (skb->len); + +skb_zerocopy (user_skb, skb, skb->len, hlen); -skb_copy_and_csum_dev (skb,nla_data (nla)); + ((struct nlmsghdr *) user_skb->data)->nlmsg_len = user_skb->len; -genlmsg_end (user_skb, upcall); err =genlmsg_unicast (net, user_skb, upcall_info->portid); - out:kfree_skb (nskb);return err;
Signature · 1 line
1.8.3.1