Skip to contentSkip to conversation

vmxnet3: upgrade to version 4

PATCH Superseded Ronak Doshi 2 participants 6 messages May 28, 2020

Subsystems (3)

v3 of 4 Superseded Latest is v4 →

lore
Share & tools
lore
#0 lore raw

v3 cover vmxnet3: upgrade to version 4

vmxnet3 emulation has recently added several new features which includes offload support for tunnel packets, support for new commands the driver can issue to emulation, change in descriptor fields, etc. This patch series extends the vmxnet3 driver to leverage these new features.

Compatibility is maintained using existing vmxnet3 versioning mechanism as follows:
- new features added to vmxnet3 emulation are associated with new vmxnet3 version viz. vmxnet3 version 4.
- emulation advertises all the versions it supports to the driver.
- during initialization, vmxnet3 driver picks the highest version number supported by both the emulation and the driver and configures emulation to run at that version.

In particular, following changes are introduced:

Patch 1:
This patch introduces utility macros for vmxnet3 version 4 comparison and updates Copyright information.

Patch 2:
This patch implements get_rss_hash_opts and set_rss_hash_opts methods to allow querying and configuring different Rx flow hash configurations which can be used to support UDP/ESP RSS.

Patch 3:
This patch introduces segmentation and checksum offload support for encapsulated packets. This avoids segmenting and calculating checksum for each segment and hence gives performance boost.

Patch 4:
With all vmxnet3 version 4 changes incorporated in the vmxnet3 driver, with this patch, the driver can configure emulation to run at vmxnet3 version 4.

Changes in v2 -> v3:
- fixed get_rss_hash_opts to return correct values for udp rss

Changes in v2:
- Fixed compilation issue due to missing closed brace
- added fallthrough comment

Ronak Doshi (4):
vmxnet3: prepare for version 4 changes
vmxnet3: add support to get/set rx flow hash
vmxnet3: add geneve and vxlan tunnel offload support
vmxnet3: update to version 4

 drivers/net/vmxnet3/Makefile          |   2 +-
 drivers/net/vmxnet3/upt1_defs.h       |   5 +-
 drivers/net/vmxnet3/vmxnet3_defs.h    |  31 ++--
 drivers/net/vmxnet3/vmxnet3_drv.c     | 164 ++++++++++++++++++---
 drivers/net/vmxnet3/vmxnet3_ethtool.c | 263 +++++++++++++++++++++++++++++++++-
 drivers/net/vmxnet3/vmxnet3_int.h     |  25 +++-
 6 files changed, 448 insertions(+), 42 deletions(-)
Signature · 1 line
2.11.0
#2 lore raw

PATCH v3 2/4 vmxnet3: add support to get/set rx flow hash

With vmxnet3 version 4, the emulation supports multiqueue(RSS) for UDP and ESP traffic. A guest can enable/disable RSS for UDP/ESP over IPv4/IPv6 by issuing commands introduced in this patch. ESP ipv6 is not yet supported in this patch.

This patch implements get_rss_hash_opts and set_rss_hash_opts methods to allow querying and configuring different Rx flow hash configurations.

---
 drivers/net/vmxnet3/vmxnet3_defs.h    |  12 ++
 drivers/net/vmxnet3/vmxnet3_drv.c     |  39 ++++++
 drivers/net/vmxnet3/vmxnet3_ethtool.c | 219 +++++++++++++++++++++++++++++++++-
 drivers/net/vmxnet3/vmxnet3_int.h     |   4 +
 4 files changed, 272 insertions(+), 2 deletions(-)
drivers/net/vmxnet3/vmxnet3_defs.h +12 -0
index c77274228a3e..aac97fac1186 100644
@@ -82,6 +82,7 @@ enum {
 	VMXNET3_CMD_RESERVED3,
 	VMXNET3_CMD_SET_COALESCE,
 	VMXNET3_CMD_REGISTER_MEMREGS,
+	VMXNET3_CMD_SET_RSS_FIELDS,
 
 	VMXNET3_CMD_FIRST_GET = 0xF00D0000,
 	VMXNET3_CMD_GET_QUEUE_STATUS = VMXNET3_CMD_FIRST_GET,
@@ -96,6 +97,7 @@ enum {
 	VMXNET3_CMD_GET_RESERVED1,
 	VMXNET3_CMD_GET_TXDATA_DESC_SIZE,
 	VMXNET3_CMD_GET_COALESCE,
+	VMXNET3_CMD_GET_RSS_FIELDS,
 };
 
 /*
@@ -685,12 +687,22 @@ struct Vmxnet3_MemRegs {
 	struct Vmxnet3_MemoryRegion		memRegs[1];
 };
 
+enum Vmxnet3_RSSField {
+	VMXNET3_RSS_FIELDS_TCPIP4 = 0x0001,
+	VMXNET3_RSS_FIELDS_TCPIP6 = 0x0002,
+	VMXNET3_RSS_FIELDS_UDPIP4 = 0x0004,
+	VMXNET3_RSS_FIELDS_UDPIP6 = 0x0008,
+	VMXNET3_RSS_FIELDS_ESPIP4 = 0x0010,
+	VMXNET3_RSS_FIELDS_ESPIP6 = 0x0020,
+};
+
 /* If the command data <= 16 bytes, use the shared memory directly.
  * otherwise, use variable length configuration descriptor.
  */
 union Vmxnet3_CmdInfo {
 	struct Vmxnet3_VariableLenConfDesc	varConf;
 	struct Vmxnet3_SetPolling		setPolling;
+	enum   Vmxnet3_RSSField                 setRssFields;
 	__le64					data[2];
 };
 
drivers/net/vmxnet3/vmxnet3_drv.c +39 -0
index ec2878f8c1f6..4ea7a40ada88 100644
@@ -2554,6 +2554,39 @@ vmxnet3_init_coalesce(struct vmxnet3_adapter *adapter)
 	spin_unlock_irqrestore(&adapter->cmd_lock, flags);
 }
 
+static void
+vmxnet3_init_rssfields(struct vmxnet3_adapter *adapter)
+{
+	struct Vmxnet3_DriverShared *shared = adapter->shared;
+	union Vmxnet3_CmdInfo *cmdInfo = &shared->cu.cmdInfo;
+	unsigned long flags;
+
+		if (!VMXNET3_VERSION_GE_4(adapter))
+			return;
+
+	spin_lock_irqsave(&adapter->cmd_lock, flags);
+
+	if (adapter->default_rss_fields) {
+		VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
+				       VMXNET3_CMD_GET_RSS_FIELDS);
+		adapter->rss_fields =
+			VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);
+	} else {
+		cmdInfo->setRssFields = adapter->rss_fields;
+		VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
+				       VMXNET3_CMD_SET_RSS_FIELDS);
+		/* Not all requested RSS may get applied, so get and
+		 * cache what was actually applied.
+		 */
+		VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
+				       VMXNET3_CMD_GET_RSS_FIELDS);
+		adapter->rss_fields =
+			VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);
+	}
+
+	spin_unlock_irqrestore(&adapter->cmd_lock, flags);
+}
+
 int
 vmxnet3_activate_dev(struct vmxnet3_adapter *adapter)
 {
@@ -2603,6 +2636,7 @@ vmxnet3_activate_dev(struct vmxnet3_adapter *adapter)
 	}
 
 	vmxnet3_init_coalesce(adapter);
+	vmxnet3_init_rssfields(adapter);
 
 	for (i = 0; i < adapter->num_rx_queues; i++) {
 		VMXNET3_WRITE_BAR0_REG(adapter,
@@ -3430,6 +3464,11 @@ vmxnet3_probe_device(struct pci_dev *pdev,
 		adapter->default_coal_mode = true;
 	}
 
+	if (VMXNET3_VERSION_GE_4(adapter)) {
+		adapter->default_rss_fields = true;
+		adapter->rss_fields = VMXNET3_RSS_FIELDS_DEFAULT;
+	}
+
 	SET_NETDEV_DEV(netdev, &pdev->dev);
 	vmxnet3_declare_features(adapter, dma64);
 
drivers/net/vmxnet3/vmxnet3_ethtool.c +217 -2

241 lines not on this page · raw message

drivers/net/vmxnet3/vmxnet3_int.h +4 -0
index e803ffad75d6..d52ccc3eeba2 100644
@@ -377,6 +377,8 @@ struct vmxnet3_adapter {
 	u16 rxdata_desc_size;
 
 	bool rxdataring_enabled;
+	bool default_rss_fields;
+	enum Vmxnet3_RSSField rss_fields;
 
 	struct work_struct work;
 
@@ -438,6 +440,8 @@ struct vmxnet3_adapter {
 
 #define VMXNET3_COAL_RBC_RATE(usecs) (1000000 / usecs)
 #define VMXNET3_COAL_RBC_USECS(rbc_rate) (1000000 / rbc_rate)
+#define VMXNET3_RSS_FIELDS_DEFAULT (VMXNET3_RSS_FIELDS_TCPIP4 | \
+				    VMXNET3_RSS_FIELDS_TCPIP6)
 
 int
 vmxnet3_quiesce_dev(struct vmxnet3_adapter *adapter);
Signature · 1 line
2.11.0
#1 lore raw

PATCH v3 1/4 vmxnet3: prepare for version 4 changes

In reply to Ronak Doshi

vmxnet3 is currently at version 3 and this patch initiates the preparation to accommodate changes for version 4. Introduced utility macros for vmxnet3 version 4 comparison and update Copyright information.

---
 drivers/net/vmxnet3/Makefile          | 2 +-
 drivers/net/vmxnet3/upt1_defs.h       | 2 +-
 drivers/net/vmxnet3/vmxnet3_defs.h    | 2 +-
 drivers/net/vmxnet3/vmxnet3_drv.c     | 2 +-
 drivers/net/vmxnet3/vmxnet3_ethtool.c | 2 +-
 drivers/net/vmxnet3/vmxnet3_int.h     | 5 ++++-
 6 files changed, 9 insertions(+), 6 deletions(-)
drivers/net/vmxnet3/Makefile +1 -1
index 8cdbb63d1bb0..c5a167a1c85c 100644
@@ -2,7 +2,7 @@
 #
 # Linux driver for VMware's vmxnet3 ethernet NIC.
 #
-# Copyright (C) 2007-2016, VMware, Inc. All Rights Reserved.
+# Copyright (C) 2007-2020, VMware, Inc. All Rights Reserved.
 #
 # This program is free software; you can redistribute it and/or modify it
 # under the terms of the GNU General Public License as published by the
drivers/net/vmxnet3/upt1_defs.h +1 -1
index db9f1fde3aac..65a203c842b2 100644
@@ -1,7 +1,7 @@
 /*
  * Linux driver for VMware's vmxnet3 ethernet NIC.
  *
- * Copyright (C) 2008-2016, VMware, Inc. All Rights Reserved.
+ * Copyright (C) 2008-2020, VMware, Inc. All Rights Reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the
drivers/net/vmxnet3/vmxnet3_defs.h +1 -1
index c3a31646189f..c77274228a3e 100644
@@ -1,7 +1,7 @@
 /*
  * Linux driver for VMware's vmxnet3 ethernet NIC.
  *
- * Copyright (C) 2008-2016, VMware, Inc. All Rights Reserved.
+ * Copyright (C) 2008-2020, VMware, Inc. All Rights Reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the
drivers/net/vmxnet3/vmxnet3_drv.c +1 -1
index 722cb054a5cd..ec2878f8c1f6 100644
@@ -1,7 +1,7 @@
 /*
  * Linux driver for VMware's vmxnet3 ethernet NIC.
  *
- * Copyright (C) 2008-2016, VMware, Inc. All Rights Reserved.
+ * Copyright (C) 2008-2020, VMware, Inc. All Rights Reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the
drivers/net/vmxnet3/vmxnet3_ethtool.c +1 -1
index 6528940ce5f3..1163eca7aba5 100644
@@ -1,7 +1,7 @@
 /*
  * Linux driver for VMware's vmxnet3 ethernet NIC.
  *
- * Copyright (C) 2008-2016, VMware, Inc. All Rights Reserved.
+ * Copyright (C) 2008-2020, VMware, Inc. All Rights Reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the
drivers/net/vmxnet3/vmxnet3_int.h +4 -1
index 1cc1cd4aaa59..e803ffad75d6 100644
@@ -1,7 +1,7 @@
 /*
  * Linux driver for VMware's vmxnet3 ethernet NIC.
  *
- * Copyright (C) 2008-2016, VMware, Inc. All Rights Reserved.
+ * Copyright (C) 2008-2020, VMware, Inc. All Rights Reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the
@@ -81,6 +81,7 @@
 	#define VMXNET3_RSS
 #endif
 
+#define VMXNET3_REV_4		3	/* Vmxnet3 Rev. 4 */
 #define VMXNET3_REV_3		2	/* Vmxnet3 Rev. 3 */
 #define VMXNET3_REV_2		1	/* Vmxnet3 Rev. 2 */
 #define VMXNET3_REV_1		0	/* Vmxnet3 Rev. 1 */
@@ -412,6 +413,8 @@ struct vmxnet3_adapter {
 	(adapter->version >= VMXNET3_REV_2 + 1)
 #define VMXNET3_VERSION_GE_3(adapter) \
 	(adapter->version >= VMXNET3_REV_3 + 1)
+#define VMXNET3_VERSION_GE_4(adapter) \
+	(adapter->version >= VMXNET3_REV_4 + 1)
 
 /* must be a multiple of VMXNET3_RING_SIZE_ALIGN */
 #define VMXNET3_DEF_TX_RING_SIZE    512
Signature · 1 line
2.11.0
#3 lore raw

PATCH v3 3/4 vmxnet3: add geneve and vxlan tunnel offload support

In reply to Ronak Doshi

Vmxnet3 version 3 device supports checksum/TSO offload. Thus, vNIC to pNIC traffic can leverage hardware checksum/TSO offloads. However, vmxnet3 does not support checksum/TSO offload for Geneve/VXLAN encapsulated packets. Thus, for a vNIC configured with an overlay, the guest stack must first segment the inner packet, compute the inner checksum for each segment and encapsulate each segment before transmitting the packet via the vNIC. This results in significant performance penalty.

This patch will enhance vmxnet3 to support Geneve/VXLAN TSO as well as checksum offload.

---
 drivers/net/vmxnet3/upt1_defs.h       |   3 +
 drivers/net/vmxnet3/vmxnet3_defs.h    |  17 +++--
 drivers/net/vmxnet3/vmxnet3_drv.c     | 120 +++++++++++++++++++++++++++-------
 drivers/net/vmxnet3/vmxnet3_ethtool.c |  42 +++++++++++-
 drivers/net/vmxnet3/vmxnet3_int.h     |  12 +++-
 5 files changed, 161 insertions(+), 33 deletions(-)
drivers/net/vmxnet3/upt1_defs.h +3 -0
index 65a203c842b2..8c014c98471c 100644
@@ -92,5 +92,8 @@ enum {
 	UPT1_F_RSS		= cpu_to_le64(0x0002),
 	UPT1_F_RXVLAN		= cpu_to_le64(0x0004),   /* VLAN tag stripping */
 	UPT1_F_LRO		= cpu_to_le64(0x0008),
+	UPT1_F_RXINNEROFLD      = cpu_to_le64(0x00010),  /* Geneve/Vxlan rx csum
+							  * offloading
+							  */
 };
 #endif
drivers/net/vmxnet3/vmxnet3_defs.h +10 -7
index aac97fac1186..a8d5ebd47c71 100644
@@ -103,14 +103,14 @@ enum {
 /*
  *	Little Endian layout of bitfields -
  *	Byte 0 :	7.....len.....0
- *	Byte 1 :	rsvd gen 13.len.8
+ *	Byte 1 :	oco gen 13.len.8
  *	Byte 2 : 	5.msscof.0 ext1  dtype
  *	Byte 3 : 	13...msscof...6
  *
  *	Big Endian layout of bitfields -
  *	Byte 0:		13...msscof...6
  *	Byte 1 : 	5.msscof.0 ext1  dtype
- *	Byte 2 :	rsvd gen 13.len.8
+ *	Byte 2 :	oco gen 13.len.8
  *	Byte 3 :	7.....len.....0
  *
  *	Thus, le32_to_cpu on the dword will allow the big endian driver to read
@@ -125,13 +125,13 @@ struct Vmxnet3_TxDesc {
 	u32 msscof:14;  /* MSS, checksum offset, flags */
 	u32 ext1:1;
 	u32 dtype:1;    /* descriptor type */
-	u32 rsvd:1;
+	u32 oco:1;
 	u32 gen:1;      /* generation bit */
 	u32 len:14;
 #else
 	u32 len:14;
 	u32 gen:1;      /* generation bit */
-	u32 rsvd:1;
+	u32 oco:1;
 	u32 dtype:1;    /* descriptor type */
 	u32 ext1:1;
 	u32 msscof:14;  /* MSS, checksum offset, flags */
@@ -157,9 +157,10 @@ struct Vmxnet3_TxDesc {
 };
 
 /* TxDesc.OM values */
-#define VMXNET3_OM_NONE		0
-#define VMXNET3_OM_CSUM		2
-#define VMXNET3_OM_TSO		3
+#define VMXNET3_OM_NONE         0
+#define VMXNET3_OM_ENCAP        1
+#define VMXNET3_OM_CSUM         2
+#define VMXNET3_OM_TSO          3
 
 /* fields in TxDesc we access w/o using bit fields */
 #define VMXNET3_TXD_EOP_SHIFT	12
@@ -226,6 +227,8 @@ struct Vmxnet3_RxDesc {
 #define VMXNET3_RXD_BTYPE_SHIFT  14
 #define VMXNET3_RXD_GEN_SHIFT    31
 
+#define VMXNET3_RCD_HDR_INNER_SHIFT  13
+
 struct Vmxnet3_RxCompDesc {
 #ifdef __BIG_ENDIAN_BITFIELD
 	u32		ext2:1;
drivers/net/vmxnet3/vmxnet3_drv.c +98 -22

202 lines not on this page · raw message

drivers/net/vmxnet3/vmxnet3_ethtool.c +41 -1

61 lines not on this page · raw message

drivers/net/vmxnet3/vmxnet3_int.h +9 -3
index d52ccc3eeba2..86db809c7592 100644
@@ -219,10 +219,16 @@ struct vmxnet3_tx_ctx {
 	bool   ipv4;
 	bool   ipv6;
 	u16 mss;
-	u32 eth_ip_hdr_size; /* only valid for pkts requesting tso or csum
-				 * offloading
+	u32    l4_offset;	/* only valid for pkts requesting tso or csum
+				 * offloading. For encap offload, it refers to
+				 * inner L4 offset i.e. it includes outer header
+				 * encap header and inner eth and ip header size
+				 */
+
+	u32	l4_hdr_size;	/* only valid if mss != 0
+				 * Refers to inner L4 hdr size for encap
+				 * offload
 				 */
-	u32 l4_hdr_size;     /* only valid if mss != 0 */
 	u32 copy_size;       /* # of bytes copied into the data ring */
 	union Vmxnet3_GenericDesc *sop_txd;
 	union Vmxnet3_GenericDesc *eop_txd;
Signature · 1 line
2.11.0
#5 lore raw

PATCH v3 4/4 vmxnet3: update to version 4

In reply to Ronak Doshi

With all vmxnet3 version 4 changes incorporated in the vmxnet3 driver, the driver can configure emulation to run at vmxnet3 version 4, provided the emulation advertises support for version 4.

---
 drivers/net/vmxnet3/vmxnet3_drv.c | 7 ++++++-
 drivers/net/vmxnet3/vmxnet3_int.h | 4 ++--
 2 files changed, 8 insertions(+), 3 deletions(-)
drivers/net/vmxnet3/vmxnet3_drv.c +6 -1
index b764f24b646b..ceafa3619858 100644
@@ -3492,7 +3492,12 @@ vmxnet3_probe_device(struct pci_dev *pdev,
 		goto err_alloc_pci;
 
 	ver = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_VRRS);
-	if (ver & (1 << VMXNET3_REV_3)) {
+	if (ver & (1 << VMXNET3_REV_4)) {
+		VMXNET3_WRITE_BAR1_REG(adapter,
+				       VMXNET3_REG_VRRS,
+				       1 << VMXNET3_REV_4);
+		adapter->version = VMXNET3_REV_4 + 1;
+	} else if (ver & (1 << VMXNET3_REV_3)) {
 		VMXNET3_WRITE_BAR1_REG(adapter,
 				       VMXNET3_REG_VRRS,
 				       1 << VMXNET3_REV_3);
drivers/net/vmxnet3/vmxnet3_int.h +2 -2
index 86db809c7592..5d2b062215a2 100644
@@ -69,12 +69,12 @@
 /*
  * Version numbers
  */
-#define VMXNET3_DRIVER_VERSION_STRING   "1.4.17.0-k"
+#define VMXNET3_DRIVER_VERSION_STRING   "1.5.0.0-k"
 
 /* Each byte of this 32-bit integer encodes a version number in
  * VMXNET3_DRIVER_VERSION_STRING.
  */
-#define VMXNET3_DRIVER_VERSION_NUM      0x01041100
+#define VMXNET3_DRIVER_VERSION_NUM      0x01050000
 
 #if defined(CONFIG_PCI_MSI)
 	/* RSS only makes sense if MSI-X is supported. */
Signature · 1 line
2.11.0
maintainer
#4 lore raw

Re: [PATCH v3 net-next 3/4] vmxnet3: add geneve and vxlan tunnel offload support

In reply to Ronak Doshi

From: Ronak Doshi <[email protected]>
Date: Thu, 28 May 2020 14:32:02 -0700

Quoted 6 lines
drivers/net/vmxnet3/vmxnet3_drv.c +6 -0
+			BUG_ON(!(gdesc->rcd.tcp || gdesc->rcd.udp) &&
+			       !(le32_to_cpu(gdesc->dword[0]) &
+				 (1UL << VMXNET3_RCD_HDR_INNER_SHIFT)));
+			BUG_ON(gdesc->rcd.frg &&
+			       !(le32_to_cpu(gdesc->dword[0]) &
+				 (1UL << VMXNET3_RCD_HDR_INNER_SHIFT)));

...

drivers/net/vmxnet3/vmxnet3_drv.c vmxnet3_rx_csum(struct vmxnet3_adapter *adapter, · line 1228

Quoted 6 lines
drivers/net/vmxnet3/vmxnet3_drv.c +6 -0
+			BUG_ON(!(gdesc->rcd.tcp || gdesc->rcd.udp) &&
+			       !(le32_to_cpu(gdesc->dword[0]) &
+				 (1UL << VMXNET3_RCD_HDR_INNER_SHIFT)));
+			BUG_ON(gdesc->rcd.frg &&
+			       !(le32_to_cpu(gdesc->dword[0]) &
+				 (1UL << VMXNET3_RCD_HDR_INNER_SHIFT)));

Just to be clear I'm not applying stuff like this.