i40e: Introduce devlink interface.

VF representor/control netdevs are created for each VF if the switch mode is set to 'switchdev'. These netdevs can be used to control and configure VFs from PFs namespace. They enable exposing VF statistics, configuring ntuple filters, additional mac/vlans to VFs etc.

Sample script to create VF representors

     # devlink dev set pci/0000:05:00.0 smode switchdev
     # echo 2 > /sys/class/net/enp5s0f0/device/sriov_numvfs
     # ip l show
     297: enp5s0f0: <BROADCAST,MULTICAST> mtu 1500 qdisc noop portid 6805ca2e7268 state DOWN mode DEFAULT group default qlen 1000
     link/ether 68:05:ca:2e:72:68 brd ff:ff:ff:ff:ff:ff
     vf 0 MAC 00:00:00:00:00:00, spoof checking on, link-state auto, trust off
     vf 1 MAC 00:00:00:00:00:00, spoof checking on, link-state auto, trust off
     299: enp5s0f0-vf0: <BROADCAST,MULTICAST> mtu 1500 qdisc noop state DOWN mode DEFAULT group default qlen 1000
     link/ether 00:00:00:00:00:00 brd ff:ff:ff:ff:ff:ff
     300: enp5s0f0-vf1: <BROADCAST,MULTICAST> mtu 1500 qdisc noop state DOWN mode DEFAULT group default qlen 1000
     link/ether 00:00:00:00:00:00 brd ff:ff:ff:ff:ff:ff

Signed-off-by Sridhar Samudrala <[email protected]>

---
  drivers/net/ethernet/intel/i40e/i40e_main.c        | 13 +++-
  drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c | 90 ++++++++++++++++++++++
  drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.h | 14 ++++
  3 files changed, 116 insertions(+), 1 deletion(-)

drivers/net/ethernet/intel/i40e/i40e_main.c +12 -1

index bb44f09..208da9f 100644
@@ -10700,14 +10700,25 @@ static int i40e_devlink_eswitch_mode_get(struct devlink *devlink, u16 *mode)
  static int i40e_devlink_eswitch_mode_set(struct devlink *devlink, u16 mode)
  {
  	struct i40e_pf *pf = devlink_priv(devlink);
-	int err = 0;
+	struct i40e_vf *vf;
+	int i, err = 0;
 
  	if (mode == pf->eswitch_mode)
  		goto done;
 
  	switch (mode) {
  	case DEVLINK_ESWITCH_MODE_LEGACY:
+		for (i = 0; i < pf->num_alloc_vfs; i++) {
+			vf = &(pf->vf[i]);
+			i40e_free_vf_netdev(vf);
+		}
+		pf->eswitch_mode = mode;
+		break;
  	case DEVLINK_ESWITCH_MODE_SWITCHDEV:
+		for (i = 0; i < pf->num_alloc_vfs; i++) {
+			vf = &(pf->vf[i]);
+			i40e_alloc_vf_netdev(vf, i);
+		}
  		pf->eswitch_mode = mode;
  		break;
  	default:

drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c +90 -0

index 4bd66d7..d41c37c 100644
@@ -1000,6 +1000,90 @@ complete_reset:
  	clear_bit(__I40E_VF_DISABLE, &pf->state);
  }
 
+static int i40e_vf_netdev_open(struct net_device *dev)
+{
+	return 0;
+}
+
+static int i40e_vf_netdev_stop(struct net_device *dev)
+{
+	return 0;
+}
+
+static const struct net_device_ops i40e_vf_netdev_ops = {
+	.ndo_open       = i40e_vf_netdev_open,
+	.ndo_stop       = i40e_vf_netdev_stop,
+};
+
+/**
+ * i40e_alloc_vf_netdev
+ * @vf: pointer to the VF structure
+ * @vf_num: VF number
+ *
+ * Create VF representor/control netdev
+ **/
+int i40e_alloc_vf_netdev(struct i40e_vf *vf, u16 vf_num)
+{
+	struct net_device *netdev;
+	char netdev_name[IFNAMSIZ];
+	struct i40e_vf_netdev_priv *priv;
+	struct i40e_pf *pf = vf->pf;
+	struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
+	int err;
+
+	snprintf(netdev_name, IFNAMSIZ, "%s-vf%d", vsi->netdev->name, vf_num);
+	netdev = alloc_netdev(sizeof(struct i40e_vf_netdev_priv), netdev_name,
+			      NET_NAME_UNKNOWN, ether_setup);
+	if (!netdev) {
+		dev_err(&pf->pdev->dev, "alloc_netdev failed for vf:%d\n",
+			vf_num);
+		return -ENOMEM;
+	}
+
+	pf->vf[vf_num].ctrl_netdev = netdev;
+
+	priv = netdev_priv(netdev);
+	priv->vf = &(pf->vf[vf_num]);
+
+	netdev->netdev_ops = &i40e_vf_netdev_ops;
+
+	netif_carrier_off(netdev);
+	netif_tx_disable(netdev);
+
+	err = register_netdev(netdev);
+	if (err) {
+		dev_err(&pf->pdev->dev, "register_netdev failed for vf: %s\n",
+			vf->ctrl_netdev->name);
+		free_netdev(netdev);
+		return err;
+	}
+
+	dev_info(&pf->pdev->dev, "VF representor(%s) created for VF %d\n",
+		 vf->ctrl_netdev->name, vf_num);
+
+	return 0;
+}
+
+/**
+ * i40e_free_vf_netdev
+ * @vf: pointer to the VF structure
+ *
+ * Free VF representor/control netdev
+ **/
+void i40e_free_vf_netdev(struct i40e_vf *vf)
+{
+	struct i40e_pf *pf = vf->pf;
+
+	if (!vf->ctrl_netdev)
+		return;
+
+	dev_info(&pf->pdev->dev, "Freeing VF representor(%s)\n",
+		 vf->ctrl_netdev->name);
+
+	unregister_netdev(vf->ctrl_netdev);
+	free_netdev(vf->ctrl_netdev);
+}
+
  /**
   * i40e_free_vfs
   * @pf: pointer to the PF structure

@@ -1042,6 +1126,9 @@ void i40e_free_vfs(struct i40e_pf *pf)
  			i40e_free_vf_res(&pf->vf[i]);
  		/* disable qp mappings */
  		i40e_disable_vf_mappings(&pf->vf[i]);
+
+		if (pf->eswitch_mode == DEVLINK_ESWITCH_MODE_SWITCHDEV)
+			i40e_free_vf_netdev(&pf->vf[i]);
  	}
 
  	kfree(pf->vf);

@@ -1110,6 +1197,9 @@ int i40e_alloc_vfs(struct i40e_pf *pf, u16 num_alloc_vfs)
  		/* VF resources get allocated during reset */
  		i40e_reset_vf(&vfs[i], false);
 
+		if (pf->eswitch_mode == DEVLINK_ESWITCH_MODE_SWITCHDEV)
+			i40e_alloc_vf_netdev(&vfs[i], i);
+
  	}
  	pf->num_alloc_vfs = num_alloc_vfs;
 

drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.h +14 -0

index 8751741..8446547 100644
@@ -72,10 +72,21 @@ enum i40e_vf_capabilities {
  	I40E_VIRTCHNL_VF_CAP_IWARP,
  };
 
+/* VF Ctrl netdev private structure */
+struct i40e_vf_netdev_priv {
+	struct i40e_vf *vf;
+};
+
  /* VF information structure */
  struct i40e_vf {
  	struct i40e_pf *pf;
 
+	/* VF representor netdev that allows control and configuration of
+	 * VFs from the host. Enables returning VF stats, configuring ntuple
+	 * filters on VFs, adding multiple mac/vlans to VFs etc.
+	 */
+	struct net_device *ctrl_netdev;
+
  	/* VF id in the PF space */
  	s16 vf_id;
  	/* all VF vsis connect to the same parent */

@@ -142,4 +153,7 @@ int i40e_ndo_set_vf_spoofchk(struct net_device *netdev, int vf_id, bool enable);
  void i40e_vc_notify_link_state(struct i40e_pf *pf);
  void i40e_vc_notify_reset(struct i40e_pf *pf);
 
+int i40e_alloc_vf_netdev(struct i40e_vf *vf, u16 vf_num);
+void i40e_free_vf_netdev(struct i40e_vf *vf);
+
  #endif /* _I40E_VIRTCHNL_PF_H_ */

Sample script that shows ethtool stats on VF representor netdev
PF: enp5s0f0, VF0: enp5s2 VF_REP0: enp5s0f0-vf0

    # devlink dev set pci/0000:05:00.0 smode switchdev
    # echo 2 > /sys/class/net/enp5s0f0/device/sriov_numvfs
    # ip link set enp5s2 up
    # ethtool -S enp5s0f0-vf0
    NIC statistics:
      rx_bytes: 0
      rx_unicast: 0
      rx_multicast: 0
      rx_broadcast: 0
      rx_discards: 0
      rx_unknown_protocol: 0
      tx_bytes: 140
      tx_unicast: 0
      tx_multicast: 2
      tx_broadcast: 0
      tx_discards: 0
      tx_errors: 0

Signed-off-by Sridhar Samudrala <[email protected]>

---
  drivers/net/ethernet/intel/i40e/i40e.h             |  1 +
  drivers/net/ethernet/intel/i40e/i40e_ethtool.c     | 67 ++++++++++++++++++++++
  drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c |  1 +
  3 files changed, 69 insertions(+)

drivers/net/ethernet/intel/i40e/i40e.h +1 -0

index 724bd82..2d35c0d 100644
@@ -875,4 +875,5 @@ i40e_status i40e_get_npar_bw_setting(struct i40e_pf *pf);
  i40e_status i40e_set_npar_bw_setting(struct i40e_pf *pf);
  i40e_status i40e_commit_npar_bw_setting(struct i40e_pf *pf);
  void i40e_print_link_message(struct i40e_vsi *vsi, bool isup);
+void i40e_set_vf_netdev_ethtool_ops(struct net_device *netdev);
  #endif /* _I40E_H_ */

drivers/net/ethernet/intel/i40e/i40e_ethtool.c +67 -0

index c912e04..0c3bffa 100644
@@ -3040,3 +3040,70 @@ void i40e_set_ethtool_ops(struct net_device *netdev)
  {
  	netdev->ethtool_ops = &i40e_ethtool_ops;
  }
+
+static const char i40e_vf_netdev_ethtool_sset[][ETH_GSTRING_LEN] = {
+	"rx_bytes",
+	"rx_unicast",
+	"rx_multicast",
+	"rx_broadcast",
+	"rx_discards",
+	"rx_unknown_protocol",
+	"tx_bytes",
+	"tx_unicast",
+	"tx_multicast",
+	"tx_broadcast",
+	"tx_discards",
+	"tx_errors",
+};
+
+#define I40E_VF_NETDEV_ETHTOOL_STAT_COUNT \
+			ARRAY_SIZE(i40e_vf_netdev_ethtool_sset)
+
+static void i40e_vf_netdev_ethtool_get_strings(struct net_device *dev,
+					       u32 stringset,
+					       u8 *ethtool_strings)
+{
+	switch (stringset) {
+	case ETH_SS_STATS:
+		memcpy(ethtool_strings, &i40e_vf_netdev_ethtool_sset,
+		       sizeof(i40e_vf_netdev_ethtool_sset));
+		break;
+	}
+}
+
+static int i40e_vf_netdev_ethtool_get_sset_count(struct net_device *dev,
+						 int stringset)
+{
+	switch (stringset) {
+	case ETH_SS_STATS:
+		return I40E_VF_NETDEV_ETHTOOL_STAT_COUNT;
+	default:
+		return -EOPNOTSUPP;
+	}
+}
+
+static void i40e_vf_netdev_ethtool_get_stats(struct net_device *dev,
+				struct ethtool_stats *target_ethtool_stats,
+				u64 *target_stat_values)
+{
+	struct i40e_vf_netdev_priv *priv = netdev_priv(dev);
+	struct i40e_vf *vf = priv->vf;
+	struct i40e_pf *pf = vf->pf;
+	struct i40e_vsi *vsi;
+
+	vsi = pf->vsi[vf->lan_vsi_idx];
+	i40e_update_stats(vsi);
+	memcpy(target_stat_values, &vsi->eth_stats,
+	       I40E_VF_NETDEV_ETHTOOL_STAT_COUNT * 8);
+}
+
+static const struct ethtool_ops i40e_vf_netdev_ethtool_ops = {
+	.get_strings		= i40e_vf_netdev_ethtool_get_strings,
+	.get_ethtool_stats      = i40e_vf_netdev_ethtool_get_stats,
+	.get_sset_count         = i40e_vf_netdev_ethtool_get_sset_count,
+};
+
+void i40e_set_vf_netdev_ethtool_ops(struct net_device *netdev)
+{
+	netdev->ethtool_ops = &i40e_vf_netdev_ethtool_ops;
+}

drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c +1 -0

index d41c37c..313a53c 100644
@@ -1046,6 +1046,7 @@ int i40e_alloc_vf_netdev(struct i40e_vf *vf, u16 vf_num)
  	priv->vf = &(pf->vf[vf_num]);
 
  	netdev->netdev_ops = &i40e_vf_netdev_ops;
+	i40e_set_vf_netdev_ethtool_ops(netdev);
 
  	netif_carrier_off(netdev);
  	netif_tx_disable(netdev);

Add initial devlink support to set/get the mode of SRIOV switch. This patch allows the mode to be set to either 'legacy' or 'switchdev', but doesn't implement any functionality to create vf representors in switchdev mode.

With smode support in iproute2 'devlink' utility, switch mode can be set and get via following commands.

     # devlink dev smode pci/0000:05:00.0
     mode: legacy
     # devlink dev set pci/0000:05:00.0 smode switchdev
     # devlink dev smode pci/0000:05:00.0
     mode: switchdev

Signed-off-by Sridhar Samudrala <[email protected]>

---
  drivers/net/ethernet/intel/Kconfig          |  1 +
  drivers/net/ethernet/intel/i40e/i40e.h      |  3 ++
  drivers/net/ethernet/intel/i40e/i40e_main.c | 84 ++++++++++++++++++++++++++---
  3 files changed, 80 insertions(+), 8 deletions(-)

drivers/net/ethernet/intel/Kconfig +1 -0

index c0e1743..2ede229 100644
@@ -215,6 +215,7 @@ config I40E
  	tristate "Intel(R) Ethernet Controller XL710 Family support"
  	select PTP_1588_CLOCK
  	depends on PCI
+	depends on MAY_USE_DEVLINK
  	---help---
  	  This driver supports Intel(R) Ethernet Controller XL710 Family of
  	  devices.  For more information on how to identify your adapter, go

drivers/net/ethernet/intel/i40e/i40e.h +3 -0

index 2a88291..724bd82 100644
@@ -53,6 +53,8 @@
  #include <linux/clocksource.h>
  #include <linux/net_tstamp.h>
  #include <linux/ptp_clock_kernel.h>
+#include <net/devlink.h>
+
  #include "i40e_type.h"
  #include "i40e_prototype.h"
  #ifdef I40E_FCOE

@@ -446,6 +448,7 @@ struct i40e_pf {
  	u32 ioremap_len;
  	u32 fd_inv;
  	u16 phy_led_val;
+	enum devlink_eswitch_mode eswitch_mode;
  };
 
  enum i40e_filter_state {

drivers/net/ethernet/intel/i40e/i40e_main.c +76 -8

index 339d99b..bb44f09 100644
@@ -27,6 +27,7 @@
  #include <linux/etherdevice.h>
  #include <linux/of_net.h>
  #include <linux/pci.h>
+#include <net/devlink.h>
 
  /* Local includes */
  #include "i40e.h"

@@ -10671,6 +10672,58 @@ static void i40e_get_platform_mac_addr(struct pci_dev *pdev, struct i40e_pf *pf)
  }
 
  /**
+ * i40e_devlink_eswitch_mode_get
+ *
+ * @devlink: pointer to devlink struct
+ * @mode: sr-iov switch mode pointer
+ *
+ * Returns the switch mode of the associated PF in the @mode pointer.
+ */
+static int i40e_devlink_eswitch_mode_get(struct devlink *devlink, u16 *mode)
+{
+	struct i40e_pf *pf = devlink_priv(devlink);
+
+	*mode = pf->eswitch_mode;
+
+	return 0;
+}
+
+/**
+ * i40e_devlink_eswitch_mode_set
+ *
+ * @devlink: pointer to devlink struct
+ * @mode: sr-iov switch mode
+ *
+ * Set the switch mode of the associated PF.
+ * Returns 0 on success and -EOPNOTSUPP on error.
+ */
+static int i40e_devlink_eswitch_mode_set(struct devlink *devlink, u16 mode)
+{
+	struct i40e_pf *pf = devlink_priv(devlink);
+	int err = 0;
+
+	if (mode == pf->eswitch_mode)
+		goto done;
+
+	switch (mode) {
+	case DEVLINK_ESWITCH_MODE_LEGACY:
+	case DEVLINK_ESWITCH_MODE_SWITCHDEV:
+		pf->eswitch_mode = mode;
+		break;
+	default:
+		err = -EOPNOTSUPP;
+	}
+
+done:
+	return err;
+}
+
+static const struct devlink_ops i40e_devlink_ops = {
+	.eswitch_mode_get = i40e_devlink_eswitch_mode_get,
+	.eswitch_mode_set = i40e_devlink_eswitch_mode_set,
+};
+
+/**
   * i40e_probe - Device initialization routine
   * @pdev: PCI device information struct
   * @ent: entry in i40e_pci_tbl

@@ -10687,6 +10740,7 @@ static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
  	struct i40e_pf *pf;
  	struct i40e_hw *hw;
  	static u16 pfs_found;
+	struct devlink *devlink;
  	u16 wol_nvm_bits;
  	u16 link_status;
  	int err;

@@ -10721,16 +10775,21 @@ static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
  	pci_enable_pcie_error_reporting(pdev);
  	pci_set_master(pdev);
 
+	devlink = devlink_alloc(&i40e_devlink_ops, sizeof(*pf));
+	if (!devlink) {
+		dev_err(&pdev->dev,
+			"devlink_alloc failed\n");
+		err = -ENOMEM;
+		goto err_devlink_alloc;
+	}
+
  	/* Now that we have a PCI connection, we need to do the
  	 * low level device setup.  This is primarily setting up
  	 * the Admin Queue structures and then querying for the
  	 * device's current profile information.
  	 */
-	pf = kzalloc(sizeof(*pf), GFP_KERNEL);
-	if (!pf) {
-		err = -ENOMEM;
-		goto err_pf_alloc;
-	}
+	pf = devlink_priv(devlink);
+
  	pf->next_vsi = 0;
  	pf->pdev = pdev;
  	set_bit(__I40E_DOWN, &pf->state);

@@ -11047,6 +11106,11 @@ static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
  		}
  	}
 
+	pf->eswitch_mode = DEVLINK_ESWITCH_MODE_LEGACY;
+	err = devlink_register(devlink, &pdev->dev);
+	if (err)
+		goto err_devlink_register;
+
  #ifdef CONFIG_PCI_IOV
  	/* prep for VF support */
  	if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) &&

@@ -11188,6 +11252,8 @@ static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
  	return 0;
 
  	/* Unwind what we've done if something failed in the setup */
+err_devlink_register:
+	i40e_stop_misc_vector(pf);
  err_vsis:
  	set_bit(__I40E_DOWN, &pf->state);
  	i40e_clear_interrupt_scheme(pf);

@@ -11205,8 +11271,8 @@ err_adminq_setup:
  err_pf_reset:
  	iounmap(hw->hw_addr);
  err_ioremap:
-	kfree(pf);
-err_pf_alloc:
+	devlink_free(devlink);
+err_devlink_alloc:
  	pci_disable_pcie_error_reporting(pdev);
  	pci_release_selected_regions(pdev,
  				     pci_select_bars(pdev, IORESOURCE_MEM));

@@ -11229,9 +11295,11 @@ static void i40e_remove(struct pci_dev *pdev)
  {
  	struct i40e_pf *pf = pci_get_drvdata(pdev);
  	struct i40e_hw *hw = &pf->hw;
+	struct devlink *devlink = priv_to_devlink(pf);
  	i40e_status ret_code;
  	int i;
 
+	devlink_unregister(devlink);
  	i40e_dbg_pf_exit(pf);
 
  	i40e_ptp_stop(pf);

@@ -11319,7 +11387,7 @@ static void i40e_remove(struct pci_dev *pdev)
  	kfree(pf->vsi);
 
  	iounmap(hw->hw_addr);
-	kfree(pf);
+	devlink_free(devlink);
  	pci_release_selected_regions(pdev,
  				     pci_select_bars(pdev, IORESOURCE_MEM));
 

Add initial devlink support to set/get the mode of SRIOV switch. This patch allows the mode to be set to either 'legacy' or 'switchdev', but doesn't implement any functionality to create vf representors in switchdev mode.

With smode support in iproute2 'devlink' utility, switch mode can be set and get via following commands.

    # devlink dev smode pci/0000:05:00.0
    mode: legacy
    # devlink dev set pci/0000:05:00.0 smode switchdev
    # devlink dev smode pci/0000:05:00.0
    mode: switchdev

Signed-off-by Sridhar Samudrala <[email protected]>

Quoted 8 lines · Sridhar Samudrala

Add initial devlink support to set/get the mode of SRIOV switch. This patch allows the mode to be set to either 'legacy' or 'switchdev', but doesn't implement any functionality to create vf representors in switchdev mode.

With smode support in iproute2 'devlink' utility, switch mode can be set and get via following commands.

Quoted 7 lines

    # devlink dev smode pci/0000:05:00.0
    mode: legacy
    # devlink dev set pci/0000:05:00.0 smode switchdev
    # devlink dev smode pci/0000:05:00.0
    mode: switchdev

Signed-off-by Sridhar Samudrala <[email protected]>

I really don't see much value in this patch. If you are going to support SwitchDev then just do it. Otherwise you are adding extra overhead for maintaining two different modes.

I would recommend putting this series out to netdev as an RFC. Submitting it to intel-wired-lan is kind of pointless as the audience it to small to get any valuable review.

- Alex

Quoted 26 lines · Alexander Duyck

Quoted 8 lines · Sridhar Samudrala

Add initial devlink support to set/get the mode of SRIOV switch. This patch allows the mode to be set to either 'legacy' or 'switchdev', but doesn't implement any functionality to create vf representors in switchdev mode.

With smode support in iproute2 'devlink' utility, switch mode can be set and get via following commands.

Quoted 7 lines

    # devlink dev smode pci/0000:05:00.0
    mode: legacy
    # devlink dev set pci/0000:05:00.0 smode switchdev
    # devlink dev smode pci/0000:05:00.0
    mode: switchdev

Signed-off-by Sridhar Samudrala <[email protected]>

I really don't see much value in this patch. If you are going to support SwitchDev then just do it. Otherwise you are adding extra overhead for maintaining two different modes.

I would recommend putting this series out to netdev as an RFC. Submitting it to intel-wired-lan is kind of pointless as the audience it to small to get any valuable review.

Quoted 1 line

- Alex

I argued at length about this already. Jiri and company wanted this flag to push device in and out of this mode. Here we are just following the already upstreamed and debated decision.

This is less about switchdev and more about generating VF netdevs to use with ip tools and friends.

Another option would be to just always enable VF netdevs and have no legacy mode at all. I think that would be fine it just depends on if you think having extra netdevs around will confuse the stack at all. It might create a few corner cases but one reasonable thing to do would be to just fix those cases as they appear.

Quoted 27 lines · John Fastabend

Quoted 26 lines · Alexander Duyck

Quoted 8 lines · Sridhar Samudrala

Add initial devlink support to set/get the mode of SRIOV switch. This patch allows the mode to be set to either 'legacy' or 'switchdev', but doesn't implement any functionality to create vf representors in switchdev mode.

With smode support in iproute2 'devlink' utility, switch mode can be set and get via following commands.

Quoted 7 lines

    # devlink dev smode pci/0000:05:00.0
    mode: legacy
    # devlink dev set pci/0000:05:00.0 smode switchdev
    # devlink dev smode pci/0000:05:00.0
    mode: switchdev

Signed-off-by Sridhar Samudrala <[email protected]>

I really don't see much value in this patch. If you are going to support SwitchDev then just do it. Otherwise you are adding extra overhead for maintaining two different modes.

I would recommend putting this series out to netdev as an RFC. Submitting it to intel-wired-lan is kind of pointless as the audience it to small to get any valuable review.

Quoted 5 lines

Quoted 1 line

- Alex

I argued at length about this already. Jiri and company wanted this flag to push device in and out of this mode. Here we are just following the already upstreamed and debated decision.

Yeah, I started doing more research after reviewing this patch. I see the idea behind it. I think the issue if anything is that it seems like things are a bit backwards. We probably should be enabling the SwitchDev bits first and then working on adding devlink knobs to disable things later.

Quoted 2 lines

This is less about switchdev and more about generating VF netdevs to use with ip tools and friends.

Right. One of the issues I have with this patch set is that it seems to get things backwards. They are making VFs appear that don't do much of anything and then trying to bolt on features after the fact. We probably need to focus on enabling the VF representation, and then providing the ability to switch them on and off. Also I would argue that we should actually be enabling switch features such as FDB entries instead of trying to bolt on stuff like flow director which doesn't really apply to very many switches and isn't as likely to be used on a switch port.

Quoted 6 lines

Another option would be to just always enable VF netdevs and have no legacy mode at all. I think that would be fine it just depends on if you think having extra netdevs around will confuse the stack at all. It might create a few corner cases but one reasonable thing to do would be to just fix those cases as they appear.

I'd say we are better off starting out with them just enabled and then enabling the option to disable them after the fact. If we are going to have this extra code floating around we should be defaulting it to enabled so that it is more likely to be used. The legacy option should only really be there so we can turn this off if we don't want it.

- Alex

Quoted 19 lines · Alexander Duyck

Quoted 17 lines · John Fastabend

Quoted 16 lines · Alexander Duyck

Quoted 14 lines · Sridhar Samudrala

Add initial devlink support to set/get the mode of SRIOV switch. This patch allows the mode to be set to either 'legacy' or 'switchdev', but doesn't implement any functionality to create vf representors in switchdev mode.

With smode support in iproute2 'devlink' utility, switch mode can be set and get via following commands.

     # devlink dev smode pci/0000:05:00.0
     mode: legacy
     # devlink dev set pci/0000:05:00.0 smode switchdev
     # devlink dev smode pci/0000:05:00.0
     mode: switchdev

Quoted 8 lines

Quoted 8 lines

Quoted 8 lines

Quoted 1 line

Signed-off-by Sridhar Samudrala <[email protected]>

I really don't see much value in this patch. If you are going to support SwitchDev then just do it. Otherwise you are adding extra overhead for maintaining two different modes.

I would recommend putting this series out to netdev as an RFC. Submitting it to intel-wired-lan is kind of pointless as the audience it to small to get any valuable review.

Quoted 11 lines

Quoted 5 lines

Quoted 2 lines

- Alex

I argued at length about this already. Jiri and company wanted this flag to push device in and out of this mode. Here we are just following the already upstreamed and debated decision.

Yeah, I started doing more research after reviewing this patch. I see the idea behind it. I think the issue if anything is that it seems like things are a bit backwards. We probably should be enabling the SwitchDev bits first and then working on adding devlink knobs to disable things later.

Sure, although its not clear to me exactly which switchdev bits are useful for an edge NIC like this. Getting switch ids is one thing that will become useful when we enable multiple bridges.

Otherwise I don't see what l2 switchdev blocks are useful vs just using the standard ndo op interfaces already in place when working on a device without learning/aging/etc. The one thing that I've never bothered to add is pushing "learned" rules down into the hardware but I'm not convinced for most use cases this is particularly interesting because you _should_ know in a managed system what MAC addresses a VM/container/etc is allowed to use ahead of time via libvirt or other mgmt stack. I haven't tested the VLAN handling though so that needs to be looked at.

And l3 switchdev routing may be interesting but its fairly low on my priority list unless someone is really excited about it.

Quoted 11 lines

Quoted 2 lines

This is less about switchdev and more about generating VF netdevs to use with ip tools and friends.

Right. One of the issues I have with this patch set is that it seems to get things backwards. They are making VFs appear that don't do much of anything and then trying to bolt on features after the fact. We probably need to focus on enabling the VF representation, and then providing the ability to switch them on and off. Also I would argue that we should actually be enabling switch features such as FDB entries instead of trying to bolt on stuff like flow director which doesn't really apply to very many switches and isn't as likely to be used on a switch port.

Fair enough. Organizing the patches better seems OK to me. I plan to use the 'tc' offloaded mechanisms not the ethtool flow director interface for virtual switch offloads.

Quoted 5 lines

Quoted 5 lines

Another option would be to just always enable VF netdevs and have no legacy mode at all. I think that would be fine it just depends on if you think having extra netdevs around will confuse the stack at all. It might create a few corner cases but one reasonable thing to do would be to just fix those cases as they appear.

Quoted 7 lines

I'd say we are better off starting out with them just enabled and then enabling the option to disable them after the fact. If we are going to have this extra code floating around we should be defaulting it to enabled so that it is more likely to be used. The legacy option should only really be there so we can turn this off if we don't want it.

Works for me.

Quoted 30 lines · John Fastabend

Quoted 28 lines · Alexander Duyck

Quoted 25 lines · John Fastabend

Quoted 23 lines · Alexander Duyck

Quoted 19 lines · Sridhar Samudrala

Add initial devlink support to set/get the mode of SRIOV switch. This patch allows the mode to be set to either 'legacy' or 'switchdev', but
doesn't implement any functionality to create vf representors in switchdev
mode.

With smode support in iproute2 'devlink' utility, switch mode can be set
and get via following commands.

     # devlink dev smode pci/0000:05:00.0
     mode: legacy
     # devlink dev set pci/0000:05:00.0 smode switchdev
     # devlink dev smode pci/0000:05:00.0
     mode: switchdev

Signed-off-by Sridhar Samudrala <[email protected]>

Quoted 7 lines

Quoted 7 lines

Quoted 7 lines

Quoted 7 lines

I really don't see much value in this patch. If you are going to support SwitchDev then just do it. Otherwise you are adding extra overhead for maintaining two different modes.

I would recommend putting this series out to netdev as an RFC. Submitting it to intel-wired-lan is kind of pointless as the audience it to small to get any valuable review.

As these patches have only changes to i40e driver, i didn't include netdev. Will do so when
i submit v2.

Quoted 13 lines

Quoted 7 lines

Quoted 6 lines

Quoted 2 lines

- Alex

I argued at length about this already. Jiri and company wanted this flag to push device in and out of this mode. Here we are just following the already upstreamed and debated decision.

Quoted 6 lines

Yeah, I started doing more research after reviewing this patch. I see the idea behind it. I think the issue if anything is that it seems like things are a bit backwards. We probably should be enabling the SwitchDev bits first and then working on adding devlink knobs to disable things later.

Quoted 3 lines

Sure, although its not clear to me exactly which switchdev bits are useful for an edge NIC like this. Getting switch ids is one thing that will become useful when we enable multiple bridges.

This patchset is not really adding any switchdev ops. We are calling the mode in which the
PF driver creates VF representors as SWITCHDEV mode (this is based on the netdev
discussion of mellanox patches).

Amritha has a patch to add switchdev ops to VF representor netdevs that enables returning
switch id via switchdev_port_attr_get() op.

Quoted 37 lines

Otherwise I don't see what l2 switchdev blocks are useful vs just using the standard ndo op interfaces already in place when working on a device without learning/aging/etc. The one thing that I've never bothered to add is pushing "learned" rules down into the hardware but I'm not convinced for most use cases this is particularly interesting because you _should_ know in a managed system what MAC addresses a VM/container/etc is allowed to use ahead of time via libvirt or other mgmt stack. I haven't tested the VLAN handling though so that needs to be looked at.

And l3 switchdev routing may be interesting but its fairly low on my priority list unless someone is really excited about it.

Quoted 12 lines

Quoted 2 lines

This is less about switchdev and more about generating VF netdevs to use with ip tools and friends.

Right. One of the issues I have with this patch set is that it seems to get things backwards. They are making VFs appear that don't do much of anything and then trying to bolt on features after the fact. We probably need to focus on enabling the VF representation, and then providing the ability to switch them on and off. Also I would argue that we should actually be enabling switch features such as FDB entries instead of trying to bolt on stuff like flow director which doesn't really apply to very many switches and isn't as likely to be used on a switch port.

Fair enough. Organizing the patches better seems OK to me. I plan to use the 'tc' offloaded mechanisms not the ethtool flow director interface for virtual switch offloads.

Quoted 6 lines

Quoted 5 lines

Another option would be to just always enable VF netdevs and have no legacy mode at all. I think that would be fine it just depends on if you think having extra netdevs around will confuse the stack at all. It might create a few corner cases but one reasonable thing to do would be to just fix those cases as they appear.

Quoted 9 lines

Quoted 7 lines

I'd say we are better off starting out with them just enabled and then enabling the option to disable them after the fact. If we are going to have this extra code floating around we should be defaulting it to enabled so that it is more likely to be used. The legacy option should only really be there so we can turn this off if we don't want it.

Works for me.

OK. If we all agree that 'creating VF netdevs' by default is the right way to go, I will rearrange the
patchset in the following order.
- Introduce VF representor netdevs (create VF netdevs by default when
sr-iov VFs are enabled)
- enable ethtool stats on VF representors
- enable ntuple filters on VF representors

- introduce devlink to enable the switch mode to be changed to 'legacy'
(no VF netdevs)

Thanks
Sridhar

Quoted 103 lines · Sridhar

Quoted 23 lines · John Fastabend

Quoted 22 lines · Alexander Duyck

Quoted 20 lines · John Fastabend

Quoted 19 lines · Alexander Duyck

Quoted 17 lines · Sridhar Samudrala

Add initial devlink support to set/get the mode of SRIOV switch. This patch allows the mode to be set to either 'legacy' or 'switchdev', but
doesn't implement any functionality to create vf representors in switchdev
mode.

With smode support in iproute2 'devlink' utility, switch mode can be set
and get via following commands.

      # devlink dev smode pci/0000:05:00.0
      mode: legacy
      # devlink dev set pci/0000:05:00.0 smode switchdev
      # devlink dev smode pci/0000:05:00.0
      mode: switchdev

Quoted 8 lines

Quoted 8 lines

Quoted 8 lines

Quoted 8 lines

Quoted 1 line

Signed-off-by Sridhar Samudrala <[email protected]>

I really don't see much value in this patch. If you are going to support SwitchDev then just do it. Otherwise you are adding extra overhead for maintaining two different modes.

I would recommend putting this series out to netdev as an RFC. Submitting it to intel-wired-lan is kind of pointless as the audience it to small to get any valuable review.

As these patches have only changes to i40e driver, i didn't include netdev. Will do so when
i submit v2.

Quoted 10 lines

Quoted 10 lines

Quoted 4 lines

Quoted 1 line

- Alex

I argued at length about this already. Jiri and company wanted this flag to push device in and out of this mode. Here we are just following the already upstreamed and debated decision.

Yeah, I started doing more research after reviewing this patch. I see the idea behind it. I think the issue if anything is that it seems like things are a bit backwards. We probably should be enabling the SwitchDev bits first and then working on adding devlink knobs to disable things later.

Quoted 3 lines

Sure, although its not clear to me exactly which switchdev bits are useful for an edge NIC like this. Getting switch ids is one thing that will become useful when we enable multiple bridges.

This patchset is not really adding any switchdev ops. We are calling the mode in which the
PF driver creates VF representors as SWITCHDEV mode (this is based on the netdev
discussion of mellanox patches).

Amritha has a patch to add switchdev ops to VF representor netdevs that enables returning
switch id via switchdev_port_attr_get() op.

Quoted 33 lines

Otherwise I don't see what l2 switchdev blocks are useful vs just using the standard ndo op interfaces already in place when working on a device without learning/aging/etc. The one thing that I've never bothered to add is pushing "learned" rules down into the hardware but I'm not convinced for most use cases this is particularly interesting because you _should_ know in a managed system what MAC addresses a VM/container/etc is allowed to use ahead of time via libvirt or other mgmt stack. I haven't tested the VLAN handling though so that needs to be looked at.

And l3 switchdev routing may be interesting but its fairly low on my priority list unless someone is really excited about it.

Quoted 11 lines

Quoted 2 lines

This is less about switchdev and more about generating VF netdevs to use with ip tools and friends.

Right. One of the issues I have with this patch set is that it seems to get things backwards. They are making VFs appear that don't do much of anything and then trying to bolt on features after the fact. We probably need to focus on enabling the VF representation, and then providing the ability to switch them on and off. Also I would argue that we should actually be enabling switch features such as FDB entries instead of trying to bolt on stuff like flow director which doesn't really apply to very many switches and isn't as likely to be used on a switch port.

Fair enough. Organizing the patches better seems OK to me. I plan to use the 'tc' offloaded mechanisms not the ethtool flow director interface for virtual switch offloads.

Quoted 5 lines

Quoted 5 lines

Another option would be to just always enable VF netdevs and have no legacy mode at all. I think that would be fine it just depends on if you think having extra netdevs around will confuse the stack at all. It might create a few corner cases but one reasonable thing to do would be to just fix those cases as they appear.

Quoted 9 lines

Quoted 7 lines

I'd say we are better off starting out with them just enabled and then enabling the option to disable them after the fact. If we are going to have this extra code floating around we should be defaulting it to enabled so that it is more likely to be used. The legacy option should only really be there so we can turn this off if we don't want it.

Works for me.

Quoted 7 lines

OK. If we all agree that 'creating VF netdevs' by default is the right way to go, I will rearrange the
patchset in the following order.
- Introduce VF representor netdevs (create VF netdevs by default when
sr-iov VFs are enabled)
- enable ethtool stats on VF representors
- enable ntuple filters on VF representors

No NTUPLE filters. This is not the way the community will want to do this, and this is not how I want to do this.

The easy way to think of this is that NTUPLE is just an alias for Rx network flow classification. Now where are the VF representors supposed to receive traffic from? It is the Tx path of the VF. So by putting NTUPLE filters on the VF representors you are implying you can get in the Tx path for the VF for all traffic, and last I knew that was not the case. As such it makes no sense to do it this way.

Quoted 6 lines

- introduce devlink to enable the switch mode to be changed to 'legacy'
(no VF netdevs)

Thanks
Sridhar

You have to think of the VF representors as switch ports. They don't actually represent the VFs. So if you think about it a switch port will only have a few real features. You can control the VLAN membership, tagged/untagged, what FDB entries are assigned to the port, the link speed, if the link is up/down, etc. So some of your first steps might be to focus on the L1 and L2 type items. So for example you might see if we can do things like control the upper limit of the MTU for the VF by controlling the MTU for the VF representor. See if you can at least make it so that the MAC and VLAN values for the VF show up as FDB entries for the VF representor and so forth.

- Alex

The idea behind NTUPLE filters on VF representors is to match RX traffic that is destined
for the corresponding VF. Basically when adding the flow director filter, we are setting the
dest vsi as the VSI of the corresponding VF.
Without VF representors, this is currently done by overloading user-def value with VF id.

Quoted 8 lines · Sridhar

The idea behind NTUPLE filters on VF representors is to match RX traffic that is destined
for the corresponding VF. Basically when adding the flow director filter, we are setting the
dest vsi as the VSI of the corresponding VF.
Without VF representors, this is currently done by overloading user-def value with VF id.

Yes, and for now that is preferable to what you are doing since it makes no sense.

I will try to clarify my point. First, instead of calling these VF representors lets call them VF port representor, or VFPR for short (trying to avoid VFR because it sounds too much like VRF which is completely unrelated). So the whole point of a VFPR is to present the VF as a port on your logical switch. You should mostly only be capable of doing on it what you would do with an actual switch. So for example if I have a switch connected to NIC. Now most switches have certain options for controlling things. For example I can force the switch port onto a given VLAN so that the port is isolated from other traffic. I can control the max frame size allowed to be sent or received over the port. I can block what traffic the port is allowed to send by restricting MAC addresses via ACLs, and I can manually program the forwarding table via FDB entries.

The problem is Flow Director also know as NTUPLE or Rx NFC only operates on the Rx side of NICs. In your VFPR the Rx side represents
packets coming from your VF, not packets going to it. If you
transmit on a VFPR the packet should be received only by the VF, if you receive from it the packet should only be from the VF. So really any Flow Director rule you add to the VFPR should not impact the VF, but instead should impact the VFPR and the PF since the PF will add the rule and it should only apply on traffic received on the VFPR from the VF.

At some point in the future we can look at adding flow Tx offloads via either TC or SwitchDev. For now adding NTUPLE is a strong no-go. The VFPR are not the place to do it unless you are just wanting to add custom rules to the PF itself that won't impact the VF.

- Alex

With smode support in iproute2 'devlink' utility, switch mode can be set and get via following commands.

??? # devlink dev smode pci/0000:05:00.0
??? mode: legacy
??? # devlink dev set pci/0000:05:00.0 smode switchdev ??? # devlink dev smode pci/0000:05:00.0
??? mode: switchdev

Signed-off-by Sridhar Samudrala <[email protected]>

---

?drivers/net/ethernet/intel/Kconfig????????? |? 1 +
?drivers/net/ethernet/intel/i40e/i40e.h????? |? 3 ++
?drivers/net/ethernet/intel/i40e/i40e_main.c | 84

++++++++++++++++++++++++++---
?3 files changed, 80 insertions(+), 8 deletions(-)