Skip to contentSkip to conversation

pci: Add pci_num_vf function

RFC PATCH In review Jeff Kirsher 4 participants 10 messages Dec 18, 2009 – Dec 21, 2009

Subsystems (3)
lore
Share & tools
lore
#0 lore raw

RFC PATCH v2 1/6 pci: Add pci_num_vf function

From: Williams, Mitch A <[email protected]>

Add and export pci_num_vf to allow other subsystems to determine how many virtual function devices are associated with an SR-IOV physical function device.

---

 drivers/pci/iov.c   |   15 +++++++++++++++
 include/linux/pci.h |    5 +++++
 2 files changed, 20 insertions(+), 0 deletions(-)
drivers/pci/iov.c +15 -0
index b2a448e..52ee6d3 100644
@@ -706,6 +706,21 @@ irqreturn_t pci_sriov_migration(struct pci_dev *dev)
 }
 EXPORT_SYMBOL_GPL(pci_sriov_migration);
 
+/**
+ * pci_num_vf - return number of VFs associated with a PF device_release_driver
+ * @dev: the PCI device
+ *
+ * Returns number of VFs, or 0 if SR-IOV is not enabled.
+ */
+int pci_num_vf(struct pci_dev *dev)
+{
+	if (!dev->is_physfn)
+		return 0;
+	else
+		return dev->sriov->nr_virtfn;
+}
+EXPORT_SYMBOL_GPL(pci_num_vf);
+
 static int ats_alloc_one(struct pci_dev *dev, int ps)
 {
 	int pos;
include/linux/pci.h +5 -0
index 04771b9..c44d6e0 100644
@@ -1282,6 +1282,7 @@ void __iomem *pci_ioremap_bar(struct pci_dev *pdev, int bar);
 extern int pci_enable_sriov(struct pci_dev *dev, int nr_virtfn);
 extern void pci_disable_sriov(struct pci_dev *dev);
 extern irqreturn_t pci_sriov_migration(struct pci_dev *dev);
+extern int pci_num_vf(struct pci_dev *dev);
 #else
 static inline int pci_enable_sriov(struct pci_dev *dev, int nr_virtfn)
 {
@@ -1294,6 +1295,10 @@ static inline irqreturn_t pci_sriov_migration(struct pci_dev *dev)
 {
 	return IRQ_NONE;
 }
+static inline int pci_num_vf(struct pci_dev *dev)
+{
+	return 0;
+}
 #endif
 
 #if defined(CONFIG_HOTPLUG_PCI) || defined(CONFIG_HOTPLUG_PCI_MODULE)
#1 lore raw

RFC PATCH v2 2/6 net: Add netif_num_vf function

From: Williams, Mitch A <[email protected]>

Add a convenience function to determine how many VF devices are associated with a given PF network interface. If the device is not an SR-IOV physical function device, the function returns 0.

---

 include/linux/netdevice.h |   27 +++++++++++++++++++++++++++
 1 files changed, 27 insertions(+), 0 deletions(-)
include/linux/netdevice.h +27 -0
index a3fccc8..f6e521e 100644
@@ -38,6 +38,7 @@
 #include <asm/byteorder.h>
 
 #include <linux/device.h>
+#include <linux/pci.h>
 #include <linux/percpu.h>
 #include <linux/rculist.h>
 #include <linux/dmaengine.h>
@@ -1509,6 +1510,32 @@ static inline int netif_is_multiqueue(const struct net_device *dev)
 {
 	return (dev->num_tx_queues > 1);
 }
+/**
+ *	netif_num_vf - return number of SR-IOV VFs
+ *	@dev: network device
+ *
+ *	Return the number of SR-IOV virtual function devices that are
+ *	associated with this physical function device. Returns 0 if
+ *	device is not a PF device, or if SR-IOV not enabled.
+ */
+static inline int netif_num_vf(const struct net_device *dev)
+{
+	struct pci_dev *pdev;
+	int retval = 0;
+
+	if (!dev)
+		goto out;
+
+	if (!dev->dev.parent)
+		goto out;
+
+	if (dev->dev.parent->bus == &pci_bus_type) {
+		pdev = to_pci_dev(dev->dev.parent);
+		retval = pci_num_vf(pdev);
+	}
+out:
+	return retval;
+}
 
 /* Use this variant when it is known for sure that it
  * is executing from hardware interrupt context or with hardware interrupts
#2 lore raw

Re: [RFC PATCH v2 2/6] net: Add netif_num_vf function

Quoted 5 lines · Jeff Kirsher

From: Williams, Mitch A <[email protected]>

Add a convenience function to determine how many VF devices are associated with a given PF network interface. If the device is not an SR-IOV physical function device, the function returns 0.

[...]

Quoted 5 lines
include/linux/netdevice.h +5 -0
+/**
+ *	netif_num_vf - return number of SR-IOV VFs
+ *	@dev: network device
+ *
+ *	Return the number of SR-IOV virtual function devices that are

include/linux/netdevice.h static inline int netif_is_multiqueue(const struct net_device *dev) · line 1527

Quoted 10 lines
include/linux/netdevice.h +10 -0
+ *	associated with this physical function device. Returns 0 if
+ *	device is not a PF device, or if SR-IOV not enabled.
+ */
+static inline int netif_num_vf(const struct net_device *dev)
+{
+	struct pci_dev *pdev;
+	int retval = 0;
+
+	if (!dev)
+		goto out;

Why should this function allow dev == NULL?

include/linux/netdevice.h static inline int netif_is_multiqueue(const struct net_device *dev) · line 1537

Quoted 9 lines
include/linux/netdevice.h +9 -0
+	if (!dev->dev.parent)
+		goto out;
+
+	if (dev->dev.parent->bus == &pci_bus_type) {
+		pdev = to_pci_dev(dev->dev.parent);
+		retval = pci_num_vf(pdev);
+	}
+out:
+	return retval;

[...]

It would be much clearer to write:

	struct pci_dev *pdev;

	if (dev && dev->dev.parent &&
	    dev->dev.parent->bus == &pci_bus_type) {
		pdev = to_pci_dev(dev->dev.parent);
		return pci_num_vf(pdev);
	} else {
		return 0;
	}

Ben.

Signature · 3 lines
Ben Hutchings, Senior Software Engineer, Solarflare Communications
Not speaking for my employer; that's the marketing department's job.
They asked us to note that Solarflare product names are trademarked.
maintainer
#3 lore raw

Re: [RFC PATCH v2 2/6] net: Add netif_num_vf function

In reply to Jeff Kirsher

Well, first of all this won't build without CONFIG_PCI defined.

Second of all, we're not putting PCI crap into netdevice.h

Create a generic device layer abstraction by which you can fetch these values.

Thanks.

#4 lore raw

RE: [RFC PATCH v2 2/6] net: Add netif_num_vf function

Quoted 6 lines

Subject: Re: [RFC PATCH v2 2/6] net: Add netif_num_vf function

Well, first of all this won't build without CONFIG_PCI defined.

Quoted 6 lines

Second of all, we're not putting PCI crap into netdevice.h

Create a generic device layer abstraction by which you can fetch these values.

Thanks.

OK, I'll respin this and resubmit after the new year.

-Mitch

#5 lore raw

RFC PATCH v2 3/6 if_link: Add SR-IOV configuration methods

In reply to Jeff Kirsher

From: Williams, Mitch A <[email protected]>

Add SR-IOV VF management methods to IFLA_LINKINFO. This allows userspace to use rtnetlink to configure VF network devices.

---

 include/linux/if_link.h |   28 ++++++++++++++++++++++++++++
 1 files changed, 28 insertions(+), 0 deletions(-)
include/linux/if_link.h +28 -0
index 6674791..7d0c687 100644
@@ -78,6 +78,11 @@ enum {
 #define IFLA_LINKINFO IFLA_LINKINFO
 	IFLA_NET_NS_PID,
 	IFLA_IFALIAS,
+	IFLA_NUM_VF,		/* Number of VFs if device is SR-IOV PF */
+	IFLA_VF_MAC,		/* Hardware queue specific attributes */
+	IFLA_VF_VLAN,
+	IFLA_VF_TX_RATE,	/* TX Bandwidth Allocation */
+	IFLA_VFINFO,
 	__IFLA_MAX
 };
 
@@ -196,4 +201,27 @@ enum macvlan_mode {
 	MACVLAN_MODE_BRIDGE  = 4, /* talk to bridge ports directly */
 };
 
+/* SR-IOV virtual function managment section */
+
+struct ifla_vf_mac {
+	__u32 vf;
+	__u8 mac[32]; /* MAX_ADDR_LEN */
+};
+
+struct ifla_vf_vlan {
+	__u32 vf;
+	__s32 vlan; /* 0 - 4095, -1 disables VLAN filter */
+};
+
+struct ifla_vf_tx_rate {
+	__u32 vf;
+	__u32 rate; /* Max TX bandwidth in Mbps, 0 disables throttling */
+};
+
+struct ifla_vf_info {
+	__u32 vf;
+	__u8 mac[32];
+	__s32 vlan;
+	__u32 tx_rate;
+};
 #endif /* _LINUX_IF_LINK_H */
#6 lore raw

RFC PATCH v2 4/6 net: Add netdev ops for SR-IOV configuration

In reply to Jeff Kirsher

From: Williams, Mitch A <[email protected]>

Add netdev ops for configuring SR-IOV VF devices through the PF driver.

---

 include/linux/netdevice.h |   17 +++++++++++++++++
 1 files changed, 17 insertions(+), 0 deletions(-)
include/linux/netdevice.h +17 -0
index f6e521e..0e42077 100644
@@ -28,6 +28,7 @@
 #include <linux/if.h>
 #include <linux/if_ether.h>
 #include <linux/if_packet.h>
+#include <linux/if_link.h>
 
 #ifdef __KERNEL__
 #include <linux/timer.h>
@@ -611,6 +612,13 @@ struct netdev_queue {
  *	this function is called when a VLAN id is unregistered.
  *
  * void (*ndo_poll_controller)(struct net_device *dev);
+ *
+ *	SR-IOV management functions.
+ * int (*ndo_set_vf_mac)(struct net_device *dev, int vf, u8* mac);
+ * int (*ndo_set_vf_vlan)(struct net_device *dev, int vf, u16 vlan);
+ * int (*ndo_set_vf_tx_rate)(struct net_device *dev, int vf, int rate);
+ * int (*ndo_get_vf_config)(struct net_device *dev,
+ *			    int vf, struct ifla_vf_info *ivf);
  */
 #define HAVE_NET_DEVICE_OPS
 struct net_device_ops {
@@ -660,6 +668,15 @@ struct net_device_ops {
 #define HAVE_NETDEV_POLL
 	void                    (*ndo_poll_controller)(struct net_device *dev);
 #endif
+	int			(*ndo_set_vf_mac)(struct net_device *dev,
+						  int queue, u8 *mac);
+	int			(*ndo_set_vf_vlan)(struct net_device *dev,
+						   int queue, u16 vlan);
+	int			(*ndo_set_vf_tx_rate)(struct net_device *dev,
+						      int vf, int rate);
+	int			(*ndo_get_vf_config)(struct net_device *dev,
+						     int vf,
+						     struct ifla_vf_info *ivf);
 #if defined(CONFIG_FCOE) || defined(CONFIG_FCOE_MODULE)
 	int			(*ndo_fcoe_enable)(struct net_device *dev);
 	int			(*ndo_fcoe_disable)(struct net_device *dev);
#7 lore raw

Re: [RFC PATCH v2 4/6] net: Add netdev ops for SR-IOV configuration

Quoted 3 lines · Jeff Kirsher

From: Williams, Mitch A <[email protected]>

Add netdev ops for configuring SR-IOV VF devices through the PF driver.

Though this be restricted to VFs, though?

It is possible to have virtualised interfaces appear as PFs, so they can easily be passed-through to guests with no IOV support. HP's 'Virtual Connect Flex-10' NICs do this, but I'm not sure how they are configured and whether it is possible to do so from Linux as opposed to firmware.

Solarstorm controllers also have an on-NIC page table and page- segregated queues that can be mapped into PV guests, which could perhaps be made configurable in the same way.

[...]

include/linux/netdevice.h struct net_device_ops { · line 679

Quoted 13 lines
include/linux/netdevice.h +9 -0
@@ -660,6 +668,15 @@ struct net_device_ops {
 #define HAVE_NETDEV_POLL
 	void                    (*ndo_poll_controller)(struct net_device *dev);
 #endif
+	int			(*ndo_set_vf_mac)(struct net_device *dev,
+						  int queue, u8 *mac);
+	int			(*ndo_set_vf_vlan)(struct net_device *dev,
+						   int queue, u16 vlan);
+	int			(*ndo_set_vf_tx_rate)(struct net_device *dev,
+						      int vf, int rate);
+	int			(*ndo_get_vf_config)(struct net_device *dev,
+						     int vf,
+						     struct ifla_vf_info *ivf);

[...]

If these remain specific to IOV, presumably they should be guarded by #ifdef CONFIG_PCI_IOV.

Ben.

Signature · 3 lines
Ben Hutchings, Senior Software Engineer, Solarflare Communications
Not speaking for my employer; that's the marketing department's job.
They asked us to note that Solarflare product names are trademarked.
#8 lore raw

RFC PATCH v2 5/6 rtnetlink: Add VF config code to rtnetlink

In reply to Jeff Kirsher

From: Williams, Mitch A <[email protected]>

Add code to allow rtnetlink clients to query and set VF information through the PF driver.

---

 net/core/rtnetlink.c |   59 ++++++++++++++++++++++++++++++++++++++++++++++++++
 1 files changed, 59 insertions(+), 0 deletions(-)
net/core/rtnetlink.c +59 -0
index 794bcb8..090b9fc 100644
@@ -580,6 +580,11 @@ static void copy_rtnl_link_stats(struct rtnl_link_stats *a,
 	a->tx_compressed = b->tx_compressed;
 };
 
+static inline int rtnl_vfinfo_size(const struct net_device *dev)
+{
+	return netif_num_vf(dev) * sizeof(struct ifla_vf_info);
+}
+
 static inline size_t if_nlmsg_size(const struct net_device *dev)
 {
 	return NLMSG_ALIGN(sizeof(struct ifinfomsg))
@@ -597,6 +602,8 @@ static inline size_t if_nlmsg_size(const struct net_device *dev)
 	       + nla_total_size(4) /* IFLA_MASTER */
 	       + nla_total_size(1) /* IFLA_OPERSTATE */
 	       + nla_total_size(1) /* IFLA_LINKMODE */
+	       + nla_total_size(4) /* IFLA_NUM_VF */
+	       + nla_total_size(rtnl_vfinfo_size(dev)) /* IFLA_VFINFO */
 	       + rtnl_link_get_size(dev); /* IFLA_LINKINFO */
 }
 
@@ -665,6 +672,16 @@ static int rtnl_fill_ifinfo(struct sk_buff *skb, struct net_device *dev,
 	stats = dev_get_stats(dev);
 	copy_rtnl_link_stats(nla_data(attr), stats);
 
+	if ((dev->netdev_ops->ndo_get_vf_config) && netif_num_vf(dev)) {
+		int i;
+		struct ifla_vf_info ivi;
+		NLA_PUT_U32(skb, IFLA_NUM_VF, netif_num_vf(dev));
+		for (i = 0; i < netif_num_vf(dev); i++) {
+			if (dev->netdev_ops->ndo_get_vf_config(dev, i, &ivi))
+				break;
+			NLA_PUT(skb, IFLA_VFINFO, sizeof(ivi), &ivi);
+		}
+	}
 	if (dev->rtnl_link_ops) {
 		if (rtnl_link_fill(skb, dev) < 0)
 			goto nla_put_failure;
@@ -725,6 +742,12 @@ const struct nla_policy ifla_policy[IFLA_MAX+1] = {
 	[IFLA_LINKINFO]		= { .type = NLA_NESTED },
 	[IFLA_NET_NS_PID]	= { .type = NLA_U32 },
 	[IFLA_IFALIAS]	        = { .type = NLA_STRING, .len = IFALIASZ-1 },
+	[IFLA_VF_MAC]		= { .type = NLA_BINARY,
+				    .len = sizeof(struct ifla_vf_mac) },
+	[IFLA_VF_VLAN]		= { .type = NLA_BINARY,
+				    .len = sizeof(struct ifla_vf_vlan) },
+	[IFLA_VF_TX_RATE]	= { .type = NLA_BINARY,
+				    .len = sizeof(struct ifla_vf_tx_rate) },
 };
 EXPORT_SYMBOL(ifla_policy);
 
@@ -898,6 +921,42 @@ static int do_setlink(struct net_device *dev, struct ifinfomsg *ifm,
 		write_unlock_bh(&dev_base_lock);
 	}
 
+	if (tb[IFLA_VF_MAC]) {
+		struct ifla_vf_mac *ivm;
+		ivm = nla_data(tb[IFLA_VF_MAC]);
+		write_lock_bh(&dev_base_lock);
+		if (ops->ndo_set_vf_mac)
+			err = ops->ndo_set_vf_mac(dev, ivm->vf, ivm->mac);
+		write_unlock_bh(&dev_base_lock);
+		if (err < 0)
+			goto errout;
+		modified = 1;
+	}
+
+	if (tb[IFLA_VF_VLAN]) {
+		struct ifla_vf_vlan *ivv;
+		ivv = nla_data(tb[IFLA_VF_VLAN]);
+		write_lock_bh(&dev_base_lock);
+		if (ops->ndo_set_vf_vlan)
+			err = ops->ndo_set_vf_vlan(dev, ivv->vf, ivv->vlan);
+		write_unlock_bh(&dev_base_lock);
+		if (err < 0)
+			goto errout;
+		modified = 1;
+	}
+	err = 0;
+
+	if (tb[IFLA_VF_TX_RATE]) {
+		struct ifla_vf_tx_rate *ivt;
+		ivt = nla_data(tb[IFLA_VF_TX_RATE]);
+		write_lock_bh(&dev_base_lock);
+		if (ops->ndo_set_vf_tx_rate)
+			err = ops->ndo_set_vf_tx_rate(dev, ivt->vf, ivt->rate);
+		write_unlock_bh(&dev_base_lock);
+		if (err < 0)
+			goto errout;
+		modified = 1;
+	}
 	err = 0;
 
 errout:
#9 lore raw

RFC PATCH v2 6/6 igb: add support for VF configuration tools

In reply to Jeff Kirsher

From: Williams, Mitch A <[email protected]>

Add support to the igb driver for VF configuration mechanisms through the PF interface.

---

 drivers/net/igb/e1000_82575.h |    3 +
 drivers/net/igb/e1000_regs.h  |    1 
 drivers/net/igb/igb.h         |    2 +
 drivers/net/igb/igb_main.c    |  105 +++++++++++++++++++++++++++++++++++++----
 4 files changed, 102 insertions(+), 9 deletions(-)
drivers/net/igb/e1000_82575.h +3 -0
index d51c992..bb53083 100644
@@ -219,6 +219,9 @@ struct e1000_adv_tx_context_desc {
 #define E1000_VLVF_LVLAN          0x00100000
 #define E1000_VLVF_VLANID_ENABLE  0x80000000
 
+#define E1000_VMVIR_VLANA_DEFAULT      0x40000000 /* Always use default VLAN */
+#define E1000_VMVIR_VLANA_NEVER        0x80000000 /* Never insert VLAN tag */
+
 #define E1000_IOVCTL 0x05BBC
 #define E1000_IOVCTL_REUSE_VFQ 0x00000001
 
drivers/net/igb/e1000_regs.h +1 -0
index dd4e6ff..abb7333 100644
@@ -310,6 +310,7 @@
 #define E1000_VMOLR(_n)        (0x05AD0 + (4 * (_n)))
 #define E1000_VLVF(_n)         (0x05D00 + (4 * (_n))) /* VLAN Virtual Machine
                                                        * Filter - RW */
+#define E1000_VMVIR(_n)        (0x03700 + (4 * (_n)))
 
 #define wr32(reg, value) (writel(value, hw->hw_addr + reg))
 #define rd32(reg) (readl(hw->hw_addr + reg))
drivers/net/igb/igb.h +2 -0
index b1c1eb8..92ca69c 100644
@@ -75,11 +75,13 @@ struct vf_data_storage {
 	u16 vlans_enabled;
 	u32 flags;
 	unsigned long last_nack;
+	u16 pf_vlan; /* When set, guest VLAN config not allowed. */
 };
 
 #define IGB_VF_FLAG_CTS            0x00000001 /* VF is clear to send data */
 #define IGB_VF_FLAG_UNI_PROMISC    0x00000002 /* VF has unicast promisc */
 #define IGB_VF_FLAG_MULTI_PROMISC  0x00000004 /* VF has multicast promisc */
+#define IGB_VF_FLAG_PF_SET_MAC     0x00000008 /* PF has set MAC address */
 
 /* RX descriptor control thresholds.
  * PTHRESH - MAC will consider prefetch if it has fewer than this number of
drivers/net/igb/igb_main.c +96 -9

171 lines not on this page · raw message