Skip to contentSkip to conversation

vfio: set vf token and gain vf device access

RFC In review [email protected] 9 participants 92 messages Mar 5, 2020 – Jul 7, 2020

lore
Share & tools
lore
#0 lore raw

RFC 1/1 vfio: set vf token and gain vf device access

From: Vamsi Attunuru <[email protected]>

vfio-pci driver enables virtual function access from the DPDK applications when those vf device's physical function is also bound to vfio driver.

Patch adds the required configuration and checks to enable DPDK applications to access both pf and it's vf devices through vfio-pci driver.

See background on vf token scheme in linux vfio driver.
http://patches.dpdk.org/cover/65915/

When a physical function is enabled with non-zero virtual functions, patch sets the UUID using VFIO_DEVICE_FEATURE ioctl from physical function's file descriptor. Same UUID is used to gain the access for the virtual functions on those physical function.

Following changes required on top of this DPDK patch
* Kernel version check for VFIO_DEVICE_FEATURE ioctl
* Use uuid gen API to generate UUID.

lib/librte_eal/linux/eal/eal_vfio.c +150 -1
index 01b5ef3..e2fdd35 100644
@@ -12,6 +12,7 @@
 #include <rte_log.h>
 #include <rte_memory.h>
 #include <rte_eal_memconfig.h>
+#include <rte_uuid.h>
 #include <rte_vfio.h>
 
 #include "eal_filesystem.h"
@@ -50,6 +51,9 @@ struct vfio_config {
 	struct user_mem_maps mem_maps;
 };
 
+rte_uuid_t uuid_token = RTE_UUID_INIT(0xf8615163, 0xdf3e, 0x46c5,
+				      0x913f, 0xf2d2f965ed0eULL);
+
 /* per-process VFIO config */
 static struct vfio_config vfio_cfgs[VFIO_MAX_CONTAINERS];
 static struct vfio_config *default_vfio_cfg = &vfio_cfgs[0];
@@ -657,6 +661,102 @@ rte_vfio_clear_group(int vfio_group_fd)
 	return 0;
 }
 
+static bool
+rte_vfio_dev_is_physfn(const char *sysfs_base, const char *dev_addr)
+{
+	char linkname[PATH_MAX];
+	char filename[PATH_MAX];
+	int ret;
+
+	memset(linkname, 0, sizeof(linkname));
+	memset(filename, 0, sizeof(filename));
+
+	/* check if physfn directory exist for this device */
+	snprintf(linkname, sizeof(linkname),
+			 "%s/%s/physfn", sysfs_base, dev_addr);
+
+	ret = readlink(linkname, filename, sizeof(filename));
+
+	/* For PFs, physfn directory does not exist */
+	if (ret < 0)
+		return true;
+
+	return false;
+}
+
+static int
+is_vf_token_required(const char *sysfs_base, const char *dev_addr)
+{
+	char *tok[16], *physfn, *physfn_drv;
+	char linkname[PATH_MAX];
+	char filename[PATH_MAX];
+	int ret;
+
+	memset(linkname, 0, sizeof(linkname));
+	memset(filename, 0, sizeof(filename));
+
+	snprintf(linkname, sizeof(linkname),
+			 "%s/%s/physfn", sysfs_base, dev_addr);
+
+	ret = readlink(linkname, filename, sizeof(filename));
+	if (ret < 0)
+		return -1;
+
+	ret = rte_strsplit(filename, sizeof(filename),
+			tok, RTE_DIM(tok), '/');
+	if (ret <= 0) {
+		RTE_LOG(ERR, EAL, " %s cannot get it's physfn\n", dev_addr);
+		return -1;
+	}
+
+	physfn = tok[ret - 1];
+
+	snprintf(linkname, sizeof(linkname),
+			 "/sys/bus/pci/devices/%s/driver", physfn);
+	ret = readlink(linkname, filename, sizeof(filename));
+	if (ret < 0)
+		return -1;
+
+	ret = rte_strsplit(filename, sizeof(filename),
+			tok, RTE_DIM(tok), '/');
+	if (ret <= 0) {
+		RTE_LOG(ERR, EAL, "  %s cannot get it's physfn driver info\n",
+			dev_addr);
+		return -1;
+	}
+
+	physfn_drv = tok[ret - 1];
+
+	if (strncmp(physfn_drv, "vfio-pci", sizeof("vfio-pci")))
+		return 1;
+
+	/* physfn is bound to vfio-pci */
+	return 0;
+}
+
+static bool
+rte_vfio_dev_has_nonzero_numvfs(const char *sysfs_base, const char *dev_addr)
+{
+	char linkname[PATH_MAX];
+	unsigned long num_vfs;
+	int ret;
+
+	if (!rte_vfio_dev_is_physfn(sysfs_base, dev_addr))
+		return false;
+
+	memset(linkname, 0, sizeof(linkname));
+
+	snprintf(linkname, sizeof(linkname),
+			 "%s/%s/sriov_numvfs", sysfs_base, dev_addr);
+
+	ret  = eal_parse_sysfs_value(linkname, &num_vfs);
+
+	if ((ret < 0) || (num_vfs == 0))
+		return false;
+
+	return true;
+}
+
 int
 rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 		int *vfio_dev_fd, struct vfio_device_info *device_info)
@@ -669,6 +769,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 	int vfio_container_fd;
 	int vfio_group_fd;
 	int iommu_group_num;
+	char dev[PATH_MAX];
 	int i, ret;
 
 	/* get group number */
@@ -683,6 +784,29 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 	if (ret < 0)
 		return -1;
 
+	snprintf(dev, sizeof(dev), "%s", dev_addr);
+
+	if (!rte_vfio_dev_is_physfn(sysfs_base, dev_addr)) {
+		char vf_token[PATH_MAX];
+		/*
+		 *  Check if vf_token is required or not,
+		 *  vf_token is required when the VF's physfn is
+		 *  binded with vfio-pci driver
+		 */
+		ret = is_vf_token_required(sysfs_base, dev_addr);
+		/* if negative, something failed */
+		if (ret < 0)
+			return -1;
+
+		if (ret == 0) {
+			/* vf_token required to open device file descriptor */
+			rte_uuid_unparse(uuid_token,
+					 vf_token, sizeof(vf_token));
+			snprintf(dev, sizeof(dev),
+				 "%s vf_token=%s", dev_addr, vf_token);
+		}
+	}
+
 	/* get the actual group fd */
 	vfio_group_fd = rte_vfio_get_group_fd(iommu_group_num);
 	if (vfio_group_fd < 0)
@@ -853,7 +977,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 	}
 
 	/* get a file descriptor for the device */
-	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev_addr);
+	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev);
 	if (*vfio_dev_fd < 0) {
 		/* if we cannot get a device fd, this implies a problem with
 		 * the VFIO group or the container not having IOMMU configured.
@@ -877,6 +1001,31 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 		rte_vfio_clear_group(vfio_group_fd);
 		return -1;
 	}
+
+	if (rte_vfio_dev_has_nonzero_numvfs(sysfs_base, dev_addr)) {
+#define VF_TOKEN  (sizeof(struct vfio_device_feature) + sizeof(rte_uuid_t))
+
+		struct vfio_device_feature *vf_token;
+		uint8_t local[VF_TOKEN];
+
+		memset(local, 0, VF_TOKEN);
+		vf_token = (struct vfio_device_feature *)local;
+		vf_token->argsz = VF_TOKEN;
+		vf_token->flags = VFIO_DEVICE_FEATURE_SET |
+					VFIO_DEVICE_FEATURE_PCI_VF_TOKEN;
+
+		memcpy(local + sizeof(struct vfio_device_feature),
+		       &uuid_token, sizeof(uuid_token));
+
+		ret = ioctl(*vfio_dev_fd, VFIO_DEVICE_FEATURE, vf_token);
+		if (ret) {
+			RTE_LOG(ERR, EAL, " Failed to set UUID on %s "
+					"error %i (%s)\n", dev_addr, errno,
+					strerror(errno));
+			return -1;
+		}
+	}
+
 	vfio_group_device_get(vfio_group_fd);
 
 	return 0;
Signature · 1 line
2.8.4
#1 lore raw

Snipped

lib/librte_eal/linux/eal/eal_vfio.c

Quoted 5 lines
lib/librte_eal/linux/eal/eal_vfio.c +0 -0
index 01b5ef3..e2fdd35 100644

DPDK supports freebsd where /dev/pci can be probed like Linux /sys/bus/pci. Will you be adding the functions in lib/librte_eal/freebsd/eal/ folder or implement as NOT supported?

What about window lib/librte_eal/windows/eal/?

snipped

lib/librte_eal/linux/eal/eal_vfio.c

Quoted 2 lines
lib/librte_eal/linux/eal/eal_vfio.c +2 -0
+	snprintf(linkname, sizeof(linkname),
+			 "%s/%s/physfn", sysfs_base, dev_addr);

Would you like to check the pointer sysfs_base or dev_addr?

Quoted 3 lines

int
rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
int *vfio_dev_fd, struct vfio_device_info *device_info) @@ -

Snipped

lib/librte_eal/linux/eal/eal_vfio.c rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr, · line 796

Quoted 1 line
lib/librte_eal/linux/eal/eal_vfio.c +1 -0
+		ret = is_vf_token_required(sysfs_base, dev_addr);

The return value from the above function is 0 or -1.

lib/librte_eal/linux/eal/eal_vfio.c rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr, · line 801

Quoted 5 lines
lib/librte_eal/linux/eal/eal_vfio.c +5 -0
+		/* if negative, something failed */
+		if (ret < 0)
+			return -1;
+
+		if (ret == 0) {

Will it be ok to skip this check?

lib/librte_eal/linux/eal/eal_vfio.c rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr, · line 807

Quoted 6 lines
lib/librte_eal/linux/eal/eal_vfio.c +6 -0
+			/* vf_token required to open device file descriptor */
+			rte_uuid_unparse(uuid_token,
+					 vf_token, sizeof(vf_token));
+			snprintf(dev, sizeof(dev),
+				 "%s vf_token=%s", dev_addr, vf_token);
+		}

snipped

general comment can we use EAL LOG for info, warn, error?

#2 lore raw

Hi Vipin,

Please see inline.

Quoted 21 lines

-----Original Message-----
From: dev <[email protected]> On Behalf Of Varghese, Vipin
Sent: Wednesday, April 1, 2020 8:45 AM
To: Vamsi Krishna Attunuru <[email protected]>; [email protected]
Cc: Jerin Jacob Kollanukkaran <[email protected]>; [email protected]; [email protected]; [email protected]
Subject: Re: [dpdk-dev] [RFC v1 1/1] vfio: set vf token and gain vf device access

Snipped

lib/librte_eal/linux/eal/eal_vfio.c

Quoted 5 lines
lib/librte_eal/linux/eal/eal_vfio.c +0 -0
index 01b5ef3..e2fdd35 100644

DPDK supports freebsd where /dev/pci can be probed like Linux /sys/bus/pci. Will you be adding the functions in lib/librte_eal/freebsd/eal/ folder or implement as NOT supported?

Quoted 1 line

What about window lib/librte_eal/windows/eal/?

If other kernel (freebsd & windows) drivers also support this vf token scheme, functions can be added in respective eal implementations.

Quoted 6 lines

snipped

lib/librte_eal/linux/eal/eal_vfio.c

Quoted 2 lines
lib/librte_eal/linux/eal/eal_vfio.c +2 -0
+	snprintf(linkname, sizeof(linkname),
+			 "%s/%s/physfn", sysfs_base, dev_addr);

Would you like to check the pointer sysfs_base or dev_addr?

Physfn pointer is being checked here.

Quoted 10 lines
Quoted 4 lines

int
rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
int *vfio_dev_fd, struct vfio_device_info *device_info) @@ -

Snipped

lib/librte_eal/linux/eal/eal_vfio.c rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr, · line 796

Quoted 1 line
lib/librte_eal/linux/eal/eal_vfio.c +1 -0
+		ret = is_vf_token_required(sysfs_base, dev_addr);

The return value from the above function is 0 or -1.

Function also returns 1.

Quoted 8 lines

lib/librte_eal/linux/eal/eal_vfio.c rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr, · line 801

Quoted 6 lines
lib/librte_eal/linux/eal/eal_vfio.c +5 -0
+		/* if negative, something failed */
+		if (ret < 0)
+			return -1;
+
+		if (ret == 0) {

Will it be ok to skip this check?

No, it's a valid case.

Quoted 11 lines

lib/librte_eal/linux/eal/eal_vfio.c rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr, · line 807

Quoted 7 lines
lib/librte_eal/linux/eal/eal_vfio.c +6 -0
+			/* vf_token required to open device file descriptor */
+			rte_uuid_unparse(uuid_token,
+					 vf_token, sizeof(vf_token));
+			snprintf(dev, sizeof(dev),
+				 "%s vf_token=%s", dev_addr, vf_token);
+		}

snipped

general comment can we use EAL LOG for info, warn, error?

#3 lore raw
In reply to [email protected]

Hi A Vamsi,

Quoted 122 lines

-----Original Message-----
From: dev <[email protected]> On Behalf Of [email protected]
Sent: Thursday, March 5, 2020 12:33
To: [email protected]
Cc: [email protected]; [email protected]; [email protected]; [email protected]; Vamsi Attunuru <[email protected]>
Subject: [dpdk-dev] [RFC v1 1/1] vfio: set vf token and gain vf device access

From: Vamsi Attunuru <[email protected]>

vfio-pci driver enables virtual function access from the DPDK applications when those vf device's physical function is also bound to vfio driver.

Patch adds the required configuration and checks to enable DPDK applications to access both pf and it's vf devices through vfio-pci driver.

See background on vf token scheme in linux vfio driver.
http://patches.dpdk.org/cover/65915/

When a physical function is enabled with non-zero virtual functions, patch sets the UUID using VFIO_DEVICE_FEATURE ioctl from physical function's file descriptor. Same UUID is used to gain the access for the virtual functions on those physical function.

Following changes required on top of this DPDK patch
* Kernel version check for VFIO_DEVICE_FEATURE ioctl
* Use uuid gen API to generate UUID.

lib/librte_eal/linux/eal/eal_vfio.c +67 -0
index 01b5ef3..e2fdd35 100644
@@ -12,6 +12,7 @@
 #include <rte_log.h>
 #include <rte_memory.h>
 #include <rte_eal_memconfig.h>
+#include <rte_uuid.h>
 #include <rte_vfio.h>
 
 #include "eal_filesystem.h"
@@ -50,6 +51,9 @@ struct vfio_config {
 	struct user_mem_maps mem_maps;
 };
 
+rte_uuid_t uuid_token = RTE_UUID_INIT(0xf8615163, 0xdf3e, 0x46c5,
+				      0x913f, 0xf2d2f965ed0eULL);
+
 /* per-process VFIO config */
 static struct vfio_config vfio_cfgs[VFIO_MAX_CONTAINERS];
 static struct vfio_config *default_vfio_cfg = &vfio_cfgs[0];
@@ -657,6 +661,102 @@ rte_vfio_clear_group(int vfio_group_fd)
 	return 0;
 }
 
+static bool
+rte_vfio_dev_is_physfn(const char *sysfs_base, const char *dev_addr)
+{
+	char linkname[PATH_MAX];
+	char filename[PATH_MAX];
+	int ret;
+
+	memset(linkname, 0, sizeof(linkname));
+	memset(filename, 0, sizeof(filename));
+
+	/* check if physfn directory exist for this device */
+	snprintf(linkname, sizeof(linkname),
+			 "%s/%s/physfn", sysfs_base, dev_addr);
+
+	ret = readlink(linkname, filename, sizeof(filename));
+
+	/* For PFs, physfn directory does not exist */
+	if (ret < 0)
+		return true;
+
+	return false;
+}
+
+static int
+is_vf_token_required(const char *sysfs_base, const char *dev_addr)
+{
+	char *tok[16], *physfn, *physfn_drv;
+	char linkname[PATH_MAX];
+	char filename[PATH_MAX];
+	int ret;
+
+	memset(linkname, 0, sizeof(linkname));
+	memset(filename, 0, sizeof(filename));
+
+	snprintf(linkname, sizeof(linkname),
+			 "%s/%s/physfn", sysfs_base, dev_addr);
+
+	ret = readlink(linkname, filename, sizeof(filename));
+	if (ret < 0)
+		return -1;
+
+	ret = rte_strsplit(filename, sizeof(filename),
+			tok, RTE_DIM(tok), '/');
+	if (ret <= 0) {
+		RTE_LOG(ERR, EAL, " %s cannot get it's physfn\n", dev_addr);
+		return -1;
+	}
+
+	physfn = tok[ret - 1];
+
+	snprintf(linkname, sizeof(linkname),
+			 "/sys/bus/pci/devices/%s/driver", physfn);
+	ret = readlink(linkname, filename, sizeof(filename));
+	if (ret < 0)
+		return -1;
+
+	ret = rte_strsplit(filename, sizeof(filename),
+			tok, RTE_DIM(tok), '/');
+	if (ret <= 0) {
+		RTE_LOG(ERR, EAL, "  %s cannot get it's physfn driver info\n",
+			dev_addr);
+		return -1;
+	}

lib/librte_eal/linux/eal/eal_vfio.c rte_vfio_clear_group(int vfio_group_fd) · line 736

Quoted 10 lines
lib/librte_eal/linux/eal/eal_vfio.c +10 -0
+
+	physfn_drv = tok[ret - 1];
+
+	if (strncmp(physfn_drv, "vfio-pci", sizeof("vfio-pci")))
+		return 1;
+
+	/* physfn is bound to vfio-pci */
+	return 0;
+}
+

Based on Alex's vfio for qemu patch (https://lore.kernel.org/lkml/[email protected]/),
and understand the SR-IOV design by reading your RFC, I scratched a simple design.

1. ./usertools/dpdk-devbind.py -b vfio-pci 0000:87:00.0
2. echo 2 > /sys/bus/pci/devices/0000:87:00.0/sriov_numvfs
3. ./x86_64-native-linux-gcc/app/testpmd -l 22-25 -n 4 -w 87:00.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 --file-prefix=pf -- -i

If no token in VF:
./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 -w 87:02.0 --file-prefix=vf1 -- -i
Kernel error: vfio-pci 0000:87:02.0: VF token required to access device

./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 -w 87:02.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 --file-prefix=vf1 -- -i

static int vfio_pci_vf_token_arg_handler(__rte_unused const char *key,
const char *value, void *opaque)

{
	if (rte_uuid_parse(value, opaque)) {
		RTE_LOG(ERR, EAL,
			"The VF token is not a valid uuid : %s\n", value);
		return -1;
	}

	return 0;
}

static int
vfio_pci_vf_token_arg(struct rte_devargs *devargs, rte_uuid_t uu)

{
	const char *key = "vf_token";
	struct rte_kvargs *kvlist;
	int ret = 0;

	if (devargs == NULL)
		return 0;

	kvlist = rte_kvargs_parse(devargs->args, NULL);
	if (kvlist == NULL)
		return 0;

	if (!rte_kvargs_count(kvlist, key))
		goto exit;

	if (rte_kvargs_process(kvlist, key,
			       vfio_pci_vf_token_arg_handler, uu) < 0)
		goto exit;

	ret = 1;

exit:

	rte_kvargs_free(kvlist);
	return ret;
}

static int
pci_vfio_map_resource_primary(struct rte_pci_device *dev)
{
struct vfio_device_info device_info = { .argsz = sizeof(device_info) };
+ rte_uuid_t vf_token = RTE_UUID_INIT(0, 0, 0, 0, 0ULL);
char pci_addr[PATH_MAX] = {0};
int vfio_dev_fd;
struct rte_pci_addr *loc = &dev->addr;
@@ -668,8 +712,9 @@ pci_vfio_map_resource_primary(struct rte_pci_device *dev)
snprintf(pci_addr, sizeof(pci_addr), PCI_PRI_FMT,
loc->domain, loc->bus, loc->devid, loc->function);

+ vfio_pci_vf_token_arg(dev->device.devargs, vf_token);
ret = rte_vfio_setup_device(rte_pci_get_sysfs_path(), pci_addr,
- &vfio_dev_fd, &device_info);
+ &vfio_dev_fd, &device_info, vf_token);
if (ret)
return ret;

int
rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
- int *vfio_dev_fd, struct vfio_device_info *device_info)
+ int *vfio_dev_fd, struct vfio_device_info *device_info,
+ rte_uuid_t vf_token)
{
struct vfio_group_status group_status = {
.argsz = sizeof(group_status)
@@ -712,6 +713,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
int vfio_container_fd;
int vfio_group_fd;
int iommu_group_num;
+ char dev[PATH_MAX];
int i, ret;

/* get group number */
@@ -895,8 +897,19 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
t->type_id, t->name);
}

+ if (!rte_uuid_is_null(vf_token)) {
+ char vf_token_str[PATH_MAX];
+
+ rte_uuid_unparse(vf_token, vf_token_str, sizeof(vf_token_str));
+ snprintf(dev, sizeof(dev),
+ "%s vf_token=%s", dev_addr, vf_token_str);
+ } else {
+ snprintf(dev, sizeof(dev),
+ "%s", dev_addr);
+ }
+
/* get a file descriptor for the device */
- *vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev_addr);
+ *vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev);

Quoted 107 lines
lib/librte_eal/linux/eal/eal_vfio.c +73 -2
+static bool
+rte_vfio_dev_has_nonzero_numvfs(const char *sysfs_base, const char *dev_addr)
+{
+	char linkname[PATH_MAX];
+	unsigned long num_vfs;
+	int ret;
+
+	if (!rte_vfio_dev_is_physfn(sysfs_base, dev_addr))
+		return false;
+
+	memset(linkname, 0, sizeof(linkname));
+
+	snprintf(linkname, sizeof(linkname),
+			 "%s/%s/sriov_numvfs", sysfs_base, dev_addr);
+
+	ret  = eal_parse_sysfs_value(linkname, &num_vfs);
+
+	if ((ret < 0) || (num_vfs == 0))
+		return false;
+
+	return true;
+}
+
 int
 rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 		int *vfio_dev_fd, struct vfio_device_info *device_info)
@@ -669,6 +769,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 	int vfio_container_fd;
 	int vfio_group_fd;
 	int iommu_group_num;
+	char dev[PATH_MAX];
 	int i, ret;
 
 	/* get group number */
@@ -683,6 +784,29 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 	if (ret < 0)
 		return -1;
 
+	snprintf(dev, sizeof(dev), "%s", dev_addr);
+
+	if (!rte_vfio_dev_is_physfn(sysfs_base, dev_addr)) {
+		char vf_token[PATH_MAX];
+		/*
+		 *  Check if vf_token is required or not,
+		 *  vf_token is required when the VF's physfn is
+		 *  binded with vfio-pci driver
+		 */
+		ret = is_vf_token_required(sysfs_base, dev_addr);
+		/* if negative, something failed */
+		if (ret < 0)
+			return -1;
+
+		if (ret == 0) {
+			/* vf_token required to open device file descriptor */
+			rte_uuid_unparse(uuid_token,
+					 vf_token, sizeof(vf_token));
+			snprintf(dev, sizeof(dev),
+				 "%s vf_token=%s", dev_addr, vf_token);
+		}
+	}
+
 	/* get the actual group fd */
 	vfio_group_fd = rte_vfio_get_group_fd(iommu_group_num);
 	if (vfio_group_fd < 0)
@@ -853,7 +977,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 	}
 
 	/* get a file descriptor for the device */
-	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev_addr);
+	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev);
 	if (*vfio_dev_fd < 0) {
 		/* if we cannot get a device fd, this implies a problem with
 		 * the VFIO group or the container not having IOMMU configured.
@@ -877,6 +1001,31 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 		rte_vfio_clear_group(vfio_group_fd);
 		return -1;
 	}
+
+	if (rte_vfio_dev_has_nonzero_numvfs(sysfs_base, dev_addr)) {
+#define VF_TOKEN  (sizeof(struct vfio_device_feature) + sizeof(rte_uuid_t))
+
+		struct vfio_device_feature *vf_token;
+		uint8_t local[VF_TOKEN];
+
+		memset(local, 0, VF_TOKEN);
+		vf_token = (struct vfio_device_feature *)local;
+		vf_token->argsz = VF_TOKEN;
+		vf_token->flags = VFIO_DEVICE_FEATURE_SET |
+					VFIO_DEVICE_FEATURE_PCI_VF_TOKEN;
+
+		memcpy(local + sizeof(struct vfio_device_feature),
+		       &uuid_token, sizeof(uuid_token));
+
+		ret = ioctl(*vfio_dev_fd, VFIO_DEVICE_FEATURE, vf_token);
+		if (ret) {
+			RTE_LOG(ERR, EAL, " Failed to set UUID on %s "
+					"error %i (%s)\n", dev_addr, errno,
+					strerror(errno));
+			return -1;
+		}
+	}
+
 	vfio_group_device_get(vfio_group_fd);
 
 	return 0;
--

2.8.4

#4 lore raw

Hi Wang,

Quoted 153 lines

-----Original Message-----
From: dev <[email protected]> On Behalf Of Wang, Haiyue
Sent: Thursday, April 2, 2020 6:13 PM
To: Vamsi Krishna Attunuru <[email protected]>; [email protected]
Cc: Jerin Jacob Kollanukkaran <[email protected]>; [email protected]; [email protected]; [email protected]
Subject: Re: [dpdk-dev] [RFC v1 1/1] vfio: set vf token and gain vf device access

Hi A Vamsi,

Quoted 16 lines

-----Original Message-----
From: dev <[email protected]> On Behalf Of [email protected]
Sent: Thursday, March 5, 2020 12:33
To: [email protected]
Cc: [email protected]; [email protected]; [email protected]; [email protected]; Vamsi Attunuru <[email protected]>
Subject: [dpdk-dev] [RFC v1 1/1] vfio: set vf token and gain vf device access

From: Vamsi Attunuru <[email protected]>

vfio-pci driver enables virtual function access from the DPDK applications when those vf device's physical function is also bound to vfio driver.

Quoted 7 lines

Patch adds the required configuration and checks to enable DPDK applications to access both pf and it's vf devices through vfio-pci driver.

See background on vf token scheme in linux vfio driver.
https://urldefense.proofpoint.com/v2/url?u=http-3A__patches.dpdk.org_c

over_65915_&d=DwIFAg&c=nKjWec2b6R0mOyPaz7xtfQ&r=2rpxxNF2qeP02g VZIWTVrW

Quoted 1 line

-6zNZz5-

uKt9pRqpR_M3U&m=1z1lwkWphoVghrYUNLI304mER44Vei152GYrXBwn4kk& s=

Quoted 102 lines

tvJ4Li8zpLMbp_p3vKQL7QV09DttNcl70YAr6BVHZm8&e=

When a physical function is enabled with non-zero virtual functions, patch sets the UUID using VFIO_DEVICE_FEATURE ioctl from physical function's file descriptor. Same UUID is used to gain the access for the virtual functions on those physical function.

Following changes required on top of this DPDK patch
* Kernel version check for VFIO_DEVICE_FEATURE ioctl
* Use uuid gen API to generate UUID.

lib/librte_eal/linux/eal/eal_vfio.c +4 -0
index 01b5ef3..e2fdd35 100644
@@ -12,6 +12,7 @@
 #include <rte_log.h>
 #include <rte_memory.h>
 #include <rte_eal_memconfig.h>
+#include <rte_uuid.h>
 #include <rte_vfio.h>
 
 #include "eal_filesystem.h"
@@ -50,6 +51,9 @@ struct vfio_config {
 	struct user_mem_maps mem_maps;
 };
 
+rte_uuid_t uuid_token = RTE_UUID_INIT(0xf8615163, 0xdf3e, 0x46c5,
+				      0x913f, 0xf2d2f965ed0eULL);
+
 /* per-process VFIO config */
 static struct vfio_config vfio_cfgs[VFIO_MAX_CONTAINERS];  static

struct vfio_config *default_vfio_cfg = &vfio_cfgs[0]; @@ -657,6 +661,102 @@ rte_vfio_clear_group(int vfio_group_fd)

lib/librte_eal/linux/eal/eal_vfio.c +62 -0
 	return 0;
 }
 
+static bool
+rte_vfio_dev_is_physfn(const char *sysfs_base, const char *dev_addr)
+{
+	char linkname[PATH_MAX];
+	char filename[PATH_MAX];
+	int ret;
+
+	memset(linkname, 0, sizeof(linkname));
+	memset(filename, 0, sizeof(filename));
+
+	/* check if physfn directory exist for this device */
+	snprintf(linkname, sizeof(linkname),
+			 "%s/%s/physfn", sysfs_base, dev_addr);
+
+	ret = readlink(linkname, filename, sizeof(filename));
+
+	/* For PFs, physfn directory does not exist */
+	if (ret < 0)
+		return true;
+
+	return false;
+}
+
+static int
+is_vf_token_required(const char *sysfs_base, const char *dev_addr) {
+	char *tok[16], *physfn, *physfn_drv;
+	char linkname[PATH_MAX];
+	char filename[PATH_MAX];
+	int ret;
+
+	memset(linkname, 0, sizeof(linkname));
+	memset(filename, 0, sizeof(filename));
+
+	snprintf(linkname, sizeof(linkname),
+			 "%s/%s/physfn", sysfs_base, dev_addr);
+
+	ret = readlink(linkname, filename, sizeof(filename));
+	if (ret < 0)
+		return -1;
+
+	ret = rte_strsplit(filename, sizeof(filename),
+			tok, RTE_DIM(tok), '/');
+	if (ret <= 0) {
+		RTE_LOG(ERR, EAL, " %s cannot get it's physfn\n", dev_addr);
+		return -1;
+	}
+
+	physfn = tok[ret - 1];
+
+	snprintf(linkname, sizeof(linkname),
+			 "/sys/bus/pci/devices/%s/driver", physfn);
+	ret = readlink(linkname, filename, sizeof(filename));
+	if (ret < 0)
+		return -1;
+
+	ret = rte_strsplit(filename, sizeof(filename),
+			tok, RTE_DIM(tok), '/');
+	if (ret <= 0) {
+		RTE_LOG(ERR, EAL, "  %s cannot get it's physfn driver info\n",
+			dev_addr);
+		return -1;
+	}

lib/librte_eal/linux/eal/eal_vfio.c rte_vfio_clear_group(int vfio_group_fd) · line 736

Quoted 10 lines
lib/librte_eal/linux/eal/eal_vfio.c +10 -0
+
+	physfn_drv = tok[ret - 1];
+
+	if (strncmp(physfn_drv, "vfio-pci", sizeof("vfio-pci")))
+		return 1;
+
+	/* physfn is bound to vfio-pci */
+	return 0;
+}
+
Quoted 8 lines

Based on Alex's vfio for qemu patch
(https://urldefense.proofpoint.com/v2/url?u=https-
3A__lore.kernel.org_lkml_20200204161737.34696b91- 40w520.home_&d=DwIFAg&c=nKjWec2b6R0mOyPaz7xtfQ&r=2rpxxNF2qeP0 2gVZIWTVrW-6zNZz5- uKt9pRqpR_M3U&m=1z1lwkWphoVghrYUNLI304mER44Vei152GYrXBwn4kk& s=0pgRda84NrL2omZa0mn-ufSEzKksSi440O2AYImZLaI&e= ), and understand the SR-IOV design by reading your RFC, I scratched a simple design.

Thanks, looks good to me, I guess now it serves the QEMU based use cases as well.

Quoted 237 lines

1. ./usertools/dpdk-devbind.py -b vfio-pci 0000:87:00.0 2. echo 2 > /sys/bus/pci/devices/0000:87:00.0/sriov_numvfs
3. ./x86_64-native-linux-gcc/app/testpmd -l 22-25 -n 4 -w 87:00.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 --file-prefix=pf -- -i

If no token in VF:
./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 -w 87:02.0 --file- prefix=vf1 -- -i
Kernel error: vfio-pci 0000:87:02.0: VF token required to access device

./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 -w 87:02.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 --file-prefix=vf1 -
- -i

static int vfio_pci_vf_token_arg_handler(__rte_unused const char *key,

			      const char *value, void *opaque) {
	if (rte_uuid_parse(value, opaque)) {
		RTE_LOG(ERR, EAL,
			"The VF token is not a valid uuid : %s\n", value);
		return -1;
	}

	return 0;
}

static int

vfio_pci_vf_token_arg(struct rte_devargs *devargs, rte_uuid_t uu) {
	const char *key = "vf_token";
	struct rte_kvargs *kvlist;
	int ret = 0;

	if (devargs == NULL)
		return 0;

	kvlist = rte_kvargs_parse(devargs->args, NULL);
	if (kvlist == NULL)
		return 0;

	if (!rte_kvargs_count(kvlist, key))
		goto exit;

	if (rte_kvargs_process(kvlist, key,
			       vfio_pci_vf_token_arg_handler, uu) < 0)
		goto exit;

	ret = 1;

exit:

	rte_kvargs_free(kvlist);
	return ret;
}

static int
pci_vfio_map_resource_primary(struct rte_pci_device *dev) {

(quoted diff) +1 -0
 	struct vfio_device_info device_info = { .argsz = sizeof(device_info) };
+	rte_uuid_t vf_token = RTE_UUID_INIT(0, 0, 0, 0, 0ULL);
 	char pci_addr[PATH_MAX] = {0};
 	int vfio_dev_fd;
 	struct rte_pci_addr *loc = &dev->addr; @@ -668,8 +712,9 @@

pci_vfio_map_resource_primary(struct rte_pci_device *dev)

(quoted diff) +2 -1
 	snprintf(pci_addr, sizeof(pci_addr), PCI_PRI_FMT,
 			loc->domain, loc->bus, loc->devid, loc->function);
 
+	vfio_pci_vf_token_arg(dev->device.devargs, vf_token);
 	ret = rte_vfio_setup_device(rte_pci_get_sysfs_path(), pci_addr,
-					&vfio_dev_fd, &device_info);
+					&vfio_dev_fd, &device_info,

vf_token);

(quoted diff) +0 -0
 	if (ret)
 		return ret;

int
rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,

lib/librte_eal/linux/eal/eal_vfio.c +3 -1
-		int *vfio_dev_fd, struct vfio_device_info *device_info)
+		int *vfio_dev_fd, struct vfio_device_info *device_info,
+		rte_uuid_t vf_token)
 {
 	struct vfio_group_status group_status = {
 			.argsz = sizeof(group_status)
@@ -712,6 +713,7 @@ rte_vfio_setup_device(const char *sysfs_base, const
 char *dev_addr,
 	int vfio_container_fd;
 	int vfio_group_fd;
 	int iommu_group_num;
+	char dev[PATH_MAX];
 	int i, ret;
 
lib/librte_eal/linux/eal/eal_vfio.c +12 -1
 	/* get group number */
@@ -895,8 +897,19 @@ rte_vfio_setup_device(const char *sysfs_base,
 const char *dev_addr,
 				t->type_id, t->name);
 	}
 
+	if (!rte_uuid_is_null(vf_token)) {
+		char vf_token_str[PATH_MAX];
+
+		rte_uuid_unparse(vf_token, vf_token_str,
 sizeof(vf_token_str));
+		snprintf(dev, sizeof(dev),
+			 "%s vf_token=%s", dev_addr, vf_token_str);
+	} else {
+		snprintf(dev, sizeof(dev),
+			 "%s", dev_addr);
+	}
+
 	/* get a file descriptor for the device */
-	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD,
 dev_addr);
+	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD,

dev);

Quoted 16 lines
lib/librte_eal/linux/eal/eal_vfio.c +16 -0
+static bool
+rte_vfio_dev_has_nonzero_numvfs(const char *sysfs_base, const char
+*dev_addr) {
+	char linkname[PATH_MAX];
+	unsigned long num_vfs;
+	int ret;
+
+	if (!rte_vfio_dev_is_physfn(sysfs_base, dev_addr))
+		return false;
+
+	memset(linkname, 0, sizeof(linkname));
+
+	snprintf(linkname, sizeof(linkname),
+			 "%s/%s/sriov_numvfs", sysfs_base, dev_addr);
+
+	ret  = eal_parse_sysfs_value(linkname, &num_vfs);

lib/librte_eal/linux/eal/eal_vfio.c rte_vfio_clear_group(int vfio_group_fd)

Quoted 10 lines
lib/librte_eal/linux/eal/eal_vfio.c +7 -0
+
+	if ((ret < 0) || (num_vfs == 0))
+		return false;
+
+	return true;
+}
+
 int
 rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 		int *vfio_dev_fd, struct vfio_device_info *device_info) @@ -

669,6

Quoted 1 line

+769,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char

*dev_addr,

lib/librte_eal/linux/eal/eal_vfio.c rte_vfio_setup_device(const char *sysfs_base,

Quoted 8 lines
lib/librte_eal/linux/eal/eal_vfio.c +1 -0
 	int vfio_container_fd;
 	int vfio_group_fd;
 	int iommu_group_num;
+	char dev[PATH_MAX];
 	int i, ret;
 
 	/* get group number */
@@ -683,6 +784,29 @@ rte_vfio_setup_device(const char *sysfs_base,

const char *dev_addr,

lib/librte_eal/linux/eal/eal_vfio.c rte_vfio_setup_device(const char *sysfs_base,

Quoted 30 lines
lib/librte_eal/linux/eal/eal_vfio.c +23 -0
 	if (ret < 0)
 		return -1;
 
+	snprintf(dev, sizeof(dev), "%s", dev_addr);
+
+	if (!rte_vfio_dev_is_physfn(sysfs_base, dev_addr)) {
+		char vf_token[PATH_MAX];
+		/*
+		 *  Check if vf_token is required or not,
+		 *  vf_token is required when the VF's physfn is
+		 *  binded with vfio-pci driver
+		 */
+		ret = is_vf_token_required(sysfs_base, dev_addr);
+		/* if negative, something failed */
+		if (ret < 0)
+			return -1;
+
+		if (ret == 0) {
+			/* vf_token required to open device file descriptor */
+			rte_uuid_unparse(uuid_token,
+					 vf_token, sizeof(vf_token));
+			snprintf(dev, sizeof(dev),
+				 "%s vf_token=%s", dev_addr, vf_token);
+		}
+	}
+
 	/* get the actual group fd */
 	vfio_group_fd = rte_vfio_get_group_fd(iommu_group_num);
 	if (vfio_group_fd < 0)
@@ -853,7 +977,7 @@ rte_vfio_setup_device(const char *sysfs_base,

const char *dev_addr,

lib/librte_eal/linux/eal/eal_vfio.c rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,

Quoted 4 lines
lib/librte_eal/linux/eal/eal_vfio.c +0 -1
 	}
 
 	/* get a file descriptor for the device */
-	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD,

dev_addr);

lib/librte_eal/linux/eal/eal_vfio.c rte_vfio_setup_device(const char *sysfs_base, · line 980

Quoted 1 line
lib/librte_eal/linux/eal/eal_vfio.c +1 -0
+	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD,

dev);

lib/librte_eal/linux/eal/eal_vfio.c rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,

Quoted 3 lines
lib/librte_eal/linux/eal/eal_vfio.c +0 -0
 	if (*vfio_dev_fd < 0) {
 		/* if we cannot get a device fd, this implies a problem with
 		 * the VFIO group or the container not having IOMMU

configured.

lib/librte_eal/linux/eal/eal_vfio.c rte_vfio_setup_device(const char *sysfs_base,

Quoted 1 line
lib/librte_eal/linux/eal/eal_vfio.c +0 -0
@@ -877,6 +1001,31 @@ rte_vfio_setup_device(const char *sysfs_base,

const char *dev_addr,

lib/librte_eal/linux/eal/eal_vfio.c rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr, · line 1005

Quoted 5 lines
lib/librte_eal/linux/eal/eal_vfio.c +2 -0
 		rte_vfio_clear_group(vfio_group_fd);
 		return -1;
 	}
+
+	if (rte_vfio_dev_has_nonzero_numvfs(sysfs_base, dev_addr)) {

#define

lib/librte_eal/linux/eal/eal_vfio.c rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr, · line 1016

Quoted 10 lines
lib/librte_eal/linux/eal/eal_vfio.c +10 -0
+VF_TOKEN  (sizeof(struct vfio_device_feature) + sizeof(rte_uuid_t))
+
+		struct vfio_device_feature *vf_token;
+		uint8_t local[VF_TOKEN];
+
+		memset(local, 0, VF_TOKEN);
+		vf_token = (struct vfio_device_feature *)local;
+		vf_token->argsz = VF_TOKEN;
+		vf_token->flags = VFIO_DEVICE_FEATURE_SET |
+
	VFIO_DEVICE_FEATURE_PCI_VF_TOKEN;
Quoted 18 lines
lib/librte_eal/linux/eal/eal_vfio.c +13 -1
+
+		memcpy(local + sizeof(struct vfio_device_feature),
+		       &uuid_token, sizeof(uuid_token));
+
+		ret = ioctl(*vfio_dev_fd, VFIO_DEVICE_FEATURE, vf_token);
+		if (ret) {
+			RTE_LOG(ERR, EAL, " Failed to set UUID on %s "
+					"error %i (%s)\n", dev_addr, errno,
+					strerror(errno));
+			return -1;
+		}
+	}
+
 	vfio_group_device_get(vfio_group_fd);
 
 	return 0;
--

2.8.4

#5 lore raw

Hi Vamsi,

Fixed one dev_args issue, and send the whole patch to express the whole design:
https://patchwork.dpdk.org/patch/68114/

BR,
Haiyue

Quoted 410 lines

-----Original Message-----
From: Vamsi Krishna Attunuru <[email protected]>
Sent: Friday, April 10, 2020 15:28
To: Wang, Haiyue <[email protected]>; [email protected]
Cc: Jerin Jacob Kollanukkaran <[email protected]>; [email protected]; [email protected]; [email protected]
Subject: RE: [dpdk-dev] [RFC v1 1/1] vfio: set vf token and gain vf device access

Hi Wang,

Quoted 153 lines

-----Original Message-----
From: dev <[email protected]> On Behalf Of Wang, Haiyue
Sent: Thursday, April 2, 2020 6:13 PM
To: Vamsi Krishna Attunuru <[email protected]>; [email protected]
Cc: Jerin Jacob Kollanukkaran <[email protected]>; [email protected]; [email protected]; [email protected]
Subject: Re: [dpdk-dev] [RFC v1 1/1] vfio: set vf token and gain vf device access

Hi A Vamsi,

Quoted 16 lines

-----Original Message-----
From: dev <[email protected]> On Behalf Of [email protected]
Sent: Thursday, March 5, 2020 12:33
To: [email protected]
Cc: [email protected]; [email protected]; [email protected]; [email protected]; Vamsi Attunuru <[email protected]>
Subject: [dpdk-dev] [RFC v1 1/1] vfio: set vf token and gain vf device access

From: Vamsi Attunuru <[email protected]>

vfio-pci driver enables virtual function access from the DPDK applications when those vf device's physical function is also bound to vfio driver.

Quoted 7 lines

Patch adds the required configuration and checks to enable DPDK applications to access both pf and it's vf devices through vfio-pci driver.

See background on vf token scheme in linux vfio driver.
https://urldefense.proofpoint.com/v2/url?u=http-3A__patches.dpdk.org_c

over_65915_&d=DwIFAg&c=nKjWec2b6R0mOyPaz7xtfQ&r=2rpxxNF2qeP02g VZIWTVrW

Quoted 1 line

-6zNZz5-

uKt9pRqpR_M3U&m=1z1lwkWphoVghrYUNLI304mER44Vei152GYrXBwn4kk& s=

Quoted 102 lines

tvJ4Li8zpLMbp_p3vKQL7QV09DttNcl70YAr6BVHZm8&e=

When a physical function is enabled with non-zero virtual functions, patch sets the UUID using VFIO_DEVICE_FEATURE ioctl from physical function's file descriptor. Same UUID is used to gain the access for the virtual functions on those physical function.

Following changes required on top of this DPDK patch
* Kernel version check for VFIO_DEVICE_FEATURE ioctl
* Use uuid gen API to generate UUID.

lib/librte_eal/linux/eal/eal_vfio.c +4 -0
index 01b5ef3..e2fdd35 100644
@@ -12,6 +12,7 @@
 #include <rte_log.h>
 #include <rte_memory.h>
 #include <rte_eal_memconfig.h>
+#include <rte_uuid.h>
 #include <rte_vfio.h>
 
 #include "eal_filesystem.h"
@@ -50,6 +51,9 @@ struct vfio_config {
 	struct user_mem_maps mem_maps;
 };
 
+rte_uuid_t uuid_token = RTE_UUID_INIT(0xf8615163, 0xdf3e, 0x46c5,
+				      0x913f, 0xf2d2f965ed0eULL);
+
 /* per-process VFIO config */
 static struct vfio_config vfio_cfgs[VFIO_MAX_CONTAINERS];  static

struct vfio_config *default_vfio_cfg = &vfio_cfgs[0]; @@ -657,6 +661,102 @@ rte_vfio_clear_group(int vfio_group_fd)

lib/librte_eal/linux/eal/eal_vfio.c +62 -0
 	return 0;
 }
 
+static bool
+rte_vfio_dev_is_physfn(const char *sysfs_base, const char *dev_addr)
+{
+	char linkname[PATH_MAX];
+	char filename[PATH_MAX];
+	int ret;
+
+	memset(linkname, 0, sizeof(linkname));
+	memset(filename, 0, sizeof(filename));
+
+	/* check if physfn directory exist for this device */
+	snprintf(linkname, sizeof(linkname),
+			 "%s/%s/physfn", sysfs_base, dev_addr);
+
+	ret = readlink(linkname, filename, sizeof(filename));
+
+	/* For PFs, physfn directory does not exist */
+	if (ret < 0)
+		return true;
+
+	return false;
+}
+
+static int
+is_vf_token_required(const char *sysfs_base, const char *dev_addr) {
+	char *tok[16], *physfn, *physfn_drv;
+	char linkname[PATH_MAX];
+	char filename[PATH_MAX];
+	int ret;
+
+	memset(linkname, 0, sizeof(linkname));
+	memset(filename, 0, sizeof(filename));
+
+	snprintf(linkname, sizeof(linkname),
+			 "%s/%s/physfn", sysfs_base, dev_addr);
+
+	ret = readlink(linkname, filename, sizeof(filename));
+	if (ret < 0)
+		return -1;
+
+	ret = rte_strsplit(filename, sizeof(filename),
+			tok, RTE_DIM(tok), '/');
+	if (ret <= 0) {
+		RTE_LOG(ERR, EAL, " %s cannot get it's physfn\n", dev_addr);
+		return -1;
+	}
+
+	physfn = tok[ret - 1];
+
+	snprintf(linkname, sizeof(linkname),
+			 "/sys/bus/pci/devices/%s/driver", physfn);
+	ret = readlink(linkname, filename, sizeof(filename));
+	if (ret < 0)
+		return -1;
+
+	ret = rte_strsplit(filename, sizeof(filename),
+			tok, RTE_DIM(tok), '/');
+	if (ret <= 0) {
+		RTE_LOG(ERR, EAL, "  %s cannot get it's physfn driver info\n",
+			dev_addr);
+		return -1;
+	}

lib/librte_eal/linux/eal/eal_vfio.c rte_vfio_clear_group(int vfio_group_fd) · line 736

Quoted 10 lines
lib/librte_eal/linux/eal/eal_vfio.c +10 -0
+
+	physfn_drv = tok[ret - 1];
+
+	if (strncmp(physfn_drv, "vfio-pci", sizeof("vfio-pci")))
+		return 1;
+
+	/* physfn is bound to vfio-pci */
+	return 0;
+}
+
Quoted 8 lines

Based on Alex's vfio for qemu patch
(https://urldefense.proofpoint.com/v2/url?u=https-
3A__lore.kernel.org_lkml_20200204161737.34696b91- 40w520.home_&d=DwIFAg&c=nKjWec2b6R0mOyPaz7xtfQ&r=2rpxxNF2qeP0 2gVZIWTVrW-6zNZz5- uKt9pRqpR_M3U&m=1z1lwkWphoVghrYUNLI304mER44Vei152GYrXBwn4kk& s=0pgRda84NrL2omZa0mn-ufSEzKksSi440O2AYImZLaI&e= ), and understand the SR-IOV design by reading your RFC, I scratched a simple design.

Thanks, looks good to me, I guess now it serves the QEMU based use cases as well.

Quoted 237 lines

1. ./usertools/dpdk-devbind.py -b vfio-pci 0000:87:00.0 2. echo 2 > /sys/bus/pci/devices/0000:87:00.0/sriov_numvfs
3. ./x86_64-native-linux-gcc/app/testpmd -l 22-25 -n 4 -w 87:00.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 --file-prefix=pf -- -i

If no token in VF:
./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 -w 87:02.0 --file- prefix=vf1 -- -i
Kernel error: vfio-pci 0000:87:02.0: VF token required to access device

./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 -w 87:02.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 --file-prefix=vf1 -
- -i

static int vfio_pci_vf_token_arg_handler(__rte_unused const char *key,

			      const char *value, void *opaque) {
	if (rte_uuid_parse(value, opaque)) {
		RTE_LOG(ERR, EAL,
			"The VF token is not a valid uuid : %s\n", value);
		return -1;
	}

	return 0;
}

static int

vfio_pci_vf_token_arg(struct rte_devargs *devargs, rte_uuid_t uu) {
	const char *key = "vf_token";
	struct rte_kvargs *kvlist;
	int ret = 0;

	if (devargs == NULL)
		return 0;

	kvlist = rte_kvargs_parse(devargs->args, NULL);
	if (kvlist == NULL)
		return 0;

	if (!rte_kvargs_count(kvlist, key))
		goto exit;

	if (rte_kvargs_process(kvlist, key,
			       vfio_pci_vf_token_arg_handler, uu) < 0)
		goto exit;

	ret = 1;

exit:

	rte_kvargs_free(kvlist);
	return ret;
}

static int
pci_vfio_map_resource_primary(struct rte_pci_device *dev) {

(quoted diff) +1 -0
 	struct vfio_device_info device_info = { .argsz = sizeof(device_info) };
+	rte_uuid_t vf_token = RTE_UUID_INIT(0, 0, 0, 0, 0ULL);
 	char pci_addr[PATH_MAX] = {0};
 	int vfio_dev_fd;
 	struct rte_pci_addr *loc = &dev->addr; @@ -668,8 +712,9 @@

pci_vfio_map_resource_primary(struct rte_pci_device *dev)

(quoted diff) +2 -1
 	snprintf(pci_addr, sizeof(pci_addr), PCI_PRI_FMT,
 			loc->domain, loc->bus, loc->devid, loc->function);
 
+	vfio_pci_vf_token_arg(dev->device.devargs, vf_token);
 	ret = rte_vfio_setup_device(rte_pci_get_sysfs_path(), pci_addr,
-					&vfio_dev_fd, &device_info);
+					&vfio_dev_fd, &device_info,

vf_token);

(quoted diff) +0 -0
 	if (ret)
 		return ret;

int
rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,

lib/librte_eal/linux/eal/eal_vfio.c +3 -1
-		int *vfio_dev_fd, struct vfio_device_info *device_info)
+		int *vfio_dev_fd, struct vfio_device_info *device_info,
+		rte_uuid_t vf_token)
 {
 	struct vfio_group_status group_status = {
 			.argsz = sizeof(group_status)
@@ -712,6 +713,7 @@ rte_vfio_setup_device(const char *sysfs_base, const
 char *dev_addr,
 	int vfio_container_fd;
 	int vfio_group_fd;
 	int iommu_group_num;
+	char dev[PATH_MAX];
 	int i, ret;
 
lib/librte_eal/linux/eal/eal_vfio.c +12 -1
 	/* get group number */
@@ -895,8 +897,19 @@ rte_vfio_setup_device(const char *sysfs_base,
 const char *dev_addr,
 				t->type_id, t->name);
 	}
 
+	if (!rte_uuid_is_null(vf_token)) {
+		char vf_token_str[PATH_MAX];
+
+		rte_uuid_unparse(vf_token, vf_token_str,
 sizeof(vf_token_str));
+		snprintf(dev, sizeof(dev),
+			 "%s vf_token=%s", dev_addr, vf_token_str);
+	} else {
+		snprintf(dev, sizeof(dev),
+			 "%s", dev_addr);
+	}
+
 	/* get a file descriptor for the device */
-	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD,
 dev_addr);
+	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD,

dev);

Quoted 16 lines
lib/librte_eal/linux/eal/eal_vfio.c +16 -0
+static bool
+rte_vfio_dev_has_nonzero_numvfs(const char *sysfs_base, const char
+*dev_addr) {
+	char linkname[PATH_MAX];
+	unsigned long num_vfs;
+	int ret;
+
+	if (!rte_vfio_dev_is_physfn(sysfs_base, dev_addr))
+		return false;
+
+	memset(linkname, 0, sizeof(linkname));
+
+	snprintf(linkname, sizeof(linkname),
+			 "%s/%s/sriov_numvfs", sysfs_base, dev_addr);
+
+	ret  = eal_parse_sysfs_value(linkname, &num_vfs);

lib/librte_eal/linux/eal/eal_vfio.c rte_vfio_clear_group(int vfio_group_fd)

Quoted 10 lines
lib/librte_eal/linux/eal/eal_vfio.c +7 -0
+
+	if ((ret < 0) || (num_vfs == 0))
+		return false;
+
+	return true;
+}
+
 int
 rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 		int *vfio_dev_fd, struct vfio_device_info *device_info) @@ -

669,6

Quoted 1 line

+769,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char

*dev_addr,

lib/librte_eal/linux/eal/eal_vfio.c rte_vfio_setup_device(const char *sysfs_base,

Quoted 8 lines
lib/librte_eal/linux/eal/eal_vfio.c +1 -0
 	int vfio_container_fd;
 	int vfio_group_fd;
 	int iommu_group_num;
+	char dev[PATH_MAX];
 	int i, ret;
 
 	/* get group number */
@@ -683,6 +784,29 @@ rte_vfio_setup_device(const char *sysfs_base,

const char *dev_addr,

lib/librte_eal/linux/eal/eal_vfio.c rte_vfio_setup_device(const char *sysfs_base,

Quoted 30 lines
lib/librte_eal/linux/eal/eal_vfio.c +23 -0
 	if (ret < 0)
 		return -1;
 
+	snprintf(dev, sizeof(dev), "%s", dev_addr);
+
+	if (!rte_vfio_dev_is_physfn(sysfs_base, dev_addr)) {
+		char vf_token[PATH_MAX];
+		/*
+		 *  Check if vf_token is required or not,
+		 *  vf_token is required when the VF's physfn is
+		 *  binded with vfio-pci driver
+		 */
+		ret = is_vf_token_required(sysfs_base, dev_addr);
+		/* if negative, something failed */
+		if (ret < 0)
+			return -1;
+
+		if (ret == 0) {
+			/* vf_token required to open device file descriptor */
+			rte_uuid_unparse(uuid_token,
+					 vf_token, sizeof(vf_token));
+			snprintf(dev, sizeof(dev),
+				 "%s vf_token=%s", dev_addr, vf_token);
+		}
+	}
+
 	/* get the actual group fd */
 	vfio_group_fd = rte_vfio_get_group_fd(iommu_group_num);
 	if (vfio_group_fd < 0)
@@ -853,7 +977,7 @@ rte_vfio_setup_device(const char *sysfs_base,

const char *dev_addr,

lib/librte_eal/linux/eal/eal_vfio.c rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,

Quoted 4 lines
lib/librte_eal/linux/eal/eal_vfio.c +0 -1
 	}
 
 	/* get a file descriptor for the device */
-	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD,

dev_addr);

lib/librte_eal/linux/eal/eal_vfio.c rte_vfio_setup_device(const char *sysfs_base, · line 980

Quoted 1 line
lib/librte_eal/linux/eal/eal_vfio.c +1 -0
+	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD,

dev);

lib/librte_eal/linux/eal/eal_vfio.c rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,

Quoted 3 lines
lib/librte_eal/linux/eal/eal_vfio.c +0 -0
 	if (*vfio_dev_fd < 0) {
 		/* if we cannot get a device fd, this implies a problem with
 		 * the VFIO group or the container not having IOMMU

configured.

lib/librte_eal/linux/eal/eal_vfio.c rte_vfio_setup_device(const char *sysfs_base,

Quoted 1 line
lib/librte_eal/linux/eal/eal_vfio.c +0 -0
@@ -877,6 +1001,31 @@ rte_vfio_setup_device(const char *sysfs_base,

const char *dev_addr,

lib/librte_eal/linux/eal/eal_vfio.c rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr, · line 1005

Quoted 5 lines
lib/librte_eal/linux/eal/eal_vfio.c +2 -0
 		rte_vfio_clear_group(vfio_group_fd);
 		return -1;
 	}
+
+	if (rte_vfio_dev_has_nonzero_numvfs(sysfs_base, dev_addr)) {

#define

lib/librte_eal/linux/eal/eal_vfio.c rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr, · line 1016

Quoted 10 lines
lib/librte_eal/linux/eal/eal_vfio.c +10 -0
+VF_TOKEN  (sizeof(struct vfio_device_feature) + sizeof(rte_uuid_t))
+
+		struct vfio_device_feature *vf_token;
+		uint8_t local[VF_TOKEN];
+
+		memset(local, 0, VF_TOKEN);
+		vf_token = (struct vfio_device_feature *)local;
+		vf_token->argsz = VF_TOKEN;
+		vf_token->flags = VFIO_DEVICE_FEATURE_SET |
+
	VFIO_DEVICE_FEATURE_PCI_VF_TOKEN;
Quoted 18 lines
lib/librte_eal/linux/eal/eal_vfio.c +13 -1
+
+		memcpy(local + sizeof(struct vfio_device_feature),
+		       &uuid_token, sizeof(uuid_token));
+
+		ret = ioctl(*vfio_dev_fd, VFIO_DEVICE_FEATURE, vf_token);
+		if (ret) {
+			RTE_LOG(ERR, EAL, " Failed to set UUID on %s "
+					"error %i (%s)\n", dev_addr, errno,
+					strerror(errno));
+			return -1;
+		}
+	}
+
 	vfio_group_device_get(vfio_group_fd);
 
 	return 0;
--

2.8.4

#6 lore raw

RFC v2 1/1 eal: add VFIO-PCI SR-IOV support

In reply to [email protected]

The kernel module vfio-pci introduces the VF token to enable SR-IOV support.

The VF token can be set by a vfio-pci based PF driver and must be known by the vfio-pci based VF driver in order to gain access to the device.

An example VF token option would take this form:

1. ./usertools/dpdk-devbind.py -b vfio-pci 0000:87:00.0

2. echo 2 > /sys/bus/pci/devices/0000:87:00.0/sriov_numvfs

3. Start the PF:

  ./x86_64-native-linux-gcc/app/testpmd -l 22-25 -n 4 \
         -w 87:00.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 \
         --file-prefix=pf -- -i

4. Start the VF:

   ./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 \
         -w 87:02.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 \
         --file-prefix=vf1 -- -i

Test based on vfio-pci patch: https://patchwork.ozlabs.org/cover/1253222/

---
Based on RFC v1: https://patchwork.dpdk.org/patch/66281/

---
 drivers/bus/pci/linux/pci_vfio.c  | 56 +++++++++++++++++++++++++++++--
 lib/librte_eal/freebsd/eal.c      |  3 +-
 lib/librte_eal/include/rte_vfio.h |  8 ++++-
 lib/librte_eal/linux/eal_vfio.c   | 20 +++++++++--
 4 files changed, 80 insertions(+), 7 deletions(-)
drivers/bus/pci/linux/pci_vfio.c +54 -2
index 64cd84a68..7f99337c7 100644
@@ -11,6 +11,7 @@
 #include <sys/mman.h>
 #include <stdbool.h>
 
+#include <rte_devargs.h>
 #include <rte_log.h>
 #include <rte_pci.h>
 #include <rte_bus_pci.h>
@@ -644,11 +645,59 @@ pci_vfio_msix_is_mappable(int vfio_dev_fd, int msix_region)
 	return ret;
 }
 
+static void
+vfio_pci_vf_token_arg(struct rte_devargs *devargs, rte_uuid_t uu)
+{
+#define VF_TOKEN_ARG "vf_token="
+	char c, *p, *vf_token;
+
+	if (devargs == NULL)
+		return;
+
+	p = strstr(devargs->args, VF_TOKEN_ARG);
+	if (!p)
+		return;
+
+	vf_token = p + strlen(VF_TOKEN_ARG);
+	if (strlen(vf_token) < (RTE_UUID_STRLEN - 1))
+		return;
+
+	c = vf_token[RTE_UUID_STRLEN - 1];
+	if (c != '\0' && c != ',')
+		return;
+
+	vf_token[RTE_UUID_STRLEN - 1] = '\0';
+	if (rte_uuid_parse(vf_token, uu)) {
+		RTE_LOG(ERR, EAL,
+			"The VF token is not a valid uuid : %s\n", vf_token);
+		vf_token[RTE_UUID_STRLEN - 1] = c;
+		return;
+	}
+
+	RTE_LOG(DEBUG, EAL,
+		"The VF token is found : %s\n", vf_token);
+
+	vf_token[RTE_UUID_STRLEN - 1] = c;
+
+	/* Purge this vfio-pci specific token from the device arguments */
+	if (c != '\0') {
+		/* 1. Handle the case : 'vf_token=uuid,arg1=val1' */
+		memmove(p, vf_token + RTE_UUID_STRLEN,
+			strlen(vf_token + RTE_UUID_STRLEN) + 1);
+	} else {
+		/* 2. Handle the case : 'arg1=val1,vf_token=uuid' */
+		if (p != devargs->args)
+			p--;
+
+		*p = '\0';
+	}
+}
 
 static int
 pci_vfio_map_resource_primary(struct rte_pci_device *dev)
 {
 	struct vfio_device_info device_info = { .argsz = sizeof(device_info) };
+	rte_uuid_t vf_token = RTE_UUID_INIT(0, 0, 0, 0, 0ULL);
 	char pci_addr[PATH_MAX] = {0};
 	int vfio_dev_fd;
 	struct rte_pci_addr *loc = &dev->addr;
@@ -668,8 +717,9 @@ pci_vfio_map_resource_primary(struct rte_pci_device *dev)
 	snprintf(pci_addr, sizeof(pci_addr), PCI_PRI_FMT,
 			loc->domain, loc->bus, loc->devid, loc->function);
 
+	vfio_pci_vf_token_arg(dev->device.devargs, vf_token);
 	ret = rte_vfio_setup_device(rte_pci_get_sysfs_path(), pci_addr,
-					&vfio_dev_fd, &device_info);
+					&vfio_dev_fd, &device_info, vf_token);
 	if (ret)
 		return ret;
 
@@ -797,6 +847,7 @@ static int
 pci_vfio_map_resource_secondary(struct rte_pci_device *dev)
 {
 	struct vfio_device_info device_info = { .argsz = sizeof(device_info) };
+	rte_uuid_t vf_token = RTE_UUID_INIT(0, 0, 0, 0, 0ULL);
 	char pci_addr[PATH_MAX] = {0};
 	int vfio_dev_fd;
 	struct rte_pci_addr *loc = &dev->addr;
@@ -830,8 +881,9 @@ pci_vfio_map_resource_secondary(struct rte_pci_device *dev)
 		return -1;
 	}
 
+	vfio_pci_vf_token_arg(dev->device.devargs, vf_token);
 	ret = rte_vfio_setup_device(rte_pci_get_sysfs_path(), pci_addr,
-					&vfio_dev_fd, &device_info);
+					&vfio_dev_fd, &device_info, vf_token);
 	if (ret)
 		return ret;
 
lib/librte_eal/freebsd/eal.c +2 -1
index 6ae37e7e6..cbb4c400e 100644
@@ -995,7 +995,8 @@ rte_eal_vfio_intr_mode(void)
 int rte_vfio_setup_device(__rte_unused const char *sysfs_base,
 		      __rte_unused const char *dev_addr,
 		      __rte_unused int *vfio_dev_fd,
-		      __rte_unused struct vfio_device_info *device_info)
+		      __rte_unused struct vfio_device_info *device_info
+		      __rte_unused rte_uuid_t vf_token)
 {
 	return -1;
 }
lib/librte_eal/include/rte_vfio.h +7 -1
index 20ed8c45a..1f9e22d82 100644
@@ -16,6 +16,8 @@ extern "C" {
 
 #include <stdint.h>
 
+#include <rte_uuid.h>
+
 /*
  * determine if VFIO is present on the system
  */
@@ -102,13 +104,17 @@ struct vfio_device_info;
  * @param device_info
  *   Device information.
  *
+ * @param vf_token
+ *   VF token.
+ *
  * @return
  *   0 on success.
  *   <0 on failure.
  *   >1 if the device cannot be managed this way.
  */
 int rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
-		int *vfio_dev_fd, struct vfio_device_info *device_info);
+		int *vfio_dev_fd, struct vfio_device_info *device_info,
+		rte_uuid_t vf_token);
 
 /**
  * Release a device mapped to a VFIO-managed I/O MMU group.
lib/librte_eal/linux/eal_vfio.c +17 -3
index 4502aefed..8c0ad04e5 100644
@@ -702,7 +702,8 @@ rte_vfio_clear_group(int vfio_group_fd)
 
 int
 rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
-		int *vfio_dev_fd, struct vfio_device_info *device_info)
+		int *vfio_dev_fd, struct vfio_device_info *device_info,
+		rte_uuid_t vf_token)
 {
 	struct vfio_group_status group_status = {
 			.argsz = sizeof(group_status)
@@ -712,6 +713,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 	int vfio_container_fd;
 	int vfio_group_fd;
 	int iommu_group_num;
+	char dev[PATH_MAX];
 	int i, ret;
 
 	/* get group number */
@@ -895,8 +897,19 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 				t->type_id, t->name);
 	}
 
+	if (!rte_uuid_is_null(vf_token)) {
+		char vf_token_str[RTE_UUID_STRLEN];
+
+		rte_uuid_unparse(vf_token, vf_token_str, sizeof(vf_token_str));
+		snprintf(dev, sizeof(dev),
+			 "%s vf_token=%s", dev_addr, vf_token_str);
+	} else {
+		snprintf(dev, sizeof(dev),
+			 "%s", dev_addr);
+	}
+
 	/* get a file descriptor for the device */
-	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev_addr);
+	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev);
 	if (*vfio_dev_fd < 0) {
 		/* if we cannot get a device fd, this implies a problem with
 		 * the VFIO group or the container not having IOMMU configured.
@@ -2081,7 +2094,8 @@ int
 rte_vfio_setup_device(__rte_unused const char *sysfs_base,
 		__rte_unused const char *dev_addr,
 		__rte_unused int *vfio_dev_fd,
-		__rte_unused struct vfio_device_info *device_info)
+		__rte_unused struct vfio_device_info *device_info
+		__rte_unused rte_uuid_t vf_token)
 {
 	return -1;
 }
Signature · 1 line
2.26.0
author Acked-by on v2 1/1
#7 lore raw

Re: [dpdk-dev] [EXT] [RFC v2] eal: add VFIO-PCI SR-IOV support

Quoted 265 lines

-----Original Message-----
From: Haiyue Wang <[email protected]>
Sent: Friday, April 10, 2020 1:03 PM
To: [email protected]; [email protected]; Vamsi Krishna Attunuru <[email protected]>; Jerin Jacob Kollanukkaran <[email protected]>; [email protected]; [email protected]
Cc: Haiyue Wang <[email protected]>
Subject: [EXT] [RFC v2] eal: add VFIO-PCI SR-IOV support

External Email

----------------------------------------------------------------------
The kernel module vfio-pci introduces the VF token to enable SR-IOV support.

The VF token can be set by a vfio-pci based PF driver and must be known by the vfio-pci based VF driver in order to gain access to the device.

An example VF token option would take this form:

1. ./usertools/dpdk-devbind.py -b vfio-pci 0000:87:00.0

2. echo 2 > /sys/bus/pci/devices/0000:87:00.0/sriov_numvfs

3. Start the PF:

  ./x86_64-native-linux-gcc/app/testpmd -l 22-25 -n 4 \
         -w 87:00.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 \
         --file-prefix=pf -- -i

4. Start the VF:

   ./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 \
         -w 87:02.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 \
         --file-prefix=vf1 -- -i

Test based on vfio-pci patch:
https://urldefense.proofpoint.com/v2/url?u=https-
3A__patchwork.ozlabs.org_cover_1253222_&d=DwIDAg&c=nKjWec2b6R0m OyPaz7xtfQ&r=WllrYaumVkxaWjgKto6E_rtDQshhIhik2jkvzFyRhW8&m=SFD0o TfFgnU88wT2X7qMWRDen3KFV8oSOlqz3WQm3nI&s=CU2jxKUsy4oiI26apob DF8BJebDrheBiu3eKlF3e76E&e=

---
Based on RFC v1: https://urldefense.proofpoint.com/v2/url?u=https-
3A__patchwork.dpdk.org_patch_66281_&d=DwIDAg&c=nKjWec2b6R0mOyP az7xtfQ&r=WllrYaumVkxaWjgKto6E_rtDQshhIhik2jkvzFyRhW8&m=SFD0oTfFg nU88wT2X7qMWRDen3KFV8oSOlqz3WQm3nI&s=3dHnLcXpkRVOZs6wdZQylc Je0oNNBs77hXE6SOuLO7o&e=

---
 drivers/bus/pci/linux/pci_vfio.c  | 56 +++++++++++++++++++++++++++++--
 lib/librte_eal/freebsd/eal.c      |  3 +-
 lib/librte_eal/include/rte_vfio.h |  8 ++++-
 lib/librte_eal/linux/eal_vfio.c   | 20 +++++++++--
 4 files changed, 80 insertions(+), 7 deletions(-)
drivers/bus/pci/linux/pci_vfio.c +48 -0
index 64cd84a68..7f99337c7 100644
@@ -11,6 +11,7 @@
 #include <sys/mman.h>
 #include <stdbool.h>
 
+#include <rte_devargs.h>
 #include <rte_log.h>
 #include <rte_pci.h>
 #include <rte_bus_pci.h>
@@ -644,11 +645,59 @@ pci_vfio_msix_is_mappable(int vfio_dev_fd, int
 msix_region)
 	return ret;
 }
 
+static void
+vfio_pci_vf_token_arg(struct rte_devargs *devargs, rte_uuid_t uu) {
+#define VF_TOKEN_ARG "vf_token="
+	char c, *p, *vf_token;
+
+	if (devargs == NULL)
+		return;
+
+	p = strstr(devargs->args, VF_TOKEN_ARG);
+	if (!p)
+		return;
+
+	vf_token = p + strlen(VF_TOKEN_ARG);
+	if (strlen(vf_token) < (RTE_UUID_STRLEN - 1))
+		return;
+
+	c = vf_token[RTE_UUID_STRLEN - 1];
+	if (c != '\0' && c != ',')
+		return;
+
+	vf_token[RTE_UUID_STRLEN - 1] = '\0';
+	if (rte_uuid_parse(vf_token, uu)) {
+		RTE_LOG(ERR, EAL,
+			"The VF token is not a valid uuid : %s\n", vf_token);
+		vf_token[RTE_UUID_STRLEN - 1] = c;
+		return;
+	}
+
+	RTE_LOG(DEBUG, EAL,
+		"The VF token is found : %s\n", vf_token);
+
+	vf_token[RTE_UUID_STRLEN - 1] = c;
+
+	/* Purge this vfio-pci specific token from the device arguments */
+	if (c != '\0') {
+		/* 1. Handle the case : 'vf_token=uuid,arg1=val1' */
+		memmove(p, vf_token + RTE_UUID_STRLEN,
+			strlen(vf_token + RTE_UUID_STRLEN) + 1);
+	} else {
+		/* 2. Handle the case : 'arg1=val1,vf_token=uuid' */
+		if (p != devargs->args)
+			p--;
+
+		*p = '\0';
+	}
+}
 
 static int
 pci_vfio_map_resource_primary(struct rte_pci_device *dev)  {
 	struct vfio_device_info device_info = { .argsz = sizeof(device_info) };
+	rte_uuid_t vf_token = RTE_UUID_INIT(0, 0, 0, 0, 0ULL);
 	char pci_addr[PATH_MAX] = {0};
 	int vfio_dev_fd;
 	struct rte_pci_addr *loc = &dev->addr; @@ -668,8 +717,9 @@

pci_vfio_map_resource_primary(struct rte_pci_device *dev)

drivers/bus/pci/linux/pci_vfio.c +2 -1
 	snprintf(pci_addr, sizeof(pci_addr), PCI_PRI_FMT,
 			loc->domain, loc->bus, loc->devid, loc->function);
 
+	vfio_pci_vf_token_arg(dev->device.devargs, vf_token);
 	ret = rte_vfio_setup_device(rte_pci_get_sysfs_path(), pci_addr,
-					&vfio_dev_fd, &device_info);
+					&vfio_dev_fd, &device_info,

vf_token);

drivers/bus/pci/linux/pci_vfio.c +1 -0
 	if (ret)
 		return ret;
@@ -797,6 +847,7 @@ static int
 pci_vfio_map_resource_secondary(struct rte_pci_device *dev)  {
 	struct vfio_device_info device_info = { .argsz = sizeof(device_info) };
+	rte_uuid_t vf_token = RTE_UUID_INIT(0, 0, 0, 0, 0ULL);
 	char pci_addr[PATH_MAX] = {0};
 	int vfio_dev_fd;
 	struct rte_pci_addr *loc = &dev->addr; @@ -830,8 +881,9 @@
 pci_vfio_map_resource_secondary(struct rte_pci_device *dev)
drivers/bus/pci/linux/pci_vfio.c +2 -1
 		return -1;
 	}
 
+	vfio_pci_vf_token_arg(dev->device.devargs, vf_token);
 	ret = rte_vfio_setup_device(rte_pci_get_sysfs_path(), pci_addr,
-					&vfio_dev_fd, &device_info);
+					&vfio_dev_fd, &device_info,

vf_token);

drivers/bus/pci/linux/pci_vfio.c +0 -0
 	if (ret)
 		return ret;
lib/librte_eal/freebsd/eal.c index renamed from lib/librte_eal/freebsd/eal.c +0 -0

6ae37e7e6..cbb4c400e 100644

lib/librte_eal/freebsd/eal.c +2 -1
@@ -995,7 +995,8 @@ rte_eal_vfio_intr_mode(void)  int
 rte_vfio_setup_device(__rte_unused const char *sysfs_base,
 		      __rte_unused const char *dev_addr,
 		      __rte_unused int *vfio_dev_fd,
-		      __rte_unused struct vfio_device_info *device_info)
+		      __rte_unused struct vfio_device_info *device_info
+		      __rte_unused rte_uuid_t vf_token)
 {
 	return -1;
 }
lib/librte_eal/include/rte_vfio.h +7 -1
index 20ed8c45a..1f9e22d82 100644
@@ -16,6 +16,8 @@ extern "C" {
 
 #include <stdint.h>
 
+#include <rte_uuid.h>
+
 /*
  * determine if VFIO is present on the system
  */
@@ -102,13 +104,17 @@ struct vfio_device_info;
  * @param device_info
  *   Device information.
  *
+ * @param vf_token
+ *   VF token.
+ *
  * @return
  *   0 on success.
  *   <0 on failure.
  *   >1 if the device cannot be managed this way.
  */
 int rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
-		int *vfio_dev_fd, struct vfio_device_info *device_info);
+		int *vfio_dev_fd, struct vfio_device_info *device_info,
+		rte_uuid_t vf_token);
 
 /**
  * Release a device mapped to a VFIO-managed I/O MMU group.
lib/librte_eal/linux/eal_vfio.c +3 -1
index 4502aefed..8c0ad04e5 100644
@@ -702,7 +702,8 @@ rte_vfio_clear_group(int vfio_group_fd)
 
 int
 rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
-		int *vfio_dev_fd, struct vfio_device_info *device_info)
+		int *vfio_dev_fd, struct vfio_device_info *device_info,
+		rte_uuid_t vf_token)
 {
 	struct vfio_group_status group_status = {
 			.argsz = sizeof(group_status)
@@ -712,6 +713,7 @@ rte_vfio_setup_device(const char *sysfs_base, const
 char *dev_addr,
 	int vfio_container_fd;
 	int vfio_group_fd;
 	int iommu_group_num;
+	char dev[PATH_MAX];
 	int i, ret;
 
lib/librte_eal/linux/eal_vfio.c +12 -1
 	/* get group number */
@@ -895,8 +897,19 @@ rte_vfio_setup_device(const char *sysfs_base,
 const char *dev_addr,
 				t->type_id, t->name);
 	}
 
+	if (!rte_uuid_is_null(vf_token)) {
+		char vf_token_str[RTE_UUID_STRLEN];
+
+		rte_uuid_unparse(vf_token, vf_token_str,
 sizeof(vf_token_str));
+		snprintf(dev, sizeof(dev),
+			 "%s vf_token=%s", dev_addr, vf_token_str);
+	} else {
+		snprintf(dev, sizeof(dev),
+			 "%s", dev_addr);
+	}
+
 	/* get a file descriptor for the device */
-	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD,
 dev_addr);
+	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD,

dev);

lib/librte_eal/linux/eal_vfio.c +0 -0
 	if (*vfio_dev_fd < 0) {
 		/* if we cannot get a device fd, this implies a problem with
 		 * the VFIO group or the container not having IOMMU

configured.

lib/librte_eal/linux/eal_vfio.c +2 -2
@@ -2081,7 +2094,8 @@ int
 rte_vfio_setup_device(__rte_unused const char *sysfs_base,
 		__rte_unused const char *dev_addr,
 		__rte_unused int *vfio_dev_fd,
-		__rte_unused struct vfio_device_info *device_info)
+		__rte_unused struct vfio_device_info *device_info
+		__rte_unused rte_uuid_t vf_token)
 {
 	return -1;
 }
--
Quoted 1 line

2.26.0

#8 lore raw

PATCH 1/1 eal: add VFIO-PCI SR-IOV support

In reply to [email protected]

The kernel module vfio-pci introduces the VF token to enable SR-IOV support since 5.7.

The VF token can be set by a vfio-pci based PF driver and must be known by the vfio-pci based VF driver in order to gain access to the device.

An example VF token option would take this form:

1. Install vfio-pci with option 'enable_sriov=1'

2. ./usertools/dpdk-devbind.py -b vfio-pci 0000:87:00.0

3. echo 2 > /sys/bus/pci/devices/0000:87:00.0/sriov_numvfs

4. Start the PF:

  ./x86_64-native-linux-gcc/app/testpmd -l 22-25 -n 4 \
         -w 87:00.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 \
         --file-prefix=pf -- -i

5. Start the VF:

   ./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 \
         -w 87:02.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 \
         --file-prefix=vf1 -- -i

---
v1: Update the commit message.

RFC v2: https://patchwork.dpdk.org/patch/68114/
Based on Vamsi's RFC v1, and Alex's patch for Qemu
[https://lore.kernel.org/lkml/[email protected]/]:
Use the devarg to pass-down the VF token.

RFC v1: https://patchwork.dpdk.org/patch/66281/ by Vamsi.

---
 drivers/bus/pci/linux/pci_vfio.c  | 56 +++++++++++++++++++++++++++++--
 lib/librte_eal/freebsd/eal.c      |  3 +-
 lib/librte_eal/include/rte_vfio.h |  8 ++++-
 lib/librte_eal/linux/eal_vfio.c   | 20 +++++++++--
 4 files changed, 80 insertions(+), 7 deletions(-)
drivers/bus/pci/linux/pci_vfio.c +54 -2
index 64cd84a68..7f99337c7 100644
@@ -11,6 +11,7 @@
 #include <sys/mman.h>
 #include <stdbool.h>
 
+#include <rte_devargs.h>
 #include <rte_log.h>
 #include <rte_pci.h>
 #include <rte_bus_pci.h>
@@ -644,11 +645,59 @@ pci_vfio_msix_is_mappable(int vfio_dev_fd, int msix_region)
 	return ret;
 }
 
+static void
+vfio_pci_vf_token_arg(struct rte_devargs *devargs, rte_uuid_t uu)
+{
+#define VF_TOKEN_ARG "vf_token="
+	char c, *p, *vf_token;
+
+	if (devargs == NULL)
+		return;
+
+	p = strstr(devargs->args, VF_TOKEN_ARG);
+	if (!p)
+		return;
+
+	vf_token = p + strlen(VF_TOKEN_ARG);
+	if (strlen(vf_token) < (RTE_UUID_STRLEN - 1))
+		return;
+
+	c = vf_token[RTE_UUID_STRLEN - 1];
+	if (c != '\0' && c != ',')
+		return;
+
+	vf_token[RTE_UUID_STRLEN - 1] = '\0';
+	if (rte_uuid_parse(vf_token, uu)) {
+		RTE_LOG(ERR, EAL,
+			"The VF token is not a valid uuid : %s\n", vf_token);
+		vf_token[RTE_UUID_STRLEN - 1] = c;
+		return;
+	}
+
+	RTE_LOG(DEBUG, EAL,
+		"The VF token is found : %s\n", vf_token);
+
+	vf_token[RTE_UUID_STRLEN - 1] = c;
+
+	/* Purge this vfio-pci specific token from the device arguments */
+	if (c != '\0') {
+		/* 1. Handle the case : 'vf_token=uuid,arg1=val1' */
+		memmove(p, vf_token + RTE_UUID_STRLEN,
+			strlen(vf_token + RTE_UUID_STRLEN) + 1);
+	} else {
+		/* 2. Handle the case : 'arg1=val1,vf_token=uuid' */
+		if (p != devargs->args)
+			p--;
+
+		*p = '\0';
+	}
+}
 
 static int
 pci_vfio_map_resource_primary(struct rte_pci_device *dev)
 {
 	struct vfio_device_info device_info = { .argsz = sizeof(device_info) };
+	rte_uuid_t vf_token = RTE_UUID_INIT(0, 0, 0, 0, 0ULL);
 	char pci_addr[PATH_MAX] = {0};
 	int vfio_dev_fd;
 	struct rte_pci_addr *loc = &dev->addr;
@@ -668,8 +717,9 @@ pci_vfio_map_resource_primary(struct rte_pci_device *dev)
 	snprintf(pci_addr, sizeof(pci_addr), PCI_PRI_FMT,
 			loc->domain, loc->bus, loc->devid, loc->function);
 
+	vfio_pci_vf_token_arg(dev->device.devargs, vf_token);
 	ret = rte_vfio_setup_device(rte_pci_get_sysfs_path(), pci_addr,
-					&vfio_dev_fd, &device_info);
+					&vfio_dev_fd, &device_info, vf_token);
 	if (ret)
 		return ret;
 
@@ -797,6 +847,7 @@ static int
 pci_vfio_map_resource_secondary(struct rte_pci_device *dev)
 {
 	struct vfio_device_info device_info = { .argsz = sizeof(device_info) };
+	rte_uuid_t vf_token = RTE_UUID_INIT(0, 0, 0, 0, 0ULL);
 	char pci_addr[PATH_MAX] = {0};
 	int vfio_dev_fd;
 	struct rte_pci_addr *loc = &dev->addr;
@@ -830,8 +881,9 @@ pci_vfio_map_resource_secondary(struct rte_pci_device *dev)
 		return -1;
 	}
 
+	vfio_pci_vf_token_arg(dev->device.devargs, vf_token);
 	ret = rte_vfio_setup_device(rte_pci_get_sysfs_path(), pci_addr,
-					&vfio_dev_fd, &device_info);
+					&vfio_dev_fd, &device_info, vf_token);
 	if (ret)
 		return ret;
 
lib/librte_eal/freebsd/eal.c +2 -1
index 6ae37e7e6..cbb4c400e 100644
@@ -995,7 +995,8 @@ rte_eal_vfio_intr_mode(void)
 int rte_vfio_setup_device(__rte_unused const char *sysfs_base,
 		      __rte_unused const char *dev_addr,
 		      __rte_unused int *vfio_dev_fd,
-		      __rte_unused struct vfio_device_info *device_info)
+		      __rte_unused struct vfio_device_info *device_info
+		      __rte_unused rte_uuid_t vf_token)
 {
 	return -1;
 }
lib/librte_eal/include/rte_vfio.h +7 -1
index 20ed8c45a..1f9e22d82 100644
@@ -16,6 +16,8 @@ extern "C" {
 
 #include <stdint.h>
 
+#include <rte_uuid.h>
+
 /*
  * determine if VFIO is present on the system
  */
@@ -102,13 +104,17 @@ struct vfio_device_info;
  * @param device_info
  *   Device information.
  *
+ * @param vf_token
+ *   VF token.
+ *
  * @return
  *   0 on success.
  *   <0 on failure.
  *   >1 if the device cannot be managed this way.
  */
 int rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
-		int *vfio_dev_fd, struct vfio_device_info *device_info);
+		int *vfio_dev_fd, struct vfio_device_info *device_info,
+		rte_uuid_t vf_token);
 
 /**
  * Release a device mapped to a VFIO-managed I/O MMU group.
lib/librte_eal/linux/eal_vfio.c +17 -3
index 4502aefed..8c0ad04e5 100644
@@ -702,7 +702,8 @@ rte_vfio_clear_group(int vfio_group_fd)
 
 int
 rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
-		int *vfio_dev_fd, struct vfio_device_info *device_info)
+		int *vfio_dev_fd, struct vfio_device_info *device_info,
+		rte_uuid_t vf_token)
 {
 	struct vfio_group_status group_status = {
 			.argsz = sizeof(group_status)
@@ -712,6 +713,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 	int vfio_container_fd;
 	int vfio_group_fd;
 	int iommu_group_num;
+	char dev[PATH_MAX];
 	int i, ret;
 
 	/* get group number */
@@ -895,8 +897,19 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 				t->type_id, t->name);
 	}
 
+	if (!rte_uuid_is_null(vf_token)) {
+		char vf_token_str[RTE_UUID_STRLEN];
+
+		rte_uuid_unparse(vf_token, vf_token_str, sizeof(vf_token_str));
+		snprintf(dev, sizeof(dev),
+			 "%s vf_token=%s", dev_addr, vf_token_str);
+	} else {
+		snprintf(dev, sizeof(dev),
+			 "%s", dev_addr);
+	}
+
 	/* get a file descriptor for the device */
-	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev_addr);
+	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev);
 	if (*vfio_dev_fd < 0) {
 		/* if we cannot get a device fd, this implies a problem with
 		 * the VFIO group or the container not having IOMMU configured.
@@ -2081,7 +2094,8 @@ int
 rte_vfio_setup_device(__rte_unused const char *sysfs_base,
 		__rte_unused const char *dev_addr,
 		__rte_unused int *vfio_dev_fd,
-		__rte_unused struct vfio_device_info *device_info)
+		__rte_unused struct vfio_device_info *device_info
+		__rte_unused rte_uuid_t vf_token)
 {
 	return -1;
 }
Signature · 1 line
2.26.0
#9 lore raw

PATCH v2 1/1 eal: add VFIO-PCI SR-IOV support

In reply to [email protected]

The kernel module vfio-pci introduces the VF token to enable SR-IOV support since 5.7.

The VF token can be set by a vfio-pci based PF driver and must be known by the vfio-pci based VF driver in order to gain access to the device.

An example VF token option would take this form:

1. Install vfio-pci with option 'enable_sriov=1'

2. ./usertools/dpdk-devbind.py -b vfio-pci 0000:87:00.0

3. echo 2 > /sys/bus/pci/devices/0000:87:00.0/sriov_numvfs

4. Start the PF:

  ./x86_64-native-linux-gcc/app/testpmd -l 22-25 -n 4 \
         -w 87:00.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 \
         --file-prefix=pf -- -i

5. Start the VF:

   ./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 \
         -w 87:02.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 \
         --file-prefix=vf1 -- -i

---
v2: Fix the FreeBSD build error.

v1: https://patchwork.dpdk.org/patch/68237/
Update the commit message.

RFC v2: https://patchwork.dpdk.org/patch/68114/
Based on Vamsi's RFC v1, and Alex's patch for Qemu
[https://lore.kernel.org/lkml/[email protected]/]:
Use the devarg to pass-down the VF token.

RFC v1: https://patchwork.dpdk.org/patch/66281/ by Vamsi.

---
 drivers/bus/pci/linux/pci_vfio.c  | 56 +++++++++++++++++++++++++++++--
 lib/librte_eal/freebsd/eal.c      |  3 +-
 lib/librte_eal/include/rte_vfio.h |  8 ++++-
 lib/librte_eal/linux/eal_vfio.c   | 20 +++++++++--
 4 files changed, 80 insertions(+), 7 deletions(-)
drivers/bus/pci/linux/pci_vfio.c +54 -2
index 64cd84a68..7f99337c7 100644
@@ -11,6 +11,7 @@
 #include <sys/mman.h>
 #include <stdbool.h>
 
+#include <rte_devargs.h>
 #include <rte_log.h>
 #include <rte_pci.h>
 #include <rte_bus_pci.h>
@@ -644,11 +645,59 @@ pci_vfio_msix_is_mappable(int vfio_dev_fd, int msix_region)
 	return ret;
 }
 
+static void
+vfio_pci_vf_token_arg(struct rte_devargs *devargs, rte_uuid_t uu)
+{
+#define VF_TOKEN_ARG "vf_token="
+	char c, *p, *vf_token;
+
+	if (devargs == NULL)
+		return;
+
+	p = strstr(devargs->args, VF_TOKEN_ARG);
+	if (!p)
+		return;
+
+	vf_token = p + strlen(VF_TOKEN_ARG);
+	if (strlen(vf_token) < (RTE_UUID_STRLEN - 1))
+		return;
+
+	c = vf_token[RTE_UUID_STRLEN - 1];
+	if (c != '\0' && c != ',')
+		return;
+
+	vf_token[RTE_UUID_STRLEN - 1] = '\0';
+	if (rte_uuid_parse(vf_token, uu)) {
+		RTE_LOG(ERR, EAL,
+			"The VF token is not a valid uuid : %s\n", vf_token);
+		vf_token[RTE_UUID_STRLEN - 1] = c;
+		return;
+	}
+
+	RTE_LOG(DEBUG, EAL,
+		"The VF token is found : %s\n", vf_token);
+
+	vf_token[RTE_UUID_STRLEN - 1] = c;
+
+	/* Purge this vfio-pci specific token from the device arguments */
+	if (c != '\0') {
+		/* 1. Handle the case : 'vf_token=uuid,arg1=val1' */
+		memmove(p, vf_token + RTE_UUID_STRLEN,
+			strlen(vf_token + RTE_UUID_STRLEN) + 1);
+	} else {
+		/* 2. Handle the case : 'arg1=val1,vf_token=uuid' */
+		if (p != devargs->args)
+			p--;
+
+		*p = '\0';
+	}
+}
 
 static int
 pci_vfio_map_resource_primary(struct rte_pci_device *dev)
 {
 	struct vfio_device_info device_info = { .argsz = sizeof(device_info) };
+	rte_uuid_t vf_token = RTE_UUID_INIT(0, 0, 0, 0, 0ULL);
 	char pci_addr[PATH_MAX] = {0};
 	int vfio_dev_fd;
 	struct rte_pci_addr *loc = &dev->addr;
@@ -668,8 +717,9 @@ pci_vfio_map_resource_primary(struct rte_pci_device *dev)
 	snprintf(pci_addr, sizeof(pci_addr), PCI_PRI_FMT,
 			loc->domain, loc->bus, loc->devid, loc->function);
 
+	vfio_pci_vf_token_arg(dev->device.devargs, vf_token);
 	ret = rte_vfio_setup_device(rte_pci_get_sysfs_path(), pci_addr,
-					&vfio_dev_fd, &device_info);
+					&vfio_dev_fd, &device_info, vf_token);
 	if (ret)
 		return ret;
 
@@ -797,6 +847,7 @@ static int
 pci_vfio_map_resource_secondary(struct rte_pci_device *dev)
 {
 	struct vfio_device_info device_info = { .argsz = sizeof(device_info) };
+	rte_uuid_t vf_token = RTE_UUID_INIT(0, 0, 0, 0, 0ULL);
 	char pci_addr[PATH_MAX] = {0};
 	int vfio_dev_fd;
 	struct rte_pci_addr *loc = &dev->addr;
@@ -830,8 +881,9 @@ pci_vfio_map_resource_secondary(struct rte_pci_device *dev)
 		return -1;
 	}
 
+	vfio_pci_vf_token_arg(dev->device.devargs, vf_token);
 	ret = rte_vfio_setup_device(rte_pci_get_sysfs_path(), pci_addr,
-					&vfio_dev_fd, &device_info);
+					&vfio_dev_fd, &device_info, vf_token);
 	if (ret)
 		return ret;
 
lib/librte_eal/freebsd/eal.c +2 -1
index 6ae37e7e6..a92584795 100644
@@ -995,7 +995,8 @@ rte_eal_vfio_intr_mode(void)
 int rte_vfio_setup_device(__rte_unused const char *sysfs_base,
 		      __rte_unused const char *dev_addr,
 		      __rte_unused int *vfio_dev_fd,
-		      __rte_unused struct vfio_device_info *device_info)
+		      __rte_unused struct vfio_device_info *device_info,
+		      __rte_unused rte_uuid_t vf_token)
 {
 	return -1;
 }
lib/librte_eal/include/rte_vfio.h +7 -1
index 20ed8c45a..1f9e22d82 100644
@@ -16,6 +16,8 @@ extern "C" {
 
 #include <stdint.h>
 
+#include <rte_uuid.h>
+
 /*
  * determine if VFIO is present on the system
  */
@@ -102,13 +104,17 @@ struct vfio_device_info;
  * @param device_info
  *   Device information.
  *
+ * @param vf_token
+ *   VF token.
+ *
  * @return
  *   0 on success.
  *   <0 on failure.
  *   >1 if the device cannot be managed this way.
  */
 int rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
-		int *vfio_dev_fd, struct vfio_device_info *device_info);
+		int *vfio_dev_fd, struct vfio_device_info *device_info,
+		rte_uuid_t vf_token);
 
 /**
  * Release a device mapped to a VFIO-managed I/O MMU group.
lib/librte_eal/linux/eal_vfio.c +17 -3
index 4502aefed..8c0ad04e5 100644
@@ -702,7 +702,8 @@ rte_vfio_clear_group(int vfio_group_fd)
 
 int
 rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
-		int *vfio_dev_fd, struct vfio_device_info *device_info)
+		int *vfio_dev_fd, struct vfio_device_info *device_info,
+		rte_uuid_t vf_token)
 {
 	struct vfio_group_status group_status = {
 			.argsz = sizeof(group_status)
@@ -712,6 +713,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 	int vfio_container_fd;
 	int vfio_group_fd;
 	int iommu_group_num;
+	char dev[PATH_MAX];
 	int i, ret;
 
 	/* get group number */
@@ -895,8 +897,19 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 				t->type_id, t->name);
 	}
 
+	if (!rte_uuid_is_null(vf_token)) {
+		char vf_token_str[RTE_UUID_STRLEN];
+
+		rte_uuid_unparse(vf_token, vf_token_str, sizeof(vf_token_str));
+		snprintf(dev, sizeof(dev),
+			 "%s vf_token=%s", dev_addr, vf_token_str);
+	} else {
+		snprintf(dev, sizeof(dev),
+			 "%s", dev_addr);
+	}
+
 	/* get a file descriptor for the device */
-	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev_addr);
+	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev);
 	if (*vfio_dev_fd < 0) {
 		/* if we cannot get a device fd, this implies a problem with
 		 * the VFIO group or the container not having IOMMU configured.
@@ -2081,7 +2094,8 @@ int
 rte_vfio_setup_device(__rte_unused const char *sysfs_base,
 		__rte_unused const char *dev_addr,
 		__rte_unused int *vfio_dev_fd,
-		__rte_unused struct vfio_device_info *device_info)
+		__rte_unused struct vfio_device_info *device_info,
+		__rte_unused rte_uuid_t vf_token)
 {
 	return -1;
 }
Signature · 1 line
2.26.0
#10 lore raw

PATCH v3 1/1 eal: add VFIO-PCI SR-IOV support

In reply to [email protected]

The kernel module vfio-pci introduces the VF token to enable SR-IOV support since 5.7.

The VF token can be set by a vfio-pci based PF driver and must be known by the vfio-pci based VF driver in order to gain access to the device.

An example VF token option would take this form:

1. Install vfio-pci with option 'enable_sriov=1'

2. ./usertools/dpdk-devbind.py -b vfio-pci 0000:87:00.0

3. echo 2 > /sys/bus/pci/devices/0000:87:00.0/sriov_numvfs

4. Start the PF:

  ./x86_64-native-linux-gcc/app/testpmd -l 22-25 -n 4 \
         -w 87:00.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 \
         --file-prefix=pf -- -i

5. Start the VF:

   ./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 \
         -w 87:02.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 \
         --file-prefix=vf1 -- -i

---
v3: Fix the Travis build failed:
(1). rte_uuid.h:97:55: error: unknown type name ‘size_t’ (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’

v2: https://patchwork.dpdk.org/patch/68240/
Fix the FreeBSD build error.

v1: https://patchwork.dpdk.org/patch/68237/
Update the commit message.

RFC v2: https://patchwork.dpdk.org/patch/68114/
Based on Vamsi's RFC v1, and Alex's patch for Qemu
[https://lore.kernel.org/lkml/[email protected]/]:
Use the devarg to pass-down the VF token.

RFC v1: https://patchwork.dpdk.org/patch/66281/ by Vamsi.

---
 drivers/bus/pci/linux/pci_vfio.c  | 56 +++++++++++++++++++++++++++++--
 lib/librte_eal/freebsd/eal.c      |  3 +-
 lib/librte_eal/include/rte_uuid.h |  2 ++
 lib/librte_eal/include/rte_vfio.h |  8 ++++-
 lib/librte_eal/linux/eal_vfio.c   | 20 +++++++++--
 5 files changed, 82 insertions(+), 7 deletions(-)
drivers/bus/pci/linux/pci_vfio.c +54 -2
index 64cd84a68..7f99337c7 100644
@@ -11,6 +11,7 @@
 #include <sys/mman.h>
 #include <stdbool.h>
 
+#include <rte_devargs.h>
 #include <rte_log.h>
 #include <rte_pci.h>
 #include <rte_bus_pci.h>
@@ -644,11 +645,59 @@ pci_vfio_msix_is_mappable(int vfio_dev_fd, int msix_region)
 	return ret;
 }
 
+static void
+vfio_pci_vf_token_arg(struct rte_devargs *devargs, rte_uuid_t uu)
+{
+#define VF_TOKEN_ARG "vf_token="
+	char c, *p, *vf_token;
+
+	if (devargs == NULL)
+		return;
+
+	p = strstr(devargs->args, VF_TOKEN_ARG);
+	if (!p)
+		return;
+
+	vf_token = p + strlen(VF_TOKEN_ARG);
+	if (strlen(vf_token) < (RTE_UUID_STRLEN - 1))
+		return;
+
+	c = vf_token[RTE_UUID_STRLEN - 1];
+	if (c != '\0' && c != ',')
+		return;
+
+	vf_token[RTE_UUID_STRLEN - 1] = '\0';
+	if (rte_uuid_parse(vf_token, uu)) {
+		RTE_LOG(ERR, EAL,
+			"The VF token is not a valid uuid : %s\n", vf_token);
+		vf_token[RTE_UUID_STRLEN - 1] = c;
+		return;
+	}
+
+	RTE_LOG(DEBUG, EAL,
+		"The VF token is found : %s\n", vf_token);
+
+	vf_token[RTE_UUID_STRLEN - 1] = c;
+
+	/* Purge this vfio-pci specific token from the device arguments */
+	if (c != '\0') {
+		/* 1. Handle the case : 'vf_token=uuid,arg1=val1' */
+		memmove(p, vf_token + RTE_UUID_STRLEN,
+			strlen(vf_token + RTE_UUID_STRLEN) + 1);
+	} else {
+		/* 2. Handle the case : 'arg1=val1,vf_token=uuid' */
+		if (p != devargs->args)
+			p--;
+
+		*p = '\0';
+	}
+}
 
 static int
 pci_vfio_map_resource_primary(struct rte_pci_device *dev)
 {
 	struct vfio_device_info device_info = { .argsz = sizeof(device_info) };
+	rte_uuid_t vf_token = RTE_UUID_INIT(0, 0, 0, 0, 0ULL);
 	char pci_addr[PATH_MAX] = {0};
 	int vfio_dev_fd;
 	struct rte_pci_addr *loc = &dev->addr;
@@ -668,8 +717,9 @@ pci_vfio_map_resource_primary(struct rte_pci_device *dev)
 	snprintf(pci_addr, sizeof(pci_addr), PCI_PRI_FMT,
 			loc->domain, loc->bus, loc->devid, loc->function);
 
+	vfio_pci_vf_token_arg(dev->device.devargs, vf_token);
 	ret = rte_vfio_setup_device(rte_pci_get_sysfs_path(), pci_addr,
-					&vfio_dev_fd, &device_info);
+					&vfio_dev_fd, &device_info, vf_token);
 	if (ret)
 		return ret;
 
@@ -797,6 +847,7 @@ static int
 pci_vfio_map_resource_secondary(struct rte_pci_device *dev)
 {
 	struct vfio_device_info device_info = { .argsz = sizeof(device_info) };
+	rte_uuid_t vf_token = RTE_UUID_INIT(0, 0, 0, 0, 0ULL);
 	char pci_addr[PATH_MAX] = {0};
 	int vfio_dev_fd;
 	struct rte_pci_addr *loc = &dev->addr;
@@ -830,8 +881,9 @@ pci_vfio_map_resource_secondary(struct rte_pci_device *dev)
 		return -1;
 	}
 
+	vfio_pci_vf_token_arg(dev->device.devargs, vf_token);
 	ret = rte_vfio_setup_device(rte_pci_get_sysfs_path(), pci_addr,
-					&vfio_dev_fd, &device_info);
+					&vfio_dev_fd, &device_info, vf_token);
 	if (ret)
 		return ret;
 
lib/librte_eal/freebsd/eal.c +2 -1
index 6ae37e7e6..a92584795 100644
@@ -995,7 +995,8 @@ rte_eal_vfio_intr_mode(void)
 int rte_vfio_setup_device(__rte_unused const char *sysfs_base,
 		      __rte_unused const char *dev_addr,
 		      __rte_unused int *vfio_dev_fd,
-		      __rte_unused struct vfio_device_info *device_info)
+		      __rte_unused struct vfio_device_info *device_info,
+		      __rte_unused rte_uuid_t vf_token)
 {
 	return -1;
 }
lib/librte_eal/include/rte_uuid.h +2 -0
index 044afbdfa..8b42e070a 100644
@@ -15,6 +15,8 @@ extern "C" {
 #endif
 
 #include <stdbool.h>
+#include <stddef.h>
+#include <string.h>
 
 /**
  * Struct describing a Universal Unique Identifier
lib/librte_eal/include/rte_vfio.h +7 -1
index 20ed8c45a..1f9e22d82 100644
@@ -16,6 +16,8 @@ extern "C" {
 
 #include <stdint.h>
 
+#include <rte_uuid.h>
+
 /*
  * determine if VFIO is present on the system
  */
@@ -102,13 +104,17 @@ struct vfio_device_info;
  * @param device_info
  *   Device information.
  *
+ * @param vf_token
+ *   VF token.
+ *
  * @return
  *   0 on success.
  *   <0 on failure.
  *   >1 if the device cannot be managed this way.
  */
 int rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
-		int *vfio_dev_fd, struct vfio_device_info *device_info);
+		int *vfio_dev_fd, struct vfio_device_info *device_info,
+		rte_uuid_t vf_token);
 
 /**
  * Release a device mapped to a VFIO-managed I/O MMU group.
lib/librte_eal/linux/eal_vfio.c +17 -3
index 4502aefed..8c0ad04e5 100644
@@ -702,7 +702,8 @@ rte_vfio_clear_group(int vfio_group_fd)
 
 int
 rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
-		int *vfio_dev_fd, struct vfio_device_info *device_info)
+		int *vfio_dev_fd, struct vfio_device_info *device_info,
+		rte_uuid_t vf_token)
 {
 	struct vfio_group_status group_status = {
 			.argsz = sizeof(group_status)
@@ -712,6 +713,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 	int vfio_container_fd;
 	int vfio_group_fd;
 	int iommu_group_num;
+	char dev[PATH_MAX];
 	int i, ret;
 
 	/* get group number */
@@ -895,8 +897,19 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 				t->type_id, t->name);
 	}
 
+	if (!rte_uuid_is_null(vf_token)) {
+		char vf_token_str[RTE_UUID_STRLEN];
+
+		rte_uuid_unparse(vf_token, vf_token_str, sizeof(vf_token_str));
+		snprintf(dev, sizeof(dev),
+			 "%s vf_token=%s", dev_addr, vf_token_str);
+	} else {
+		snprintf(dev, sizeof(dev),
+			 "%s", dev_addr);
+	}
+
 	/* get a file descriptor for the device */
-	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev_addr);
+	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev);
 	if (*vfio_dev_fd < 0) {
 		/* if we cannot get a device fd, this implies a problem with
 		 * the VFIO group or the container not having IOMMU configured.
@@ -2081,7 +2094,8 @@ int
 rte_vfio_setup_device(__rte_unused const char *sysfs_base,
 		__rte_unused const char *dev_addr,
 		__rte_unused int *vfio_dev_fd,
-		__rte_unused struct vfio_device_info *device_info)
+		__rte_unused struct vfio_device_info *device_info
+		__rte_unused rte_uuid_t vf_token)
 {
 	return -1;
 }
Signature · 1 line
2.26.0
#11 lore raw

PATCH v4 1/1 eal: add VFIO-PCI SR-IOV support

In reply to [email protected]

The kernel module vfio-pci introduces the VF token to enable SR-IOV support since 5.7.

The VF token can be set by a vfio-pci based PF driver and must be known by the vfio-pci based VF driver in order to gain access to the device.

An example VF token option would take this form:

1. Install vfio-pci with option 'enable_sriov=1'

2. ./usertools/dpdk-devbind.py -b vfio-pci 0000:87:00.0

3. echo 2 > /sys/bus/pci/devices/0000:87:00.0/sriov_numvfs

4. Start the PF:

  ./x86_64-native-linux-gcc/app/testpmd -l 22-25 -n 4 \
         -w 87:00.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 \
         --file-prefix=pf -- -i

5. Start the VF:

   ./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 \
         -w 87:02.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 \
         --file-prefix=vf1 -- -i

---
v4: 1. Ignore rte_vfio_setup_device ABI check since it is
for Linux driver use.

v3: https://patchwork.dpdk.org/patch/68254/
Fix the Travis build failed:
(1). rte_uuid.h:97:55: error: unknown type name ‘size_t’ (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’

v2: https://patchwork.dpdk.org/patch/68240/
Fix the FreeBSD build error.

v1: https://patchwork.dpdk.org/patch/68237/
Update the commit message.

RFC v2: https://patchwork.dpdk.org/patch/68114/
Based on Vamsi's RFC v1, and Alex's patch for Qemu
[https://lore.kernel.org/lkml/[email protected]/]:
Use the devarg to pass-down the VF token.

RFC v1: https://patchwork.dpdk.org/patch/66281/ by Vamsi.

---
 devtools/libabigail.abignore      |  3 ++
 drivers/bus/pci/linux/pci_vfio.c  | 56 +++++++++++++++++++++++++++++--
 lib/librte_eal/freebsd/eal.c      |  3 +-
 lib/librte_eal/include/rte_uuid.h |  2 ++
 lib/librte_eal/include/rte_vfio.h |  8 ++++-
 lib/librte_eal/linux/eal_vfio.c   | 20 +++++++++--
 6 files changed, 85 insertions(+), 7 deletions(-)
devtools/libabigail.abignore +3 -0
index a59df8f13..d918746b4 100644
@@ -11,3 +11,6 @@
         type_kind = enum
         name = rte_crypto_asym_xform_type
         changed_enumerators = RTE_CRYPTO_ASYM_XFORM_TYPE_LIST_END
+; Ignore this function which is only relevant to linux for driver
+[suppress_type]
+	name = rte_vfio_setup_device
drivers/bus/pci/linux/pci_vfio.c +54 -2

93 lines not on this page · raw message

lib/librte_eal/freebsd/eal.c +2 -1
index 6ae37e7e6..a92584795 100644
@@ -995,7 +995,8 @@ rte_eal_vfio_intr_mode(void)
 int rte_vfio_setup_device(__rte_unused const char *sysfs_base,
 		      __rte_unused const char *dev_addr,
 		      __rte_unused int *vfio_dev_fd,
-		      __rte_unused struct vfio_device_info *device_info)
+		      __rte_unused struct vfio_device_info *device_info,
+		      __rte_unused rte_uuid_t vf_token)
 {
 	return -1;
 }
lib/librte_eal/include/rte_uuid.h +2 -0
index 044afbdfa..8b42e070a 100644
@@ -15,6 +15,8 @@ extern "C" {
 #endif
 
 #include <stdbool.h>
+#include <stddef.h>
+#include <string.h>
 
 /**
  * Struct describing a Universal Unique Identifier
lib/librte_eal/include/rte_vfio.h +7 -1
index 20ed8c45a..1f9e22d82 100644
@@ -16,6 +16,8 @@ extern "C" {
 
 #include <stdint.h>
 
+#include <rte_uuid.h>
+
 /*
  * determine if VFIO is present on the system
  */
@@ -102,13 +104,17 @@ struct vfio_device_info;
  * @param device_info
  *   Device information.
  *
+ * @param vf_token
+ *   VF token.
+ *
  * @return
  *   0 on success.
  *   <0 on failure.
  *   >1 if the device cannot be managed this way.
  */
 int rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
-		int *vfio_dev_fd, struct vfio_device_info *device_info);
+		int *vfio_dev_fd, struct vfio_device_info *device_info,
+		rte_uuid_t vf_token);
 
 /**
  * Release a device mapped to a VFIO-managed I/O MMU group.
lib/librte_eal/linux/eal_vfio.c +17 -3
index 4502aefed..916082b5d 100644
@@ -702,7 +702,8 @@ rte_vfio_clear_group(int vfio_group_fd)
 
 int
 rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
-		int *vfio_dev_fd, struct vfio_device_info *device_info)
+		int *vfio_dev_fd, struct vfio_device_info *device_info,
+		rte_uuid_t vf_token)
 {
 	struct vfio_group_status group_status = {
 			.argsz = sizeof(group_status)
@@ -712,6 +713,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 	int vfio_container_fd;
 	int vfio_group_fd;
 	int iommu_group_num;
+	char dev[PATH_MAX];
 	int i, ret;
 
 	/* get group number */
@@ -895,8 +897,19 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 				t->type_id, t->name);
 	}
 
+	if (!rte_uuid_is_null(vf_token)) {
+		char vf_token_str[RTE_UUID_STRLEN];
+
+		rte_uuid_unparse(vf_token, vf_token_str, sizeof(vf_token_str));
+		snprintf(dev, sizeof(dev),
+			 "%s vf_token=%s", dev_addr, vf_token_str);
+	} else {
+		snprintf(dev, sizeof(dev),
+			 "%s", dev_addr);
+	}
+
 	/* get a file descriptor for the device */
-	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev_addr);
+	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev);
 	if (*vfio_dev_fd < 0) {
 		/* if we cannot get a device fd, this implies a problem with
 		 * the VFIO group or the container not having IOMMU configured.
@@ -2081,7 +2094,8 @@ int
 rte_vfio_setup_device(__rte_unused const char *sysfs_base,
 		__rte_unused const char *dev_addr,
 		__rte_unused int *vfio_dev_fd,
-		__rte_unused struct vfio_device_info *device_info)
+		__rte_unused struct vfio_device_info *device_info,
+		__rte_unused rte_uuid_t vf_token)
 {
 	return -1;
 }
Signature · 1 line
2.26.0
#12 lore raw

Re: [dpdk-dev] [PATCH v4] eal: add VFIO-PCI SR-IOV support

Hi,

About the title, I think it does not convey what is new here. VFIO is not new, SR-IOV is already supported.
The title should mention the new VFIO feature in few simple words. Is it only about using VFIO for PF?

13/04/2020 10:29, Haiyue Wang:

Quoted 2 lines

v4: 1. Ignore rte_vfio_setup_device ABI check since it is
for Linux driver use.

[...]

devtools/libabigail.abignore line 16

Quoted 3 lines
devtools/libabigail.abignore +3 -0
+; Ignore this function which is only relevant to linux for driver
+[suppress_type]
+	name = rte_vfio_setup_device

Adding such exception for all internal "driver interface" functions is not scaling. Please use __rte_internal.
I am waiting for the patchset about rte_internal to be reviewed or completed. As it is not progressing, the decision is to block any patch having ABI issue because of internal false positive.
Please help, thanks.

#13 lore raw

Re: [dpdk-dev] [PATCH v4] eal: add VFIO-PCI SR-IOV support

Hi Thomas,

Quoted 8 lines

-----Original Message-----
From: Thomas Monjalon <[email protected]>
Sent: Monday, April 13, 2020 20:18
To: Wang, Haiyue <[email protected]>
Cc: [email protected]; [email protected]; [email protected]; [email protected]; [email protected]
Subject: Re: [PATCH v4] eal: add VFIO-PCI SR-IOV support

Quoted 7 lines

Hi,

About the title, I think it does not convey what is new here. VFIO is not new, SR-IOV is already supported.
The title should mention the new VFIO feature in few simple words. Is it only about using VFIO for PF?

For Both. Align with Alex's : vfio-pci: QEMU support for vfio-pci VF tokens

How about: 'eal: support for VFIO-PCI VF tokens' ?

Quoted 5 lines

13/04/2020 10:29, Haiyue Wang:

Quoted 2 lines

v4: 1. Ignore rte_vfio_setup_device ABI check since it is
for Linux driver use.

[...]

Quoted 11 lines

devtools/libabigail.abignore line 16

Quoted 3 lines
devtools/libabigail.abignore +3 -0
+; Ignore this function which is only relevant to linux for driver
+[suppress_type]
+	name = rte_vfio_setup_device

Adding such exception for all internal "driver interface" functions is not scaling. Please use __rte_internal.
I am waiting for the patchset about rte_internal to be reviewed or completed. As it is not progressing, the decision is to block any patch having ABI issue because of internal false positive.
Please help, thanks.

I will drop this patch ABI workaround, and try to catch the '__rte_internal ' design.

#14 lore raw

Re: [dpdk-dev] [PATCH v4] eal: add VFIO-PCI SR-IOV support

In reply to Haiyue Wang
Quoted 10 lines · Haiyue Wang

The kernel module vfio-pci introduces the VF token to enable SR-IOV support since 5.7.

The VF token can be set by a vfio-pci based PF driver and must be known by the vfio-pci based VF driver in order to gain access to the device.

An example VF token option would take this form:

1. Install vfio-pci with option 'enable_sriov=1'

Quoted 8 lines

2. ./usertools/dpdk-devbind.py -b vfio-pci 0000:87:00.0

3. echo 2 > /sys/bus/pci/devices/0000:87:00.0/sriov_numvfs

4. Start the PF:

  ./x86_64-native-linux-gcc/app/testpmd -l 22-25 -n 4 \
         -w 87:00.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 \
         --file-prefix=pf -- -i

Should I get a token from my head? Any?

Quoted 76 lines

5. Start the VF:

   ./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 \
         -w 87:02.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 \
         --file-prefix=vf1 -- -i

---
v4: 1. Ignore rte_vfio_setup_device ABI check since it is
for Linux driver use.

v3: https://patchwork.dpdk.org/patch/68254/
Fix the Travis build failed:
(1). rte_uuid.h:97:55: error: unknown type name ‘size_t’ (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’

v2: https://patchwork.dpdk.org/patch/68240/
Fix the FreeBSD build error.

v1: https://patchwork.dpdk.org/patch/68237/
Update the commit message.

RFC v2: https://patchwork.dpdk.org/patch/68114/
Based on Vamsi's RFC v1, and Alex's patch for Qemu
[https://lore.kernel.org/lkml/[email protected]/]:
Use the devarg to pass-down the VF token.

RFC v1: https://patchwork.dpdk.org/patch/66281/ by Vamsi.

---
 devtools/libabigail.abignore      |  3 ++
 drivers/bus/pci/linux/pci_vfio.c  | 56 +++++++++++++++++++++++++++++--
 lib/librte_eal/freebsd/eal.c      |  3 +-
 lib/librte_eal/include/rte_uuid.h |  2 ++
 lib/librte_eal/include/rte_vfio.h |  8 ++++-
 lib/librte_eal/linux/eal_vfio.c   | 20 +++++++++--
 6 files changed, 85 insertions(+), 7 deletions(-)
devtools/libabigail.abignore +3 -0
index a59df8f13..d918746b4 100644
@@ -11,3 +11,6 @@
         type_kind = enum
         name = rte_crypto_asym_xform_type
         changed_enumerators = RTE_CRYPTO_ASYM_XFORM_TYPE_LIST_END
+; Ignore this function which is only relevant to linux for driver
+[suppress_type]
+	name = rte_vfio_setup_device
drivers/bus/pci/linux/pci_vfio.c +13 -0
index 64cd84a68..7f99337c7 100644
@@ -11,6 +11,7 @@
 #include <sys/mman.h>
 #include <stdbool.h>
 
+#include <rte_devargs.h>
 #include <rte_log.h>
 #include <rte_pci.h>
 #include <rte_bus_pci.h>
@@ -644,11 +645,59 @@ pci_vfio_msix_is_mappable(int vfio_dev_fd, int msix_region)
 	return ret;
 }
 
+static void
+vfio_pci_vf_token_arg(struct rte_devargs *devargs, rte_uuid_t uu)
+{
+#define VF_TOKEN_ARG "vf_token="
+	char c, *p, *vf_token;
+
+	if (devargs == NULL)
+		return;
+
+	p = strstr(devargs->args, VF_TOKEN_ARG);
+	if (!p)
+		return;

drivers/bus/pci/linux/pci_vfio.c pci_vfio_msix_is_mappable(int vfio_dev_fd, int msix_region) · line 669

Quoted 10 lines
drivers/bus/pci/linux/pci_vfio.c +10 -0
+
+	vf_token = p + strlen(VF_TOKEN_ARG);
+	if (strlen(vf_token) < (RTE_UUID_STRLEN - 1))
+		return;
+
+	c = vf_token[RTE_UUID_STRLEN - 1];
+	if (c != '\0' && c != ',')
+		return;
+
+	vf_token[RTE_UUID_STRLEN - 1] = '\0';

Is it possible to parse and handle devargs using rte_kvargs.h?

drivers/bus/pci/linux/pci_vfio.c pci_vfio_msix_is_mappable(int vfio_dev_fd, int msix_region) · line 674

Quoted 5 lines
drivers/bus/pci/linux/pci_vfio.c +5 -0
+	if (rte_uuid_parse(vf_token, uu)) {
+		RTE_LOG(ERR, EAL,
+			"The VF token is not a valid uuid : %s\n", vf_token);
+		vf_token[RTE_UUID_STRLEN - 1] = c;
+		return;

I think that the function must return error which is handled by the caller when something bad happens (e.g. invalid UUID).

Quoted 9 lines
drivers/bus/pci/linux/pci_vfio.c +9 -0
+	}
+
+	RTE_LOG(DEBUG, EAL,
+		"The VF token is found : %s\n", vf_token);
+
+	vf_token[RTE_UUID_STRLEN - 1] = c;
+
+	/* Purge this vfio-pci specific token from the device arguments */
+	if (c != '\0') {

drivers/bus/pci/linux/pci_vfio.c pci_vfio_msix_is_mappable(int vfio_dev_fd, int msix_region) · line 693

Quoted 10 lines
drivers/bus/pci/linux/pci_vfio.c +10 -0
+		/* 1. Handle the case : 'vf_token=uuid,arg1=val1' */
+		memmove(p, vf_token + RTE_UUID_STRLEN,
+			strlen(vf_token + RTE_UUID_STRLEN) + 1);
+	} else {
+		/* 2. Handle the case : 'arg1=val1,vf_token=uuid' */
+		if (p != devargs->args)
+			p--;
+
+		*p = '\0';
+	}

Is it really required to purge? Why? If yes, it should be explained in the comment above.

drivers/bus/pci/linux/pci_vfio.c pci_vfio_msix_is_mappable(int vfio_dev_fd, int msix_region) · line 700

Quoted 7 lines
drivers/bus/pci/linux/pci_vfio.c +2 -0
+}
 
 static int
 pci_vfio_map_resource_primary(struct rte_pci_device *dev)
 {
 	struct vfio_device_info device_info = { .argsz = sizeof(device_info) };
+	rte_uuid_t vf_token = RTE_UUID_INIT(0, 0, 0, 0, 0ULL);

May be it would be better if vfio_pci_vf_token_arg() initializes it anyway instead of duplication init in two places?

Quoted 74 lines
drivers/bus/pci/linux/pci_vfio.c +5 -2
 	char pci_addr[PATH_MAX] = {0};
 	int vfio_dev_fd;
 	struct rte_pci_addr *loc = &dev->addr;
@@ -668,8 +717,9 @@ pci_vfio_map_resource_primary(struct rte_pci_device *dev)
 	snprintf(pci_addr, sizeof(pci_addr), PCI_PRI_FMT,
 			loc->domain, loc->bus, loc->devid, loc->function);
 
+	vfio_pci_vf_token_arg(dev->device.devargs, vf_token);
 	ret = rte_vfio_setup_device(rte_pci_get_sysfs_path(), pci_addr,
-					&vfio_dev_fd, &device_info);
+					&vfio_dev_fd, &device_info, vf_token);
 	if (ret)
 		return ret;
 
@@ -797,6 +847,7 @@ static int
 pci_vfio_map_resource_secondary(struct rte_pci_device *dev)
 {
 	struct vfio_device_info device_info = { .argsz = sizeof(device_info) };
+	rte_uuid_t vf_token = RTE_UUID_INIT(0, 0, 0, 0, 0ULL);
 	char pci_addr[PATH_MAX] = {0};
 	int vfio_dev_fd;
 	struct rte_pci_addr *loc = &dev->addr;
@@ -830,8 +881,9 @@ pci_vfio_map_resource_secondary(struct rte_pci_device *dev)
 		return -1;
 	}
 
+	vfio_pci_vf_token_arg(dev->device.devargs, vf_token);
 	ret = rte_vfio_setup_device(rte_pci_get_sysfs_path(), pci_addr,
-					&vfio_dev_fd, &device_info);
+					&vfio_dev_fd, &device_info, vf_token);
 	if (ret)
 		return ret;
 
lib/librte_eal/freebsd/eal.c +2 -1
index 6ae37e7e6..a92584795 100644
@@ -995,7 +995,8 @@ rte_eal_vfio_intr_mode(void)
 int rte_vfio_setup_device(__rte_unused const char *sysfs_base,
 		      __rte_unused const char *dev_addr,
 		      __rte_unused int *vfio_dev_fd,
-		      __rte_unused struct vfio_device_info *device_info)
+		      __rte_unused struct vfio_device_info *device_info,
+		      __rte_unused rte_uuid_t vf_token)
 {
 	return -1;
 }
lib/librte_eal/include/rte_uuid.h +2 -0
index 044afbdfa..8b42e070a 100644
@@ -15,6 +15,8 @@ extern "C" {
 #endif
 
 #include <stdbool.h>
+#include <stddef.h>
+#include <string.h>
 
 /**
  * Struct describing a Universal Unique Identifier
lib/librte_eal/include/rte_vfio.h +2 -0
index 20ed8c45a..1f9e22d82 100644
@@ -16,6 +16,8 @@ extern "C" {
 
 #include <stdint.h>
 
+#include <rte_uuid.h>
+
 /*
  * determine if VFIO is present on the system
  */

lib/librte_eal/include/rte_vfio.h struct vfio_device_info; · line 108

Quoted 5 lines
lib/librte_eal/include/rte_vfio.h +2 -0
@@ -102,13 +104,17 @@ struct vfio_device_info;
  * @param device_info
  *   Device information.
  *
+ * @param vf_token
+ *   VF token.

Such comments are useles and just eat space adding nothing useful. Please, make it useful and explain what is behind the parameter, when it is necessary, why? Should it be specified for PF case, VF case, both?

lib/librte_eal/include/rte_vfio.h struct vfio_device_info; · line 117

Quoted 10 lines
lib/librte_eal/include/rte_vfio.h +3 -1
+ *
  * @return
  *   0 on success.
  *   <0 on failure.
  *   >1 if the device cannot be managed this way.
  */
 int rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
-		int *vfio_dev_fd, struct vfio_device_info *device_info);
+		int *vfio_dev_fd, struct vfio_device_info *device_info,
+		rte_uuid_t vf_token);

"rte_uuid_t vf_token" looks confusing. Shouldn't it be "rte_uuid_t *vf_token"?

Quoted 18 lines
lib/librte_eal/include/rte_vfio.h +0 -0
 /**
  * Release a device mapped to a VFIO-managed I/O MMU group.
lib/librte_eal/linux/eal_vfio.c +2 -1
index 4502aefed..916082b5d 100644
@@ -702,7 +702,8 @@ rte_vfio_clear_group(int vfio_group_fd)
 
 int
 rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
-		int *vfio_dev_fd, struct vfio_device_info *device_info)
+		int *vfio_dev_fd, struct vfio_device_info *device_info,
+		rte_uuid_t vf_token)
 {
 	struct vfio_group_status group_status = {
 			.argsz = sizeof(group_status)

lib/librte_eal/linux/eal_vfio.c rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr, · line 716

Quoted 4 lines
lib/librte_eal/linux/eal_vfio.c +1 -0
@@ -712,6 +713,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 	int vfio_container_fd;
 	int vfio_group_fd;
 	int iommu_group_num;
+	char dev[PATH_MAX];

Why PATH_MAX?

Quoted 35 lines
lib/librte_eal/linux/eal_vfio.c +14 -2
 	int i, ret;
 
 	/* get group number */
@@ -895,8 +897,19 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 				t->type_id, t->name);
 	}
 
+	if (!rte_uuid_is_null(vf_token)) {
+		char vf_token_str[RTE_UUID_STRLEN];
+
+		rte_uuid_unparse(vf_token, vf_token_str, sizeof(vf_token_str));
+		snprintf(dev, sizeof(dev),
+			 "%s vf_token=%s", dev_addr, vf_token_str);
+	} else {
+		snprintf(dev, sizeof(dev),
+			 "%s", dev_addr);
+	}
+
 	/* get a file descriptor for the device */
-	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev_addr);
+	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev);
 	if (*vfio_dev_fd < 0) {
 		/* if we cannot get a device fd, this implies a problem with
 		 * the VFIO group or the container not having IOMMU configured.
@@ -2081,7 +2094,8 @@ int
 rte_vfio_setup_device(__rte_unused const char *sysfs_base,
 		__rte_unused const char *dev_addr,
 		__rte_unused int *vfio_dev_fd,
-		__rte_unused struct vfio_device_info *device_info)
+		__rte_unused struct vfio_device_info *device_info,
+		__rte_unused rte_uuid_t vf_token)
 {
 	return -1;
 }
#15 lore raw

Re: [dpdk-dev] [PATCH v4] eal: add VFIO-PCI SR-IOV support

Hi Andrew,

Quoted 109 lines

-----Original Message-----
From: Andrew Rybchenko <[email protected]>
Sent: Monday, April 13, 2020 23:38
To: Wang, Haiyue <[email protected]>; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]
Subject: Re: [dpdk-dev] [PATCH v4] eal: add VFIO-PCI SR-IOV support

Quoted 10 lines · Haiyue Wang

The kernel module vfio-pci introduces the VF token to enable SR-IOV support since 5.7.

The VF token can be set by a vfio-pci based PF driver and must be known by the vfio-pci based VF driver in order to gain access to the device.

An example VF token option would take this form:

1. Install vfio-pci with option 'enable_sriov=1'

Quoted 8 lines

2. ./usertools/dpdk-devbind.py -b vfio-pci 0000:87:00.0

3. echo 2 > /sys/bus/pci/devices/0000:87:00.0/sriov_numvfs

4. Start the PF:

  ./x86_64-native-linux-gcc/app/testpmd -l 22-25 -n 4 \
         -w 87:00.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 \
         --file-prefix=pf -- -i

Should I get a token from my head? Any?

Quoted 80 lines

5. Start the VF:

   ./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 \
         -w 87:02.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 \
         --file-prefix=vf1 -- -i

---
v4: 1. Ignore rte_vfio_setup_device ABI check since it is
for Linux driver use.

v3: https://patchwork.dpdk.org/patch/68254/
Fix the Travis build failed:
(1). rte_uuid.h:97:55: error: unknown type name ‘size_t’ (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’

v2: https://patchwork.dpdk.org/patch/68240/
Fix the FreeBSD build error.

v1: https://patchwork.dpdk.org/patch/68237/
Update the commit message.

RFC v2: https://patchwork.dpdk.org/patch/68114/
Based on Vamsi's RFC v1, and Alex's patch for Qemu
[https://lore.kernel.org/lkml/[email protected]/]:
Use the devarg to pass-down the VF token.

RFC v1: https://patchwork.dpdk.org/patch/66281/ by Vamsi.

---
 devtools/libabigail.abignore      |  3 ++
 drivers/bus/pci/linux/pci_vfio.c  | 56 +++++++++++++++++++++++++++++--
 lib/librte_eal/freebsd/eal.c      |  3 +-
 lib/librte_eal/include/rte_uuid.h |  2 ++
 lib/librte_eal/include/rte_vfio.h |  8 ++++-
 lib/librte_eal/linux/eal_vfio.c   | 20 +++++++++--
 6 files changed, 85 insertions(+), 7 deletions(-)
devtools/libabigail.abignore +3 -0
index a59df8f13..d918746b4 100644
@@ -11,3 +11,6 @@
         type_kind = enum
         name = rte_crypto_asym_xform_type
         changed_enumerators = RTE_CRYPTO_ASYM_XFORM_TYPE_LIST_END
+; Ignore this function which is only relevant to linux for driver
+[suppress_type]
+	name = rte_vfio_setup_device
drivers/bus/pci/linux/pci_vfio.c +17 -0
index 64cd84a68..7f99337c7 100644
@@ -11,6 +11,7 @@
 #include <sys/mman.h>
 #include <stdbool.h>
 
+#include <rte_devargs.h>
 #include <rte_log.h>
 #include <rte_pci.h>
 #include <rte_bus_pci.h>
@@ -644,11 +645,59 @@ pci_vfio_msix_is_mappable(int vfio_dev_fd, int msix_region)
 	return ret;
 }
 
+static void
+vfio_pci_vf_token_arg(struct rte_devargs *devargs, rte_uuid_t uu)
+{
+#define VF_TOKEN_ARG "vf_token="
+	char c, *p, *vf_token;
+
+	if (devargs == NULL)
+		return;
+
+	p = strstr(devargs->args, VF_TOKEN_ARG);
+	if (!p)
+		return;
+
+	vf_token = p + strlen(VF_TOKEN_ARG);
+	if (strlen(vf_token) < (RTE_UUID_STRLEN - 1))
+		return;
Quoted 9 lines

drivers/bus/pci/linux/pci_vfio.c pci_vfio_msix_is_mappable(int vfio_dev_fd, int msix_region) · line 669

Quoted 6 lines
drivers/bus/pci/linux/pci_vfio.c +6 -0
+
+	c = vf_token[RTE_UUID_STRLEN - 1];
+	if (c != '\0' && c != ',')
+		return;
+
+	vf_token[RTE_UUID_STRLEN - 1] = '\0';

Is it possible to parse and handle devargs using rte_kvargs.h?

Since it needs to remove the 'vf_token', as 'vf_token' is not a valid PMD related args, so need to parse and delete it.

rte_kvargs_parse(const char *args, const char * const valid_keys[])

Quoted 10 lines

drivers/bus/pci/linux/pci_vfio.c pci_vfio_msix_is_mappable(int vfio_dev_fd, int msix_region) · line 674

Quoted 5 lines
drivers/bus/pci/linux/pci_vfio.c +5 -0
+	if (rte_uuid_parse(vf_token, uu)) {
+		RTE_LOG(ERR, EAL,
+			"The VF token is not a valid uuid : %s\n", vf_token);
+		vf_token[RTE_UUID_STRLEN - 1] = c;
+		return;

I think that the function must return error which is handled by the caller when something bad happens (e.g. invalid UUID).

Yes, make sense, will add the error handling.

Quoted 14 lines
Quoted 14 lines
drivers/bus/pci/linux/pci_vfio.c +14 -0
+	}
+
+	RTE_LOG(DEBUG, EAL,
+		"The VF token is found : %s\n", vf_token);
+
+	vf_token[RTE_UUID_STRLEN - 1] = c;
+
+	/* Purge this vfio-pci specific token from the device arguments */
+	if (c != '\0') {
+		/* 1. Handle the case : 'vf_token=uuid,arg1=val1' */
+		memmove(p, vf_token + RTE_UUID_STRLEN,
+			strlen(vf_token + RTE_UUID_STRLEN) + 1);
+	} else {
+		/* 2. Handle the case : 'arg1=val1,vf_token=uuid' */
Quoted 8 lines

drivers/bus/pci/linux/pci_vfio.c pci_vfio_msix_is_mappable(int vfio_dev_fd, int msix_region) · line 693

Quoted 5 lines
drivers/bus/pci/linux/pci_vfio.c +5 -0
+		if (p != devargs->args)
+			p--;
+
+		*p = '\0';
+	}

Is it really required to purge? Why? If yes, it should be explained in the comment above.

Please see above reply.

Quoted 12 lines

drivers/bus/pci/linux/pci_vfio.c pci_vfio_msix_is_mappable(int vfio_dev_fd, int msix_region) · line 700

Quoted 8 lines
drivers/bus/pci/linux/pci_vfio.c +2 -0
+}
 
 static int
 pci_vfio_map_resource_primary(struct rte_pci_device *dev)
 {
 	struct vfio_device_info device_info = { .argsz = sizeof(device_info) };
+	rte_uuid_t vf_token = RTE_UUID_INIT(0, 0, 0, 0, 0ULL);

May be it would be better if vfio_pci_vf_token_arg() initializes it anyway instead of duplication init in two places?

+1, will update it.

Quoted 69 lines
Quoted 69 lines
drivers/bus/pci/linux/pci_vfio.c +5 -2
 	char pci_addr[PATH_MAX] = {0};
 	int vfio_dev_fd;
 	struct rte_pci_addr *loc = &dev->addr;
@@ -668,8 +717,9 @@ pci_vfio_map_resource_primary(struct rte_pci_device *dev)
 	snprintf(pci_addr, sizeof(pci_addr), PCI_PRI_FMT,
 			loc->domain, loc->bus, loc->devid, loc->function);
 
+	vfio_pci_vf_token_arg(dev->device.devargs, vf_token);
 	ret = rte_vfio_setup_device(rte_pci_get_sysfs_path(), pci_addr,
-					&vfio_dev_fd, &device_info);
+					&vfio_dev_fd, &device_info, vf_token);
 	if (ret)
 		return ret;
 
@@ -797,6 +847,7 @@ static int
 pci_vfio_map_resource_secondary(struct rte_pci_device *dev)
 {
 	struct vfio_device_info device_info = { .argsz = sizeof(device_info) };
+	rte_uuid_t vf_token = RTE_UUID_INIT(0, 0, 0, 0, 0ULL);
 	char pci_addr[PATH_MAX] = {0};
 	int vfio_dev_fd;
 	struct rte_pci_addr *loc = &dev->addr;
@@ -830,8 +881,9 @@ pci_vfio_map_resource_secondary(struct rte_pci_device *dev)
 		return -1;
 	}
 
+	vfio_pci_vf_token_arg(dev->device.devargs, vf_token);
 	ret = rte_vfio_setup_device(rte_pci_get_sysfs_path(), pci_addr,
-					&vfio_dev_fd, &device_info);
+					&vfio_dev_fd, &device_info, vf_token);
 	if (ret)
 		return ret;
 
lib/librte_eal/freebsd/eal.c +2 -1
index 6ae37e7e6..a92584795 100644
@@ -995,7 +995,8 @@ rte_eal_vfio_intr_mode(void)
 int rte_vfio_setup_device(__rte_unused const char *sysfs_base,
 		      __rte_unused const char *dev_addr,
 		      __rte_unused int *vfio_dev_fd,
-		      __rte_unused struct vfio_device_info *device_info)
+		      __rte_unused struct vfio_device_info *device_info,
+		      __rte_unused rte_uuid_t vf_token)
 {
 	return -1;
 }
lib/librte_eal/include/rte_uuid.h +2 -0
index 044afbdfa..8b42e070a 100644
@@ -15,6 +15,8 @@ extern "C" {
 #endif
 
 #include <stdbool.h>
+#include <stddef.h>
+#include <string.h>
 
 /**
  * Struct describing a Universal Unique Identifier
lib/librte_eal/include/rte_vfio.h +0 -0
index 20ed8c45a..1f9e22d82 100644
@@ -16,6 +16,8 @@ extern "C" {
 
 #include <stdint.h>
 
Quoted 17 lines

lib/librte_eal/include/rte_vfio.h struct vfio_device_info; · line 108

Quoted 11 lines
lib/librte_eal/include/rte_vfio.h +4 -0
+#include <rte_uuid.h>
+
 /*
  * determine if VFIO is present on the system
  */
@@ -102,13 +104,17 @@ struct vfio_device_info;
  * @param device_info
  *   Device information.
  *
+ * @param vf_token
+ *   VF token.

Such comments are useles and just eat space adding nothing useful. Please, make it useful and explain what is behind the parameter, when it is necessary, why? Should it be specified for PF case, VF case, both?

Will add more comments, yes for both PF and VF, as Alex's linux patch explained.

Quoted 5 lines
Quoted 5 lines
lib/librte_eal/include/rte_vfio.h +1 -0
+ *
  * @return
  *   0 on success.
  *   <0 on failure.
  *   >1 if the device cannot be managed this way.
Quoted 8 lines

lib/librte_eal/include/rte_vfio.h struct vfio_device_info; · line 117

Quoted 5 lines
lib/librte_eal/include/rte_vfio.h +2 -1
  */
 int rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
-		int *vfio_dev_fd, struct vfio_device_info *device_info);
+		int *vfio_dev_fd, struct vfio_device_info *device_info,
+		rte_uuid_t vf_token);

"rte_uuid_t vf_token" looks confusing. Shouldn't it be "rte_uuid_t *vf_token"?

This is UUID API design and type definition:

bool rte_uuid_is_null(const rte_uuid_t uu);

DPDK: typedef unsigned char rte_uuid_t[16];

vs

Linux: typedef struct {
	__u8 b[UUID_SIZE];
} uuid_t;
Quoted 10 lines
Quoted 10 lines
lib/librte_eal/include/rte_vfio.h +0 -0
 /**
  * Release a device mapped to a VFIO-managed I/O MMU group.
lib/librte_eal/linux/eal_vfio.c +0 -0
index 4502aefed..916082b5d 100644
@@ -702,7 +702,8 @@ rte_vfio_clear_group(int vfio_group_fd)
 
Quoted 15 lines

lib/librte_eal/linux/eal_vfio.c rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr, · line 716

Quoted 13 lines
lib/librte_eal/linux/eal_vfio.c +3 -1
 int
 rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
-		int *vfio_dev_fd, struct vfio_device_info *device_info)
+		int *vfio_dev_fd, struct vfio_device_info *device_info,
+		rte_uuid_t vf_token)
 {
 	struct vfio_group_status group_status = {
 			.argsz = sizeof(group_status)
@@ -712,6 +713,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 	int vfio_container_fd;
 	int vfio_group_fd;
 	int iommu_group_num;
+	char dev[PATH_MAX];

Why PATH_MAX?

Based on Vamsi's RFC v1, and found that it looked a little reasonable, ' char pci_addr[PATH_MAX] = {0}; '

static int
pci_vfio_map_resource_primary(struct rte_pci_device *dev)

{
	struct vfio_device_info device_info = { .argsz = sizeof(device_info) };
	char pci_addr[PATH_MAX] = {0}; <----
Quoted 36 lines
Quoted 36 lines
lib/librte_eal/linux/eal_vfio.c +14 -2
 	int i, ret;
 
 	/* get group number */
@@ -895,8 +897,19 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 				t->type_id, t->name);
 	}
 
+	if (!rte_uuid_is_null(vf_token)) {
+		char vf_token_str[RTE_UUID_STRLEN];
+
+		rte_uuid_unparse(vf_token, vf_token_str, sizeof(vf_token_str));
+		snprintf(dev, sizeof(dev),
+			 "%s vf_token=%s", dev_addr, vf_token_str);
+	} else {
+		snprintf(dev, sizeof(dev),
+			 "%s", dev_addr);
+	}
+
 	/* get a file descriptor for the device */
-	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev_addr);
+	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev);
 	if (*vfio_dev_fd < 0) {
 		/* if we cannot get a device fd, this implies a problem with
 		 * the VFIO group or the container not having IOMMU configured.
@@ -2081,7 +2094,8 @@ int
 rte_vfio_setup_device(__rte_unused const char *sysfs_base,
 		__rte_unused const char *dev_addr,
 		__rte_unused int *vfio_dev_fd,
-		__rte_unused struct vfio_device_info *device_info)
+		__rte_unused struct vfio_device_info *device_info,
+		__rte_unused rte_uuid_t vf_token)
 {
 	return -1;
 }
#16 lore raw

v5 cover support for VFIO-PCI VF token interface

In reply to [email protected]

v5: 1. Add the VF token parse error handling.
2. Split into two patches for different logic module.
3. Add more comments into the code for explaining the design.
4. Drop the ABI change workaround, this patch set focuses on code review.

v4: 1. Ignore rte_vfio_setup_device ABI check since it is
for Linux driver use.
https://patchwork.dpdk.org/patch/68255/

v3: Fix the Travis build failed:
(1). rte_uuid.h:97:55: error: unknown type name ‘size_t’ (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’
https://patchwork.dpdk.org/patch/68254/

v2: Fix the FreeBSD build error.

         https://patchwork.dpdk.org/patch/68240/

v1: Update the commit message.

        https://patchwork.dpdk.org/patch/68237/

RFC v2: https://patchwork.dpdk.org/patch/68114/
Based on Vamsi's RFC v1, and Alex's patch for Qemu
[https://lore.kernel.org/lkml/[email protected]/]:
Use the devarg to pass-down the VF token.

RFC v1: https://patchwork.dpdk.org/patch/66281/ by Vamsi.

Haiyue Wang (2):
eal: add uuid dependent header files explicitly
eal: support for VFIO-PCI VF token

 drivers/bus/pci/linux/pci_vfio.c  | 74 ++++++++++++++++++++++++++++++-
 lib/librte_eal/freebsd/eal.c      |  3 +-
 lib/librte_eal/include/rte_uuid.h |  2 +
 lib/librte_eal/include/rte_vfio.h | 21 ++++++++-
 lib/librte_eal/linux/eal_vfio.c   | 20 +++++++--
 5 files changed, 113 insertions(+), 7 deletions(-)
Signature · 1 line
2.26.0
#17 lore raw

PATCH v5 1/2 eal: add uuid dependent header files explicitly

Add the dependent header files explicitly, so that the user just needs to include the 'rte_uuid.h' header file directly to avoid compile error:
(1). rte_uuid.h:97:55: error: unknown type name ‘size_t’ (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’

Fixes 6bc67c497a51 ("eal: add uuid API")
---
 lib/librte_eal/include/rte_uuid.h | 2 ++
 1 file changed, 2 insertions(+)
lib/librte_eal/include/rte_uuid.h +2 -0
index 044afbdfa..8b42e070a 100644
@@ -15,6 +15,8 @@ extern "C" {
 #endif
 
 #include <stdbool.h>
+#include <stddef.h>
+#include <string.h>
 
 /**
  * Struct describing a Universal Unique Identifier
Signature · 1 line
2.26.0
#18 lore raw

PATCH v5 2/2 eal: support for VFIO-PCI VF token

In reply to Haiyue Wang

The kernel module vfio-pci introduces the VF token to enable SR-IOV support since 5.7.

The VF token can be set by a vfio-pci based PF driver and must be known by the vfio-pci based VF driver in order to gain access to the device.

An example VF token option would take this form:

1. Install vfio-pci with option 'enable_sriov=1'

2. ./usertools/dpdk-devbind.py -b vfio-pci 0000:87:00.0

3. echo 2 > /sys/bus/pci/devices/0000:87:00.0/sriov_numvfs

4. Start the PF:

  ./x86_64-native-linux-gcc/app/testpmd -l 22-25 -n 4 \
         -w 87:00.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 \
         --file-prefix=pf -- -i

5. Start the VF:

   ./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 \
         -w 87:02.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 \
         --file-prefix=vf1 -- -i
---
 drivers/bus/pci/linux/pci_vfio.c  | 74 ++++++++++++++++++++++++++++++-
 lib/librte_eal/freebsd/eal.c      |  3 +-
 lib/librte_eal/include/rte_vfio.h | 21 ++++++++-
 lib/librte_eal/linux/eal_vfio.c   | 20 +++++++--
 4 files changed, 111 insertions(+), 7 deletions(-)
drivers/bus/pci/linux/pci_vfio.c +72 -2
index 64cd84a68..efb64e2ba 100644
@@ -11,6 +11,7 @@
 #include <sys/mman.h>
 #include <stdbool.h>
 
+#include <rte_devargs.h>
 #include <rte_log.h>
 #include <rte_pci.h>
 #include <rte_bus_pci.h>
@@ -644,12 +645,72 @@ pci_vfio_msix_is_mappable(int vfio_dev_fd, int msix_region)
 	return ret;
 }
 
+static int
+vfio_pci_vf_token_arg(struct rte_devargs *devargs, rte_uuid_t uuid)
+{
+#define VF_TOKEN_ARG "vf_token="
+	char c, *p, *vf_token;
+
+	memset(uuid, 0, sizeof(rte_uuid_t));
+
+	if (devargs == NULL)
+		return 0;
+
+	p = strstr(devargs->args, VF_TOKEN_ARG);
+	if (!p)
+		return 0;
+
+	vf_token = p + strlen(VF_TOKEN_ARG);
+	if (strlen(vf_token) < (RTE_UUID_STRLEN - 1)) {
+		RTE_LOG(ERR, EAL, "The VF token length is too short\n");
+		return -1;
+	}
+
+	c = vf_token[RTE_UUID_STRLEN - 1];
+	if (c != '\0' && c != ',') {
+		RTE_LOG(ERR, EAL,
+			"The VF token ends with a invalid character : %c\n", c);
+		return -1;
+	}
+
+	vf_token[RTE_UUID_STRLEN - 1] = '\0';
+	if (rte_uuid_parse(vf_token, uuid)) {
+		RTE_LOG(ERR, EAL,
+			"The VF token is invalid : %s\n", vf_token);
+		vf_token[RTE_UUID_STRLEN - 1] = c;
+		return -1;
+	}
+
+	RTE_LOG(DEBUG, EAL,
+		"The VF token is found : %s\n", vf_token);
+
+	vf_token[RTE_UUID_STRLEN - 1] = c;
+
+	/* This VF token will be treated as a invalid device argument if the
+	 * PMD calls the rte_devargs parse API with its own valid argument list,
+	 * so it needs to purge this vfio-pci specific argument.
+	 */
+	if (c != '\0') {
+		/* 1. Handle the case : 'vf_token=uuid,arg1=val1' */
+		memmove(p, vf_token + RTE_UUID_STRLEN,
+			strlen(vf_token + RTE_UUID_STRLEN) + 1);
+	} else {
+		/* 2. Handle the case : 'arg1=val1,vf_token=uuid' */
+		if (p != devargs->args)
+			p--;
+
+		*p = '\0';
+	}
+
+	return 0;
+}
 
 static int
 pci_vfio_map_resource_primary(struct rte_pci_device *dev)
 {
 	struct vfio_device_info device_info = { .argsz = sizeof(device_info) };
 	char pci_addr[PATH_MAX] = {0};
+	rte_uuid_t vf_token;
 	int vfio_dev_fd;
 	struct rte_pci_addr *loc = &dev->addr;
 	int i, ret;
@@ -668,8 +729,12 @@ pci_vfio_map_resource_primary(struct rte_pci_device *dev)
 	snprintf(pci_addr, sizeof(pci_addr), PCI_PRI_FMT,
 			loc->domain, loc->bus, loc->devid, loc->function);
 
+	ret = vfio_pci_vf_token_arg(dev->device.devargs, vf_token);
+	if (ret)
+		return ret;
+
 	ret = rte_vfio_setup_device(rte_pci_get_sysfs_path(), pci_addr,
-					&vfio_dev_fd, &device_info);
+					&vfio_dev_fd, &device_info, vf_token);
 	if (ret)
 		return ret;
 
@@ -798,6 +863,7 @@ pci_vfio_map_resource_secondary(struct rte_pci_device *dev)
 {
 	struct vfio_device_info device_info = { .argsz = sizeof(device_info) };
 	char pci_addr[PATH_MAX] = {0};
+	rte_uuid_t vf_token;
 	int vfio_dev_fd;
 	struct rte_pci_addr *loc = &dev->addr;
 	int i, ret;
@@ -830,8 +896,12 @@ pci_vfio_map_resource_secondary(struct rte_pci_device *dev)
 		return -1;
 	}
 
+	ret = vfio_pci_vf_token_arg(dev->device.devargs, vf_token);
+	if (ret)
+		return ret;
+
 	ret = rte_vfio_setup_device(rte_pci_get_sysfs_path(), pci_addr,
-					&vfio_dev_fd, &device_info);
+					&vfio_dev_fd, &device_info, vf_token);
 	if (ret)
 		return ret;
 
lib/librte_eal/freebsd/eal.c +2 -1
index 6ae37e7e6..a92584795 100644
@@ -995,7 +995,8 @@ rte_eal_vfio_intr_mode(void)
 int rte_vfio_setup_device(__rte_unused const char *sysfs_base,
 		      __rte_unused const char *dev_addr,
 		      __rte_unused int *vfio_dev_fd,
-		      __rte_unused struct vfio_device_info *device_info)
+		      __rte_unused struct vfio_device_info *device_info,
+		      __rte_unused rte_uuid_t vf_token)
 {
 	return -1;
 }
lib/librte_eal/include/rte_vfio.h +20 -1
index 20ed8c45a..ec9de23dc 100644
@@ -16,6 +16,8 @@ extern "C" {
 
 #include <stdint.h>
 
+#include <rte_uuid.h>
+
 /*
  * determine if VFIO is present on the system
  */
@@ -102,13 +104,30 @@ struct vfio_device_info;
  * @param device_info
  *   Device information.
  *
+ * @param vf_token
+ *   Before linux 5.7, the PF bound to vfio-pci doesn't support SR-IOV to
+ *   create VFs for security reason. Now the VF token is introduced to work
+ *   as some degree of trust or collaboration between PF and VFs.
+ *
+ *   A). as VF device, if the PF is a vfio device and it is bound to the
+ *   vfio-pci driver, the user needs to provide a VF token to access the
+ *   device, in the form of appending a vf_token to the device name, for
+ *   example:
+ *     "0000:04:10.0 vf_token=bd8d9d2b-5a5f-4f5a-a211-f591514ba1f3"
+ *
+ *   B). as PF device, When presented with a PF which has VFs in use, the
+ *   user must also provide the current VF token to prove collaboration with
+ *   existing VF users.  If VFs are not in use, the VF token provided for the
+ *   PF device will act to set the VF token.
+ *
  * @return
  *   0 on success.
  *   <0 on failure.
  *   >1 if the device cannot be managed this way.
  */
 int rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
-		int *vfio_dev_fd, struct vfio_device_info *device_info);
+		int *vfio_dev_fd, struct vfio_device_info *device_info,
+		rte_uuid_t vf_token);
 
 /**
  * Release a device mapped to a VFIO-managed I/O MMU group.
lib/librte_eal/linux/eal_vfio.c +17 -3
index 4502aefed..916082b5d 100644
@@ -702,7 +702,8 @@ rte_vfio_clear_group(int vfio_group_fd)
 
 int
 rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
-		int *vfio_dev_fd, struct vfio_device_info *device_info)
+		int *vfio_dev_fd, struct vfio_device_info *device_info,
+		rte_uuid_t vf_token)
 {
 	struct vfio_group_status group_status = {
 			.argsz = sizeof(group_status)
@@ -712,6 +713,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 	int vfio_container_fd;
 	int vfio_group_fd;
 	int iommu_group_num;
+	char dev[PATH_MAX];
 	int i, ret;
 
 	/* get group number */
@@ -895,8 +897,19 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 				t->type_id, t->name);
 	}
 
+	if (!rte_uuid_is_null(vf_token)) {
+		char vf_token_str[RTE_UUID_STRLEN];
+
+		rte_uuid_unparse(vf_token, vf_token_str, sizeof(vf_token_str));
+		snprintf(dev, sizeof(dev),
+			 "%s vf_token=%s", dev_addr, vf_token_str);
+	} else {
+		snprintf(dev, sizeof(dev),
+			 "%s", dev_addr);
+	}
+
 	/* get a file descriptor for the device */
-	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev_addr);
+	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev);
 	if (*vfio_dev_fd < 0) {
 		/* if we cannot get a device fd, this implies a problem with
 		 * the VFIO group or the container not having IOMMU configured.
@@ -2081,7 +2094,8 @@ int
 rte_vfio_setup_device(__rte_unused const char *sysfs_base,
 		__rte_unused const char *dev_addr,
 		__rte_unused int *vfio_dev_fd,
-		__rte_unused struct vfio_device_info *device_info)
+		__rte_unused struct vfio_device_info *device_info,
+		__rte_unused rte_uuid_t vf_token)
 {
 	return -1;
 }
Signature · 1 line
2.26.0
#19 lore raw

v6 cover support for VFIO-PCI VF token interface

In reply to [email protected]

v6: Drop the Fixes tag in uuid, since the file has been
moved to another place, not suitable to apply on stable. And this is not a bug, just some kind of enhancement.

v5: 1. Add the VF token parse error handling.
2. Split into two patches for different logic module.
3. Add more comments into the code for explaining the design.
4. Drop the ABI change workaround, this patch set focuses on code review.
https://patchwork.dpdk.org/cover/68364/

v4: 1. Ignore rte_vfio_setup_device ABI check since it is
for Linux driver use.
https://patchwork.dpdk.org/patch/68255/

v3: Fix the Travis build failed:
(1). rte_uuid.h:97:55: error: unknown type name ‘size_t’ (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’
https://patchwork.dpdk.org/patch/68254/

v2: Fix the FreeBSD build error.

         https://patchwork.dpdk.org/patch/68240/

v1: Update the commit message.

        https://patchwork.dpdk.org/patch/68237/

RFC v2: https://patchwork.dpdk.org/patch/68114/
Based on Vamsi's RFC v1, and Alex's patch for Qemu
[https://lore.kernel.org/lkml/[email protected]/]:
Use the devarg to pass-down the VF token.

RFC v1: https://patchwork.dpdk.org/patch/66281/ by Vamsi.

Haiyue Wang (2):
eal: add uuid dependent header files explicitly
eal: support for VFIO-PCI VF token

 drivers/bus/pci/linux/pci_vfio.c  | 74 ++++++++++++++++++++++++++++++-
 lib/librte_eal/freebsd/eal.c      |  3 +-
 lib/librte_eal/include/rte_uuid.h |  2 +
 lib/librte_eal/include/rte_vfio.h | 21 ++++++++-
 lib/librte_eal/linux/eal_vfio.c   | 20 +++++++--
 5 files changed, 113 insertions(+), 7 deletions(-)
Signature · 1 line
2.26.0
#20 lore raw

PATCH v6 1/2 eal: add uuid dependent header files explicitly

Add the dependent header files explicitly, so that the user just needs to include the 'rte_uuid.h' header file directly to avoid compile error:
(1). rte_uuid.h:97:55: error: unknown type name ‘size_t’ (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’

---
 lib/librte_eal/include/rte_uuid.h | 2 ++
 1 file changed, 2 insertions(+)
lib/librte_eal/include/rte_uuid.h +2 -0
index 044afbdfa..8b42e070a 100644
@@ -15,6 +15,8 @@ extern "C" {
 #endif
 
 #include <stdbool.h>
+#include <stddef.h>
+#include <string.h>
 
 /**
  * Struct describing a Universal Unique Identifier
Signature · 1 line
2.26.0
#21 lore raw

PATCH v6 2/2 eal: support for VFIO-PCI VF token

In reply to Haiyue Wang

The kernel module vfio-pci introduces the VF token to enable SR-IOV support since 5.7.

The VF token can be set by a vfio-pci based PF driver and must be known by the vfio-pci based VF driver in order to gain access to the device.

An example VF token option would take this form:

1. Install vfio-pci with option 'enable_sriov=1'

2. ./usertools/dpdk-devbind.py -b vfio-pci 0000:87:00.0

3. echo 2 > /sys/bus/pci/devices/0000:87:00.0/sriov_numvfs

4. Start the PF:

  ./x86_64-native-linux-gcc/app/testpmd -l 22-25 -n 4 \
         -w 87:00.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 \
         --file-prefix=pf -- -i

5. Start the VF:

   ./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 \
         -w 87:02.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 \
         --file-prefix=vf1 -- -i
---
 drivers/bus/pci/linux/pci_vfio.c  | 74 ++++++++++++++++++++++++++++++-
 lib/librte_eal/freebsd/eal.c      |  3 +-
 lib/librte_eal/include/rte_vfio.h | 21 ++++++++-
 lib/librte_eal/linux/eal_vfio.c   | 20 +++++++--
 4 files changed, 111 insertions(+), 7 deletions(-)
drivers/bus/pci/linux/pci_vfio.c +72 -2
index 64cd84a68..efb64e2ba 100644
@@ -11,6 +11,7 @@
 #include <sys/mman.h>
 #include <stdbool.h>
 
+#include <rte_devargs.h>
 #include <rte_log.h>
 #include <rte_pci.h>
 #include <rte_bus_pci.h>
@@ -644,12 +645,72 @@ pci_vfio_msix_is_mappable(int vfio_dev_fd, int msix_region)
 	return ret;
 }
 
+static int
+vfio_pci_vf_token_arg(struct rte_devargs *devargs, rte_uuid_t uuid)
+{
+#define VF_TOKEN_ARG "vf_token="
+	char c, *p, *vf_token;
+
+	memset(uuid, 0, sizeof(rte_uuid_t));
+
+	if (devargs == NULL)
+		return 0;
+
+	p = strstr(devargs->args, VF_TOKEN_ARG);
+	if (!p)
+		return 0;
+
+	vf_token = p + strlen(VF_TOKEN_ARG);
+	if (strlen(vf_token) < (RTE_UUID_STRLEN - 1)) {
+		RTE_LOG(ERR, EAL, "The VF token length is too short\n");
+		return -1;
+	}
+
+	c = vf_token[RTE_UUID_STRLEN - 1];
+	if (c != '\0' && c != ',') {
+		RTE_LOG(ERR, EAL,
+			"The VF token ends with a invalid character : %c\n", c);
+		return -1;
+	}
+
+	vf_token[RTE_UUID_STRLEN - 1] = '\0';
+	if (rte_uuid_parse(vf_token, uuid)) {
+		RTE_LOG(ERR, EAL,
+			"The VF token is invalid : %s\n", vf_token);
+		vf_token[RTE_UUID_STRLEN - 1] = c;
+		return -1;
+	}
+
+	RTE_LOG(DEBUG, EAL,
+		"The VF token is found : %s\n", vf_token);
+
+	vf_token[RTE_UUID_STRLEN - 1] = c;
+
+	/* This VF token will be treated as a invalid device argument if the
+	 * PMD calls the rte_devargs parse API with its own valid argument list,
+	 * so it needs to purge this vfio-pci specific argument.
+	 */
+	if (c != '\0') {
+		/* 1. Handle the case : 'vf_token=uuid,arg1=val1' */
+		memmove(p, vf_token + RTE_UUID_STRLEN,
+			strlen(vf_token + RTE_UUID_STRLEN) + 1);
+	} else {
+		/* 2. Handle the case : 'arg1=val1,vf_token=uuid' */
+		if (p != devargs->args)
+			p--;
+
+		*p = '\0';
+	}
+
+	return 0;
+}
 
 static int
 pci_vfio_map_resource_primary(struct rte_pci_device *dev)
 {
 	struct vfio_device_info device_info = { .argsz = sizeof(device_info) };
 	char pci_addr[PATH_MAX] = {0};
+	rte_uuid_t vf_token;
 	int vfio_dev_fd;
 	struct rte_pci_addr *loc = &dev->addr;
 	int i, ret;
@@ -668,8 +729,12 @@ pci_vfio_map_resource_primary(struct rte_pci_device *dev)
 	snprintf(pci_addr, sizeof(pci_addr), PCI_PRI_FMT,
 			loc->domain, loc->bus, loc->devid, loc->function);
 
+	ret = vfio_pci_vf_token_arg(dev->device.devargs, vf_token);
+	if (ret)
+		return ret;
+
 	ret = rte_vfio_setup_device(rte_pci_get_sysfs_path(), pci_addr,
-					&vfio_dev_fd, &device_info);
+					&vfio_dev_fd, &device_info, vf_token);
 	if (ret)
 		return ret;
 
@@ -798,6 +863,7 @@ pci_vfio_map_resource_secondary(struct rte_pci_device *dev)
 {
 	struct vfio_device_info device_info = { .argsz = sizeof(device_info) };
 	char pci_addr[PATH_MAX] = {0};
+	rte_uuid_t vf_token;
 	int vfio_dev_fd;
 	struct rte_pci_addr *loc = &dev->addr;
 	int i, ret;
@@ -830,8 +896,12 @@ pci_vfio_map_resource_secondary(struct rte_pci_device *dev)
 		return -1;
 	}
 
+	ret = vfio_pci_vf_token_arg(dev->device.devargs, vf_token);
+	if (ret)
+		return ret;
+
 	ret = rte_vfio_setup_device(rte_pci_get_sysfs_path(), pci_addr,
-					&vfio_dev_fd, &device_info);
+					&vfio_dev_fd, &device_info, vf_token);
 	if (ret)
 		return ret;
 
lib/librte_eal/freebsd/eal.c +2 -1
index 6ae37e7e6..a92584795 100644
@@ -995,7 +995,8 @@ rte_eal_vfio_intr_mode(void)
 int rte_vfio_setup_device(__rte_unused const char *sysfs_base,
 		      __rte_unused const char *dev_addr,
 		      __rte_unused int *vfio_dev_fd,
-		      __rte_unused struct vfio_device_info *device_info)
+		      __rte_unused struct vfio_device_info *device_info,
+		      __rte_unused rte_uuid_t vf_token)
 {
 	return -1;
 }
lib/librte_eal/include/rte_vfio.h +20 -1
index 20ed8c45a..ec9de23dc 100644
@@ -16,6 +16,8 @@ extern "C" {
 
 #include <stdint.h>
 
+#include <rte_uuid.h>
+
 /*
  * determine if VFIO is present on the system
  */
@@ -102,13 +104,30 @@ struct vfio_device_info;
  * @param device_info
  *   Device information.
  *
+ * @param vf_token
+ *   Before linux 5.7, the PF bound to vfio-pci doesn't support SR-IOV to
+ *   create VFs for security reason. Now the VF token is introduced to work
+ *   as some degree of trust or collaboration between PF and VFs.
+ *
+ *   A). as VF device, if the PF is a vfio device and it is bound to the
+ *   vfio-pci driver, the user needs to provide a VF token to access the
+ *   device, in the form of appending a vf_token to the device name, for
+ *   example:
+ *     "0000:04:10.0 vf_token=bd8d9d2b-5a5f-4f5a-a211-f591514ba1f3"
+ *
+ *   B). as PF device, When presented with a PF which has VFs in use, the
+ *   user must also provide the current VF token to prove collaboration with
+ *   existing VF users.  If VFs are not in use, the VF token provided for the
+ *   PF device will act to set the VF token.
+ *
  * @return
  *   0 on success.
  *   <0 on failure.
  *   >1 if the device cannot be managed this way.
  */
 int rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
-		int *vfio_dev_fd, struct vfio_device_info *device_info);
+		int *vfio_dev_fd, struct vfio_device_info *device_info,
+		rte_uuid_t vf_token);
 
 /**
  * Release a device mapped to a VFIO-managed I/O MMU group.
lib/librte_eal/linux/eal_vfio.c +17 -3
index 4502aefed..916082b5d 100644
@@ -702,7 +702,8 @@ rte_vfio_clear_group(int vfio_group_fd)
 
 int
 rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
-		int *vfio_dev_fd, struct vfio_device_info *device_info)
+		int *vfio_dev_fd, struct vfio_device_info *device_info,
+		rte_uuid_t vf_token)
 {
 	struct vfio_group_status group_status = {
 			.argsz = sizeof(group_status)
@@ -712,6 +713,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 	int vfio_container_fd;
 	int vfio_group_fd;
 	int iommu_group_num;
+	char dev[PATH_MAX];
 	int i, ret;
 
 	/* get group number */
@@ -895,8 +897,19 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 				t->type_id, t->name);
 	}
 
+	if (!rte_uuid_is_null(vf_token)) {
+		char vf_token_str[RTE_UUID_STRLEN];
+
+		rte_uuid_unparse(vf_token, vf_token_str, sizeof(vf_token_str));
+		snprintf(dev, sizeof(dev),
+			 "%s vf_token=%s", dev_addr, vf_token_str);
+	} else {
+		snprintf(dev, sizeof(dev),
+			 "%s", dev_addr);
+	}
+
 	/* get a file descriptor for the device */
-	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev_addr);
+	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev);
 	if (*vfio_dev_fd < 0) {
 		/* if we cannot get a device fd, this implies a problem with
 		 * the VFIO group or the container not having IOMMU configured.
@@ -2081,7 +2094,8 @@ int
 rte_vfio_setup_device(__rte_unused const char *sysfs_base,
 		__rte_unused const char *dev_addr,
 		__rte_unused int *vfio_dev_fd,
-		__rte_unused struct vfio_device_info *device_info)
+		__rte_unused struct vfio_device_info *device_info,
+		__rte_unused rte_uuid_t vf_token)
 {
 	return -1;
 }
Signature · 1 line
2.26.0
author Tested-by on the v6 series
#22 lore raw

Re: [dpdk-dev] [EXT] [PATCH v6 0/2] support for VFIO-PCI VF token interface

In reply to Haiyue Wang
Quoted 90 lines

-----Original Message-----
From: Haiyue Wang <[email protected]>
Sent: Tuesday, April 14, 2020 8:51 AM
To: [email protected]; [email protected]; Vamsi Krishna Attunuru <[email protected]>; Jerin Jacob Kollanukkaran <[email protected]>; [email protected]; [email protected]
Cc: Haiyue Wang <[email protected]>
Subject: [EXT] [PATCH v6 0/2] support for VFIO-PCI VF token interface

External Email

----------------------------------------------------------------------
v6: Drop the Fixes tag in uuid, since the file has been
moved to another place, not suitable to apply on stable. And this is not a bug, just some kind of enhancement.

v5: 1. Add the VF token parse error handling.
2. Split into two patches for different logic module.
3. Add more comments into the code for explaining the design.
4. Drop the ABI change workaround, this patch set focuses on code review.
https://urldefense.proofpoint.com/v2/url?u=https-
3A__patchwork.dpdk.org_cover_68364_&d=DwIDaQ&c=nKjWec2b6R0mOyP az7xtfQ&r=WllrYaumVkxaWjgKto6E_rtDQshhIhik2jkvzFyRhW8&m=SxW07R9 V0iyd_fAAZm4bU3jimkCD0khdQeabCt9pESg&s=mAkB5KtaFDGFdmqfNZV4dy w1A8Xy61tm22TNM8kGSrQ&e=

v4: 1. Ignore rte_vfio_setup_device ABI check since it is
for Linux driver use.
https://urldefense.proofpoint.com/v2/url?u=https-
3A__patchwork.dpdk.org_patch_68255_&d=DwIDaQ&c=nKjWec2b6R0mOyP az7xtfQ&r=WllrYaumVkxaWjgKto6E_rtDQshhIhik2jkvzFyRhW8&m=SxW07R9 V0iyd_fAAZm4bU3jimkCD0khdQeabCt9pESg&s=Qv5ArXmk_WeQn1bTnM6JS XpNp3NEhuenHP1VsjUPzGE&e=

v3: Fix the Travis build failed:
(1). rte_uuid.h:97:55: error: unknown type name ‘size_t’ (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’
https://urldefense.proofpoint.com/v2/url?u=https-
3A__patchwork.dpdk.org_patch_68254_&d=DwIDaQ&c=nKjWec2b6R0mOyP az7xtfQ&r=WllrYaumVkxaWjgKto6E_rtDQshhIhik2jkvzFyRhW8&m=SxW07R9 V0iyd_fAAZm4bU3jimkCD0khdQeabCt9pESg&s=8U5HwQ-j- dkzQPk3E99wJauItTQLH0f48qay37pnTZ8&e=

v2: Fix the FreeBSD build error.
https://urldefense.proofpoint.com/v2/url?u=https-
3A__patchwork.dpdk.org_patch_68240_&d=DwIDaQ&c=nKjWec2b6R0mOyP az7xtfQ&r=WllrYaumVkxaWjgKto6E_rtDQshhIhik2jkvzFyRhW8&m=SxW07R9 V0iyd_fAAZm4bU3jimkCD0khdQeabCt9pESg&s=7gq0iGeXay4YQ4ydHRdCZ9_ oWDRQFtGOxGu3chq0z_s&e=

v1: Update the commit message.
https://urldefense.proofpoint.com/v2/url?u=https-
3A__patchwork.dpdk.org_patch_68237_&d=DwIDaQ&c=nKjWec2b6R0mOyP az7xtfQ&r=WllrYaumVkxaWjgKto6E_rtDQshhIhik2jkvzFyRhW8&m=SxW07R9 V0iyd_fAAZm4bU3jimkCD0khdQeabCt9pESg&s=ibSeCUMMleRvQRH9WvJDYV TfBAt-Fi4lgfdTBOZ5xwc&e=

RFC v2: https://urldefense.proofpoint.com/v2/url?u=https-
3A__patchwork.dpdk.org_patch_68114_&d=DwIDaQ&c=nKjWec2b6R0mOyP az7xtfQ&r=WllrYaumVkxaWjgKto6E_rtDQshhIhik2jkvzFyRhW8&m=SxW07R9 V0iyd_fAAZm4bU3jimkCD0khdQeabCt9pESg&s=JmSNjqCLE3Xh0_VymptqrViX VlRgpFN_PZMvM6C2X2s&e=
Based on Vamsi's RFC v1, and Alex's patch for Qemu
[https://urldefense.proofpoint.com/v2/url?u=https-
3A__lore.kernel.org_lkml_20200204161737.34696b91- 40w520.home_&d=DwIDaQ&c=nKjWec2b6R0mOyPaz7xtfQ&r=WllrYaumVkx aWjgKto6E_rtDQshhIhik2jkvzFyRhW8&m=SxW07R9V0iyd_fAAZm4bU3jimkCD 0khdQeabCt9pESg&s=7tRxVtn0y5xpxqC38hgAB7Bjz7kHR8mbD- pIFgP1E7o&e= ]:
Use the devarg to pass-down the VF token.

RFC v1: https://urldefense.proofpoint.com/v2/url?u=https-
3A__patchwork.dpdk.org_patch_66281_&d=DwIDaQ&c=nKjWec2b6R0mOyP az7xtfQ&r=WllrYaumVkxaWjgKto6E_rtDQshhIhik2jkvzFyRhW8&m=SxW07R9 V0iyd_fAAZm4bU3jimkCD0khdQeabCt9pESg&s=riYbXycUMPQxA30y75nRzs08 f2wM-uAGcQEnK9ENR7w&e= by Vamsi.

Haiyue Wang (2):
eal: add uuid dependent header files explicitly
eal: support for VFIO-PCI VF token

 drivers/bus/pci/linux/pci_vfio.c  | 74 ++++++++++++++++++++++++++++++-
 lib/librte_eal/freebsd/eal.c      |  3 +-
 lib/librte_eal/include/rte_uuid.h |  2 +  lib/librte_eal/include/rte_vfio.h | 21
++++++++-
 lib/librte_eal/linux/eal_vfio.c   | 20 +++++++--
 5 files changed, 113 insertions(+), 7 deletions(-)
Quoted 2 lines

--
2.26.0

Tested following combinations:
* Both PF & VFs are with DPDK (Single process & multi process)
* PF is with DPDK & VFs are with Kernel driver
* PF is with kernel driver & VFs are with DPDK

#23 lore raw

Re: [dpdk-dev] [PATCH v6 0/2] support for VFIO-PCI VF token interface

In reply to Haiyue Wang

Thanks a lot for working on this.

I can see the support on the kernel side has been pulled in what will be 5.7.
https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=2fb732b33b0f4d73f391066c0a970b942c600dab

I'd like to get the dpdk support in 20.05, but for this, can I have a summary of the kernel discussions?
Is there anything missing/still wip in the kernel?

On the series itself, I did not look in detail, but there is no documentation on how to use the feature.
We are missing a release note update.
And a small comment on the last changes:

Quoted 3 lines · Haiyue Wang

v6: Drop the Fixes tag in uuid, since the file has been
moved to another place, not suitable to apply on stable.

This is *not* a reason to drop the Fixes: tag. Hopefully, we still fix bugs once files have been moved.

Quoted 1 line

And this is not a bug, just some kind of enhancement.

But on this point, I suppose this is ok not to backport.

Signature · 1 line
David Marchand
#24 lore raw

Re: [dpdk-dev] [PATCH v6 0/2] support for VFIO-PCI VF token interface

Hi David,

Quoted 15 lines

-----Original Message-----
From: David Marchand <[email protected]>
Sent: Saturday, April 18, 2020 17:22
To: Wang, Haiyue <[email protected]>; Vamsi Attunuru <[email protected]>; [email protected]
Cc: dev <[email protected]>; Thomas Monjalon <[email protected]>; Jerin Jacob Kollanukkaran <[email protected]>
Subject: Re: [PATCH v6 0/2] support for VFIO-PCI VF token interface

Thanks a lot for working on this.

I can see the support on the kernel side has been pulled in what will be 5.7.
https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=2fb732b33b0f4d73f391066c
0a970b942c600dab

Quoted 8 lines

I'd like to get the dpdk support in 20.05, but for this, can I have a summary of the kernel discussions?
Is there anything missing/still wip in the kernel?

On the series itself, I did not look in detail, but there is no documentation on how to use the feature.
We are missing a release note update.

If needed in 20.05, I can add it in release note. And about how to, I just added it in commit message, and some comments in the API ' rte_vfio_setup_device '.

Not sure which document part is reasonable for adding this, any good suggestion ?

Quoted 10 lines

And a small comment on the last changes:

Quoted 3 lines · Haiyue Wang

v6: Drop the Fixes tag in uuid, since the file has been
moved to another place, not suitable to apply on stable.

This is *not* a reason to drop the Fixes: tag. Hopefully, we still fix bugs once files have been moved.

OK, will add the tag later.

Quoted 7 lines
Quoted 1 line

And this is not a bug, just some kind of enhancement.

But on this point, I suppose this is ok not to backport.

--
David Marchand

#25 lore raw

Re: [dpdk-dev] [PATCH v6 0/2] support for VFIO-PCI VF token interface

18/04/2020 11:38, Wang, Haiyue:

Quoted 11 lines

From: David Marchand <[email protected]>

Quoted 3 lines

On the series itself, I did not look in detail, but there is no documentation on how to use the feature.
We are missing a release note update.

If needed in 20.05, I can add it in release note. And about how to, I just added it in commit message, and some comments in the API ' rte_vfio_setup_device '.

Not sure which document part is reasonable for adding this, any good suggestion ?

Here: http://doc.dpdk.org/guides/linux_gsg/linux_drivers.html#vfio

#26 lore raw

Re: [dpdk-dev] [PATCH v6 0/2] support for VFIO-PCI VF token interface

Quoted 14 lines

-----Original Message-----
From: Thomas Monjalon <[email protected]>
Sent: Saturday, April 18, 2020 17:50
To: David Marchand <[email protected]>; Vamsi Attunuru <[email protected]>; [email protected]; Wang, Haiyue <[email protected]>
Cc: dev <[email protected]>; Jerin Jacob Kollanukkaran <[email protected]>
Subject: Re: [PATCH v6 0/2] support for VFIO-PCI VF token interface

18/04/2020 11:38, Wang, Haiyue:

Quoted 5 lines

From: David Marchand <[email protected]>

Quoted 3 lines

On the series itself, I did not look in detail, but there is no documentation on how to use the feature.
We are missing a release note update.

Quoted 10 lines
Quoted 6 lines

If needed in 20.05, I can add it in release note. And about how to, I just added it in commit message, and some comments in the API ' rte_vfio_setup_device '.

Not sure which document part is reasonable for adding this, any good suggestion ?

Here: http://doc.dpdk.org/guides/linux_gsg/linux_drivers.html#vfio

OK.

#27 lore raw

v7 cover support for VFIO-PCI VF token interface

In reply to [email protected]

v7: Add the Fixes tag in uuid, the release note and help
document.

v6: Drop the Fixes tag in uuid, since the file has been
moved to another place, not suitable to apply on stable. And this is not a bug, just some kind of enhancement.
https://patchwork.dpdk.org/cover/68367/

v5: 1. Add the VF token parse error handling.
2. Split into two patches for different logic module.
3. Add more comments into the code for explaining the design.
4. Drop the ABI change workaround, this patch set focuses on code review.
https://patchwork.dpdk.org/cover/68364/

v4: 1. Ignore rte_vfio_setup_device ABI check since it is
for Linux driver use.
https://patchwork.dpdk.org/patch/68255/

v3: Fix the Travis build failed:
(1). rte_uuid.h:97:55: error: unknown type name ‘size_t’ (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’
https://patchwork.dpdk.org/patch/68254/

v2: Fix the FreeBSD build error.

         https://patchwork.dpdk.org/patch/68240/

v1: Update the commit message.

        https://patchwork.dpdk.org/patch/68237/

RFC v2: https://patchwork.dpdk.org/patch/68114/
Based on Vamsi's RFC v1, and Alex's patch for Qemu
[https://lore.kernel.org/lkml/[email protected]/]:
Use the devarg to pass-down the VF token.

RFC v1: https://patchwork.dpdk.org/patch/66281/ by Vamsi.

Haiyue Wang (2):
eal: add uuid dependent header files explicitly
eal: support for VFIO-PCI VF token

 doc/guides/linux_gsg/linux_drivers.rst | 23 +++++++-
 doc/guides/rel_notes/release_20_05.rst |  5 ++
 drivers/bus/pci/linux/pci_vfio.c       | 74 +++++++++++++++++++++++++-
 lib/librte_eal/freebsd/eal.c           |  3 +-
 lib/librte_eal/include/rte_uuid.h      |  2 +
 lib/librte_eal/include/rte_vfio.h      | 21 +++++++-
 lib/librte_eal/linux/eal_vfio.c        | 20 +++++--
 7 files changed, 140 insertions(+), 8 deletions(-)
Signature · 1 line
2.26.1
#28 lore raw

PATCH v7 1/2 eal: add uuid dependent header files explicitly

Add the dependent header files explicitly, so that the user just needs to include the 'rte_uuid.h' header file directly to avoid compile error:
(1). rte_uuid.h:97:55: error: unknown type name ‘size_t’ (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’

Fixes 6bc67c497a51 ("eal: add uuid API")
---
 lib/librte_eal/include/rte_uuid.h | 2 ++
 1 file changed, 2 insertions(+)
lib/librte_eal/include/rte_uuid.h +2 -0
index 044afbdfa..8b42e070a 100644
@@ -15,6 +15,8 @@ extern "C" {
 #endif
 
 #include <stdbool.h>
+#include <stddef.h>
+#include <string.h>
 
 /**
  * Struct describing a Universal Unique Identifier
Signature · 1 line
2.26.1
#29 lore raw

PATCH v7 2/2 eal: support for VFIO-PCI VF token

In reply to Haiyue Wang

The kernel module vfio-pci introduces the VF token to enable SR-IOV support since 5.7.

The VF token can be set by a vfio-pci based PF driver and must be known by the vfio-pci based VF driver in order to gain access to the device.

---
 doc/guides/linux_gsg/linux_drivers.rst | 23 +++++++-
 doc/guides/rel_notes/release_20_05.rst |  5 ++
 drivers/bus/pci/linux/pci_vfio.c       | 74 +++++++++++++++++++++++++-
 lib/librte_eal/freebsd/eal.c           |  3 +-
 lib/librte_eal/include/rte_vfio.h      | 21 +++++++-
 lib/librte_eal/linux/eal_vfio.c        | 20 +++++--
 6 files changed, 138 insertions(+), 8 deletions(-)
doc/guides/linux_gsg/linux_drivers.rst +22 -1
index 238f3e900..97b83a287 100644
@@ -72,11 +72,32 @@ Note that in order to use VFIO, your kernel must support it.
 VFIO kernel modules have been included in the Linux kernel since version 3.6.0 and are usually present by default,
 however please consult your distributions documentation to make sure that is the case.
 
+The Linux kernel since version 5.7 supports the creation of virtual functions, and by default this feature is off.
+After enabled, the PF needs the VF token in UUID format to setup the trust or collaboration between SR-IOV PF and VFs.
+The VFs created are bound to vfio-pci module automatically, and the VF also needs VF token to gain access to the device.
+DPDK will use the keyword ``vf_token`` as device argument to pass the VF token value to PF and its related VFs.
+
+.. code-block:: console
+
+    1. sudo modprobe vfio-pci enable_sriov=1
+
+    2. ./usertools/dpdk-devbind.py -b vfio-pci 0000:87:00.0
+
+    3. echo 2 > /sys/bus/pci/devices/0000:87:00.0/sriov_numvfs
+
+    4. Start the PF:
+        ./x86_64-native-linux-gcc/app/testpmd -l 22-25 -n 4 -w 87:00.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 \
+            --file-prefix=pf -- -i
+
+    5. Start the VF:
+        ./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 -w 87:02.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 \
+            --file-prefix=vf0 -- -i
+
 Also, to use VFIO, both kernel and BIOS must support and be configured to use IO virtualization (such as Intel® VT-d).
 
 .. note::
 
-    ``vfio-pci`` module doesn't support the creation of virtual functions.
+    ``vfio-pci`` module doesn't support the creation of virtual functions before Linux version 5.7.
 
 For proper operation of VFIO when running DPDK applications as a non-privileged user, correct permissions should also be set up.
 This can be done by using the DPDK setup script (called dpdk-setup.sh and located in the usertools directory).
doc/guides/rel_notes/release_20_05.rst +5 -0
index 184967844..9f4a74693 100644
@@ -81,6 +81,11 @@ New Features
   by making use of the event device capabilities. The event mode currently supports
   only inline IPsec protocol offload.
 
+* **Added the creation of virtual functions support for vfio-pci module.**
+
+  Since Linux version 5.7, vfio-pci supports a shared VF token (UUID) to represent
+  the trust between SR-IOV PF and the created VFs. Update the method to gain access
+  to the device by appending the VF token.
 
 Removed Items
 -------------
drivers/bus/pci/linux/pci_vfio.c +72 -2

112 lines not on this page · raw message

lib/librte_eal/freebsd/eal.c +2 -1
index 80dc9aa78..86d5a5f49 100644
@@ -995,7 +995,8 @@ rte_eal_vfio_intr_mode(void)
 int rte_vfio_setup_device(__rte_unused const char *sysfs_base,
 		      __rte_unused const char *dev_addr,
 		      __rte_unused int *vfio_dev_fd,
-		      __rte_unused struct vfio_device_info *device_info)
+		      __rte_unused struct vfio_device_info *device_info,
+		      __rte_unused rte_uuid_t vf_token)
 {
 	return -1;
 }
lib/librte_eal/include/rte_vfio.h +20 -1
index 20ed8c45a..e5476ec6d 100644
@@ -16,6 +16,8 @@ extern "C" {
 
 #include <stdint.h>
 
+#include <rte_uuid.h>
+
 /*
  * determine if VFIO is present on the system
  */
@@ -102,13 +104,30 @@ struct vfio_device_info;
  * @param device_info
  *   Device information.
  *
+ * @param vf_token
+ *   Before linux 5.7, the PF bound to vfio-pci doesn't support SR-IOV to
+ *   create VFs for security reason. Now the VF token is introduced to work
+ *   as some degree of trust or collaboration between PF and VFs.
+ *
+ *   A). as VF device, if the PF is a vfio device and it is bound to the
+ *   vfio-pci driver, the user needs to provide a VF token to access the
+ *   device, in the form of appending a vf_token to the device name, for
+ *   example:
+ *     "-w 04:10.0,vf_token=bd8d9d2b-5a5f-4f5a-a211-f591514ba1f3"
+ *
+ *   B). as PF device, When presented with a PF which has VFs in use, the
+ *   user must also provide the current VF token to prove collaboration with
+ *   existing VF users.  If VFs are not in use, the VF token provided for the
+ *   PF device will act to set the VF token.
+ *
  * @return
  *   0 on success.
  *   <0 on failure.
  *   >1 if the device cannot be managed this way.
  */
 int rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
-		int *vfio_dev_fd, struct vfio_device_info *device_info);
+		int *vfio_dev_fd, struct vfio_device_info *device_info,
+		rte_uuid_t vf_token);
 
 /**
  * Release a device mapped to a VFIO-managed I/O MMU group.
lib/librte_eal/linux/eal_vfio.c +17 -3
index 4502aefed..916082b5d 100644
@@ -702,7 +702,8 @@ rte_vfio_clear_group(int vfio_group_fd)
 
 int
 rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
-		int *vfio_dev_fd, struct vfio_device_info *device_info)
+		int *vfio_dev_fd, struct vfio_device_info *device_info,
+		rte_uuid_t vf_token)
 {
 	struct vfio_group_status group_status = {
 			.argsz = sizeof(group_status)
@@ -712,6 +713,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 	int vfio_container_fd;
 	int vfio_group_fd;
 	int iommu_group_num;
+	char dev[PATH_MAX];
 	int i, ret;
 
 	/* get group number */
@@ -895,8 +897,19 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 				t->type_id, t->name);
 	}
 
+	if (!rte_uuid_is_null(vf_token)) {
+		char vf_token_str[RTE_UUID_STRLEN];
+
+		rte_uuid_unparse(vf_token, vf_token_str, sizeof(vf_token_str));
+		snprintf(dev, sizeof(dev),
+			 "%s vf_token=%s", dev_addr, vf_token_str);
+	} else {
+		snprintf(dev, sizeof(dev),
+			 "%s", dev_addr);
+	}
+
 	/* get a file descriptor for the device */
-	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev_addr);
+	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev);
 	if (*vfio_dev_fd < 0) {
 		/* if we cannot get a device fd, this implies a problem with
 		 * the VFIO group or the container not having IOMMU configured.
@@ -2081,7 +2094,8 @@ int
 rte_vfio_setup_device(__rte_unused const char *sysfs_base,
 		__rte_unused const char *dev_addr,
 		__rte_unused int *vfio_dev_fd,
-		__rte_unused struct vfio_device_info *device_info)
+		__rte_unused struct vfio_device_info *device_info,
+		__rte_unused rte_uuid_t vf_token)
 {
 	return -1;
 }
Signature · 1 line
2.26.1
#30 lore raw

Re: [dpdk-dev] [PATCH v7 2/2] eal: support for VFIO-PCI VF token

18/04/2020 13:16, Haiyue Wang:

doc/guides/linux_gsg/linux_drivers.rst Note that in order to use VFIO, your kernel must support it. · line 75

Quoted 3 lines
doc/guides/linux_gsg/linux_drivers.rst +1 -0
+The Linux kernel since version 5.7 supports the creation of virtual functions, and by default this feature is off.

Creation of virtual function has always been supported. You need to be more specific about the new use case please.

doc/guides/linux_gsg/linux_drivers.rst Note that in order to use VFIO, your kernel must support it. · line 78

Quoted 3 lines
doc/guides/linux_gsg/linux_drivers.rst +3 -0
+After enabled, the PF needs the VF token in UUID format to setup the trust or collaboration between SR-IOV PF and VFs.
+The VFs created are bound to vfio-pci module automatically, and the VF also needs VF token to gain access to the device.
+DPDK will use the keyword ``vf_token`` as device argument to pass the VF token value to PF and its related VFs.

You need to explain where the token comes from. Can it be any random UUID?

#31 lore raw

Re: [dpdk-dev] [PATCH v7 2/2] eal: support for VFIO-PCI VF token

Quoted 17 lines

-----Original Message-----
From: Thomas Monjalon <[email protected]>
Sent: Saturday, April 18, 2020 21:27
To: [email protected]; [email protected]; Wang, Haiyue <[email protected]>
Cc: [email protected]; Burakov, Anatoly <[email protected]>; [email protected]; [email protected]
Subject: Re: [PATCH v7 2/2] eal: support for VFIO-PCI VF token

18/04/2020 13:16, Haiyue Wang:

doc/guides/linux_gsg/linux_drivers.rst

Quoted 3 lines
doc/guides/linux_gsg/linux_drivers.rst +1 -0
+The Linux kernel since version 5.7 supports the creation of virtual functions, and by default this

feature is off.

Creation of virtual function has always been supported. You need to be more specific about the new use case please.

Quoted 9 lines
Quoted 1 line

+After enabled, the PF needs the VF token in UUID format to setup the trust or collaboration between

SR-IOV PF and VFs.

Quoted 1 line

+The VFs created are bound to vfio-pci module automatically, and the VF also needs VF token to gain

access to the device.

Quoted 1 line

+DPDK will use the keyword ``vf_token`` as device argument to pass the VF token value to PF and its

related VFs.

You need to explain where the token comes from. Can it be any random UUID?

Try to add more in v8, please review.

#32 lore raw

v8 cover support for VFIO-PCI VF token interface

In reply to [email protected]

v8: Update the document.

v7: Add the Fixes tag in uuid, the release note and help

    document.
    https://patchwork.dpdk.org/cover/68845/

v6: Drop the Fixes tag in uuid, since the file has been
moved to another place, not suitable to apply on stable. And this is not a bug, just some kind of enhancement.
https://patchwork.dpdk.org/cover/68367/

v5: 1. Add the VF token parse error handling.
2. Split into two patches for different logic module.
3. Add more comments into the code for explaining the design.
4. Drop the ABI change workaround, this patch set focuses on code review.
https://patchwork.dpdk.org/cover/68364/

v4: 1. Ignore rte_vfio_setup_device ABI check since it is
for Linux driver use.
https://patchwork.dpdk.org/patch/68255/

v3: Fix the Travis build failed:
(1). rte_uuid.h:97:55: error: unknown type name ‘size_t’ (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’
https://patchwork.dpdk.org/patch/68254/

v2: Fix the FreeBSD build error.

         https://patchwork.dpdk.org/patch/68240/

v1: Update the commit message.

        https://patchwork.dpdk.org/patch/68237/

RFC v2: https://patchwork.dpdk.org/patch/68114/
Based on Vamsi's RFC v1, and Alex's patch for Qemu
[https://lore.kernel.org/lkml/[email protected]/]:
Use the devarg to pass-down the VF token.

RFC v1: https://patchwork.dpdk.org/patch/66281/ by Vamsi.

Haiyue Wang (2):
eal: add uuid dependent header files explicitly
eal: support for VFIO-PCI VF token

 doc/guides/linux_gsg/linux_drivers.rst | 38 ++++++++++++-
 doc/guides/rel_notes/release_20_05.rst |  5 ++
 drivers/bus/pci/linux/pci_vfio.c       | 74 +++++++++++++++++++++++++-
 lib/librte_eal/freebsd/eal.c           |  3 +-
 lib/librte_eal/include/rte_uuid.h      |  2 +
 lib/librte_eal/include/rte_vfio.h      | 21 +++++++-
 lib/librte_eal/linux/eal_vfio.c        | 20 +++++--
 7 files changed, 155 insertions(+), 8 deletions(-)
Signature · 1 line
2.26.1
#33 lore raw

PATCH v8 1/2 eal: add uuid dependent header files explicitly

Add the dependent header files explicitly, so that the user just needs to include the 'rte_uuid.h' header file directly to avoid compile error:
(1). rte_uuid.h:97:55: error: unknown type name ‘size_t’ (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’

Fixes 6bc67c497a51 ("eal: add uuid API")
---
 lib/librte_eal/include/rte_uuid.h | 2 ++
 1 file changed, 2 insertions(+)
lib/librte_eal/include/rte_uuid.h +2 -0
index 044afbdfa..8b42e070a 100644
@@ -15,6 +15,8 @@ extern "C" {
 #endif
 
 #include <stdbool.h>
+#include <stddef.h>
+#include <string.h>
 
 /**
  * Struct describing a Universal Unique Identifier
Signature · 1 line
2.26.1
#34 lore raw

PATCH v8 2/2 eal: support for VFIO-PCI VF token

In reply to Haiyue Wang

The kernel module vfio-pci introduces the VF token to enable SR-IOV support since 5.7.

The VF token can be set by a vfio-pci based PF driver and must be known by the vfio-pci based VF driver in order to gain access to the device.

---
 doc/guides/linux_gsg/linux_drivers.rst | 38 ++++++++++++-
 doc/guides/rel_notes/release_20_05.rst |  5 ++
 drivers/bus/pci/linux/pci_vfio.c       | 74 +++++++++++++++++++++++++-
 lib/librte_eal/freebsd/eal.c           |  3 +-
 lib/librte_eal/include/rte_vfio.h      | 21 +++++++-
 lib/librte_eal/linux/eal_vfio.c        | 20 +++++--
 6 files changed, 153 insertions(+), 8 deletions(-)
doc/guides/linux_gsg/linux_drivers.rst +37 -1
index 238f3e900..3391723c9 100644
@@ -72,11 +72,47 @@ Note that in order to use VFIO, your kernel must support it.
 VFIO kernel modules have been included in the Linux kernel since version 3.6.0 and are usually present by default,
 however please consult your distributions documentation to make sure that is the case.
 
+The ``vfio-pci`` module since Linux version 5.7 supports the creation of virtual functions, this feature is disabled
+by default. When enabled, the PF needs a shared VF token (UUID) to setup the trust between SR-IOV PF and VFs. The VF
+token is any kind of valid UUID value selected by the user. When the PF device is bound to ``vfio-pci`` module, it should
+not have any VFs created, this is consistent as before for security reason.
+
+Some use cases about how to use the VF token:
+
+ - The user just uses PF only for DPDK, then no VF token is required to start the PF device.
+
+ - The user wants to creat SR-IOV VFs on the PF device which is bound to ``vfio-pci`` module, then the user needs to select
+   a valid UUID type VF token to start the PF device; after the VFs are created, this VF token is also required to access each
+   VF device.
+
+ - If the DPDK application that runs on PF device exits, and the user wants to start it with another different VF token
+   value, it will be OK if no application (DPDP or KVM) runs on VF, otherwise, it will fail to start with a kernel message
+   "[19145.688094] vfio-pci 0000:87:00.0: Incorrect VF token provided for device" shown. When all of the VFs are free, the
+   user can select a new VF token to start the PF device.
+
+The VFs created are bound to ``vfio-pci`` module automatically. DPDK will use the keyword ``vf_token`` as the device argument
+to pass the VF token value to PF and its related VFs, the PMD should not use it, and this argument will be pruned from the
+device argument list, so the PMD can parse its own valid device arguments successfully without seeing it.
+
+.. code-block:: console
+
+    1. sudo modprobe vfio-pci enable_sriov=1
+
+    2. ./usertools/dpdk-devbind.py -b vfio-pci 0000:87:00.0
+
+    3. echo 2 > /sys/bus/pci/devices/0000:87:00.0/sriov_numvfs
+
+    4. Start the PF:
+        ./x86_64-native-linux-gcc/app/testpmd -l 22-25 -n 4 -w 87:00.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 --file-prefix=pf -- -i
+
+    5. Start the VF:
+        ./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 -w 87:02.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 --file-prefix=vf0 -- -i
+
 Also, to use VFIO, both kernel and BIOS must support and be configured to use IO virtualization (such as Intel® VT-d).
 
 .. note::
 
-    ``vfio-pci`` module doesn't support the creation of virtual functions.
+    ``vfio-pci`` module doesn't support the creation of virtual functions before Linux version 5.7.
 
 For proper operation of VFIO when running DPDK applications as a non-privileged user, correct permissions should also be set up.
 This can be done by using the DPDK setup script (called dpdk-setup.sh and located in the usertools directory).
doc/guides/rel_notes/release_20_05.rst +5 -0
index 184967844..0a30f912b 100644
@@ -81,6 +81,11 @@ New Features
   by making use of the event device capabilities. The event mode currently supports
   only inline IPsec protocol offload.
 
+* **Added the support for vfio-pci new VF token interface.**
+
+  Since Linux version 5.7, vfio-pci supports a shared VF token (UUID) to represent
+  the trust between SR-IOV PF and the created VFs. Update the method to gain access
+  to the device by appending the VF token.
 
 Removed Items
 -------------
drivers/bus/pci/linux/pci_vfio.c +72 -2

112 lines not on this page · raw message

lib/librte_eal/freebsd/eal.c +2 -1
index 80dc9aa78..86d5a5f49 100644
@@ -995,7 +995,8 @@ rte_eal_vfio_intr_mode(void)
 int rte_vfio_setup_device(__rte_unused const char *sysfs_base,
 		      __rte_unused const char *dev_addr,
 		      __rte_unused int *vfio_dev_fd,
-		      __rte_unused struct vfio_device_info *device_info)
+		      __rte_unused struct vfio_device_info *device_info,
+		      __rte_unused rte_uuid_t vf_token)
 {
 	return -1;
 }
lib/librte_eal/include/rte_vfio.h +20 -1
index 20ed8c45a..e5476ec6d 100644
@@ -16,6 +16,8 @@ extern "C" {
 
 #include <stdint.h>
 
+#include <rte_uuid.h>
+
 /*
  * determine if VFIO is present on the system
  */
@@ -102,13 +104,30 @@ struct vfio_device_info;
  * @param device_info
  *   Device information.
  *
+ * @param vf_token
+ *   Before linux 5.7, the PF bound to vfio-pci doesn't support SR-IOV to
+ *   create VFs for security reason. Now the VF token is introduced to work
+ *   as some degree of trust or collaboration between PF and VFs.
+ *
+ *   A). as VF device, if the PF is a vfio device and it is bound to the
+ *   vfio-pci driver, the user needs to provide a VF token to access the
+ *   device, in the form of appending a vf_token to the device name, for
+ *   example:
+ *     "-w 04:10.0,vf_token=bd8d9d2b-5a5f-4f5a-a211-f591514ba1f3"
+ *
+ *   B). as PF device, When presented with a PF which has VFs in use, the
+ *   user must also provide the current VF token to prove collaboration with
+ *   existing VF users.  If VFs are not in use, the VF token provided for the
+ *   PF device will act to set the VF token.
+ *
  * @return
  *   0 on success.
  *   <0 on failure.
  *   >1 if the device cannot be managed this way.
  */
 int rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
-		int *vfio_dev_fd, struct vfio_device_info *device_info);
+		int *vfio_dev_fd, struct vfio_device_info *device_info,
+		rte_uuid_t vf_token);
 
 /**
  * Release a device mapped to a VFIO-managed I/O MMU group.
lib/librte_eal/linux/eal_vfio.c +17 -3
index 4502aefed..916082b5d 100644
@@ -702,7 +702,8 @@ rte_vfio_clear_group(int vfio_group_fd)
 
 int
 rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
-		int *vfio_dev_fd, struct vfio_device_info *device_info)
+		int *vfio_dev_fd, struct vfio_device_info *device_info,
+		rte_uuid_t vf_token)
 {
 	struct vfio_group_status group_status = {
 			.argsz = sizeof(group_status)
@@ -712,6 +713,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 	int vfio_container_fd;
 	int vfio_group_fd;
 	int iommu_group_num;
+	char dev[PATH_MAX];
 	int i, ret;
 
 	/* get group number */
@@ -895,8 +897,19 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 				t->type_id, t->name);
 	}
 
+	if (!rte_uuid_is_null(vf_token)) {
+		char vf_token_str[RTE_UUID_STRLEN];
+
+		rte_uuid_unparse(vf_token, vf_token_str, sizeof(vf_token_str));
+		snprintf(dev, sizeof(dev),
+			 "%s vf_token=%s", dev_addr, vf_token_str);
+	} else {
+		snprintf(dev, sizeof(dev),
+			 "%s", dev_addr);
+	}
+
 	/* get a file descriptor for the device */
-	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev_addr);
+	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev);
 	if (*vfio_dev_fd < 0) {
 		/* if we cannot get a device fd, this implies a problem with
 		 * the VFIO group or the container not having IOMMU configured.
@@ -2081,7 +2094,8 @@ int
 rte_vfio_setup_device(__rte_unused const char *sysfs_base,
 		__rte_unused const char *dev_addr,
 		__rte_unused int *vfio_dev_fd,
-		__rte_unused struct vfio_device_info *device_info)
+		__rte_unused struct vfio_device_info *device_info,
+		__rte_unused rte_uuid_t vf_token)
 {
 	return -1;
 }
Signature · 1 line
2.26.1
#35 lore raw

Re: [dpdk-dev] [PATCH v8 2/2] eal: support for VFIO-PCI VF token

Hi,

A general comment, please try to limit line length (80 is a good limit), and prefer breaking lines after a punctuation sign.
Try shortening sentences as well.

I would love seeing some documentation reviews from english native speakers.

18/04/2020 19:30, Haiyue Wang:

doc/guides/linux_gsg/linux_drivers.rst Note that in order to use VFIO, your kernel must support it. · line 82

Quoted 10 lines
doc/guides/linux_gsg/linux_drivers.rst +8 -0
+The ``vfio-pci`` module since Linux version 5.7 supports the creation of virtual functions, this feature is disabled
+by default. When enabled, the PF needs a shared VF token (UUID) to setup the trust between SR-IOV PF and VFs. The VF
+token is any kind of valid UUID value selected by the user. When the PF device is bound to ``vfio-pci`` module, it should
+not have any VFs created, this is consistent as before for security reason.
+
+Some use cases about how to use the VF token:
+
+ - The user just uses PF only for DPDK, then no VF token is required to start the PF device.

This use case is not using the token, so it should be mentioned out of this list.

Quoted 2 lines

+
+ - The user wants to creat SR-IOV VFs on the PF device which is bound to ``vfio-pci`` module, then the user needs to select

creat -> create

doc/guides/linux_gsg/linux_drivers.rst Note that in order to use VFIO, your kernel must support it. · line 89

Quoted 5 lines
doc/guides/linux_gsg/linux_drivers.rst +5 -0
+   a valid UUID type VF token to start the PF device; after the VFs are created, this VF token is also required to access each
+   VF device.
+
+ - If the DPDK application that runs on PF device exits, and the user wants to start it with another different VF token
+   value, it will be OK if no application (DPDP or KVM) runs on VF, otherwise, it will fail to start with a kernel message

DPDP -> DPDK.

Quoted 1 line

+ "[19145.688094] vfio-pci 0000:87:00.0: Incorrect VF token provided for device" shown. When all of the VFs are free, the

The timestamp is useless in the log.
Please insert the log after :: as a fixed-size font line.

Quoted 6 lines
doc/guides/linux_gsg/linux_drivers.rst +6 -0
+   user can select a new VF token to start the PF device.
+
+The VFs created are bound to ``vfio-pci`` module automatically. DPDK will use the keyword ``vf_token`` as the device argument
+to pass the VF token value to PF and its related VFs, the PMD should not use it, and this argument will be pruned from the
+device argument list, so the PMD can parse its own valid device arguments successfully without seeing it.
+

doc/guides/linux_gsg/linux_drivers.rst Note that in order to use VFIO, your kernel must support it. · line 106

Quoted 10 lines
doc/guides/linux_gsg/linux_drivers.rst +10 -0
+.. code-block:: console
+
+    1. sudo modprobe vfio-pci enable_sriov=1
+
+    2. ./usertools/dpdk-devbind.py -b vfio-pci 0000:87:00.0
+
+    3. echo 2 > /sys/bus/pci/devices/0000:87:00.0/sriov_numvfs
+
+    4. Start the PF:
+        ./x86_64-native-linux-gcc/app/testpmd -l 22-25 -n 4 -w 87:00.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 --file-prefix=pf -- -i

How did you generate this token?

doc/guides/linux_gsg/linux_drivers.rst Note that in order to use VFIO, your kernel must support it. · line 109

Quoted 3 lines
doc/guides/linux_gsg/linux_drivers.rst +3 -0
+
+    5. Start the VF:
+        ./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 -w 87:02.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 --file-prefix=vf0 -- -i

Please do not make bullets and comments part of code-block.

#36 lore raw

Re: [dpdk-dev] [PATCH v8 0/2] support for VFIO-PCI VF token interface

In reply to Haiyue Wang

Hello,

Quoted 2 lines · Haiyue Wang

v8: Update the document.

Thanks for the last version.
I had a look at the CI, I can see we are still missing bits to handle the ABI failure on rte_vfio_setup_device.

Signature · 1 line
David Marchand
#37 lore raw

Re: [dpdk-dev] [PATCH v8 0/2] support for VFIO-PCI VF token interface

Quoted 11 lines

-----Original Message-----
From: David Marchand <[email protected]>
Sent: Tuesday, April 21, 2020 00:53
To: Wang, Haiyue <[email protected]>
Cc: dev <[email protected]>; Burakov, Anatoly <[email protected]>; Thomas Monjalon <[email protected]>; Vamsi Attunuru <[email protected]>; Jerin Jacob Kollanukkaran <[email protected]>; Alex Williamson <[email protected]>
Subject: Re: [PATCH v8 0/2] support for VFIO-PCI VF token interface

Hello,

Quoted 7 lines
Quoted 2 lines · Haiyue Wang

v8: Update the document.

Thanks for the last version.
I had a look at the CI, I can see we are still missing bits to handle the ABI failure on rte_vfio_setup_device.

Yes, not handle it now.

If 'rte_vfio_setup_device' can be internal, not official DPDK API, then __rte_internal is the best way to handle ABI issue.

Quoted 4 lines

--
David Marchand

#38 lore raw

Re: [dpdk-dev] [PATCH v8 0/2] support for VFIO-PCI VF token interface

20/04/2020 19:02, Wang, Haiyue:

Quoted 8 lines

From: David Marchand <[email protected]>

Quoted 7 lines

Hello,

Quoted 2 lines · Haiyue Wang

v8: Update the document.

Quoted 8 lines
Quoted 3 lines

Thanks for the last version.
I had a look at the CI, I can see we are still missing bits to handle the ABI failure on rte_vfio_setup_device.

Yes, not handle it now.

If 'rte_vfio_setup_device' can be internal, not official DPDK API, then __rte_internal is the best way to handle ABI issue.

Please could you help finishing integration of __rte_internal?

This patch is blocked for now.

#39 lore raw

Re: [dpdk-dev] [PATCH v8 0/2] support for VFIO-PCI VF token interface

+Neil,

Quoted 22 lines

-----Original Message-----
From: Thomas Monjalon <[email protected]>
Sent: Tuesday, April 21, 2020 01:13
To: Wang, Haiyue <[email protected]>
Cc: David Marchand <[email protected]>; dev <[email protected]>; Burakov, Anatoly <[email protected]>; Vamsi Attunuru <[email protected]>; Jerin Jacob Kollanukkaran <[email protected]>; Alex Williamson <[email protected]>
Subject: Re: [PATCH v8 0/2] support for VFIO-PCI VF token interface

20/04/2020 19:02, Wang, Haiyue:

Quoted 12 lines

From: David Marchand <[email protected]>

Quoted 10 lines

Hello,

Quoted 2 lines · Haiyue Wang

v8: Update the document.

Thanks for the last version.
I had a look at the CI, I can see we are still missing bits to handle the ABI failure on rte_vfio_setup_device.

Quoted 7 lines
Quoted 4 lines

Yes, not handle it now.

If 'rte_vfio_setup_device' can be internal, not official DPDK API, then __rte_internal is the best way to handle ABI issue.

Please could you help finishing integration of __rte_internal?

I thought it should be Neil ? "Yes, I'll get back to this today" ;-)
http://inbox.dpdk.org/dev/CAJFAV8ydLkV0afEHqbh6KeA3Box0Yxb3N0MNGtMD4S9bmSgT0A@mail.gmail.com/

Quoted 2 lines

This patch is blocked for now.

#40 lore raw

Re: [dpdk-dev] [PATCH v8 0/2] support for VFIO-PCI VF token interface

20/04/2020 19:37, Wang, Haiyue:

Quoted 8 lines

From: Thomas Monjalon <[email protected]>

Quoted 7 lines

20/04/2020 19:02, Wang, Haiyue:

Quoted 6 lines

From: David Marchand <[email protected]>

Quoted 2 lines

I had a look at the CI, I can see we are still missing bits to handle the ABI failure on rte_vfio_setup_device.

Yes, not handle it now.

Quoted 7 lines
Quoted 4 lines
Quoted 2 lines

If 'rte_vfio_setup_device' can be internal, not official DPDK API, then __rte_internal is the best way to handle ABI issue.

Please could you help finishing integration of __rte_internal?

I thought it should be Neil ? "Yes, I'll get back to this today" ;-)
http://inbox.dpdk.org/dev/CAJFAV8ydLkV0afEHqbh6KeA3Box0Yxb3N0MNGtMD4S9bmSgT0A@mail.gmail.com/

It did not happen after several months.
If you want to unblock your patches, I think it is safer to finish yourself.

#41 lore raw

Re: [dpdk-dev] [PATCH v8 0/2] support for VFIO-PCI VF token interface

Hi Thomas & David,

Quoted 21 lines

-----Original Message-----
From: Thomas Monjalon <[email protected]>
Sent: Tuesday, April 21, 2020 01:43
To: Wang, Haiyue <[email protected]>
Cc: Neil Horman <[email protected]>; David Marchand <[email protected]>; dev <[email protected]>; Burakov, Anatoly <[email protected]>; Vamsi Attunuru <[email protected]>; Jerin Jacob Kollanukkaran <[email protected]>; Alex Williamson <[email protected]>
Subject: Re: [PATCH v8 0/2] support for VFIO-PCI VF token interface

20/04/2020 19:37, Wang, Haiyue:

Quoted 11 lines

From: Thomas Monjalon <[email protected]>

Quoted 10 lines

20/04/2020 19:02, Wang, Haiyue:

Quoted 8 lines

From: David Marchand <[email protected]>

Quoted 2 lines

I had a look at the CI, I can see we are still missing bits to handle the ABI failure on rte_vfio_setup_device.

Yes, not handle it now.

If 'rte_vfio_setup_device' can be internal, not official DPDK API, then __rte_internal is the best way to handle ABI issue.

Quoted 8 lines
Quoted 4 lines
Quoted 1 line

Please could you help finishing integration of __rte_internal?

I thought it should be Neil ? "Yes, I'll get back to this today" ;-)
http://inbox.dpdk.org/dev/CAJFAV8ydLkV0afEHqbh6KeA3Box0Yxb3N0MNGtMD4S9bmSgT0A@mail.gmail.com/

It did not happen after several months.
If you want to unblock your patches, I think it is safer to finish yourself.

Unfortunately, this __rte_internal will still make the ci fail to run when move the function to INTERNAL session:

devtools/libabigail.abignore +13 -2
@@ -3,6 +3,11 @@
 [suppress_variable]
         symbol_version = EXPERIMENTAL
 
+[suppress_function]
+        symbol_version = INTERNAL
+[suppress_variable]
+        symbol_version = INTERNAL
+
 
 
--- a/lib/librte_eal/rte_eal_version.map
+++ b/lib/librte_eal/rte_eal_version.map
@@ -213,7 +213,6 @@ DPDK_20.0 {
        rte_vfio_is_enabled;
        rte_vfio_noiommu_is_enabled;
        rte_vfio_release_device;
-       rte_vfio_setup_device;
        rte_vlog;
        rte_zmalloc;
        rte_zmalloc_socket;
@@ -339,3 +338,10 @@ EXPERIMENTAL {
        # added in 20.05
        rte_log_can_log;
 };
+
+INTERNAL {
+       global:
+
+       # added in 20.05
+       rte_vfio_setup_device;
+};

Functions changes summary: 1 Removed, 0 Changed, 0 Added function Variables changes summary: 0 Removed, 0 Changed, 0 Added variable

1 Removed function:

[D] 'function int rte_vfio_setup_device(const char*, const char*, int*, vfio_device_info*)' {rte_vfio_setup_device@@DPDK_20.0}

Error: ABI issue reported for 'abidiff --suppr ./devtools/libabigail.abignore --no-added-syms --headers-dir1 old_abi/include --headers-dir2 new_abi/include old_abi/dump/librte_eal.dump new_abi/dump/librte_eal.dump'

#42 lore raw

Re: [dpdk-dev] [PATCH v8 0/2] support for VFIO-PCI VF token interface

21/04/2020 03:38, Wang, Haiyue:

Quoted 24 lines

From: Thomas Monjalon <[email protected]>

Quoted 12 lines

20/04/2020 19:37, Wang, Haiyue:

Quoted 11 lines

From: Thomas Monjalon <[email protected]>

Quoted 10 lines

20/04/2020 19:02, Wang, Haiyue:

Quoted 8 lines

From: David Marchand <[email protected]>

Quoted 2 lines

I had a look at the CI, I can see we are still missing bits to handle the ABI failure on rte_vfio_setup_device.

Yes, not handle it now.

If 'rte_vfio_setup_device' can be internal, not official DPDK API, then __rte_internal is the best way to handle ABI issue.

Quoted 8 lines
Quoted 4 lines
Quoted 1 line

Please could you help finishing integration of __rte_internal?

I thought it should be Neil ? "Yes, I'll get back to this today" ;-)
http://inbox.dpdk.org/dev/CAJFAV8ydLkV0afEHqbh6KeA3Box0Yxb3N0MNGtMD4S9bmSgT0A@mail.gmail.com/

It did not happen after several months.
If you want to unblock your patches, I think it is safer to finish yourself.

Unfortunately, this __rte_internal will still make the ci fail to run when move the function to INTERNAL session:

[...]

devtools/libabigail.abignore EXPERIMENTAL { · line 347

Quoted 6 lines
devtools/libabigail.abignore +6 -0
+INTERNAL {
+       global:
+
+       # added in 20.05
+       rte_vfio_setup_device;
+};

Why do you need to move the symbol explicitly in .map?

The tool should ignore symbols moving to internal, as an exception until 20.11.

#43 lore raw

Re: [dpdk-dev] [PATCH v8 0/2] support for VFIO-PCI VF token interface

Quoted 36 lines

-----Original Message-----
From: Thomas Monjalon <[email protected]>
Sent: Tuesday, April 21, 2020 10:13
To: David Marchand <[email protected]>; Wang, Haiyue <[email protected]>
Cc: Neil Horman <[email protected]>; dev <[email protected]>; Burakov, Anatoly <[email protected]>; Vamsi Attunuru <[email protected]>; Jerin Jacob Kollanukkaran <[email protected]>; Yigit, Ferruh <[email protected]>; Richardson, Bruce <[email protected]>; Kinsella, Ray <[email protected]>
Subject: Re: [PATCH v8 0/2] support for VFIO-PCI VF token interface

21/04/2020 03:38, Wang, Haiyue:

Quoted 24 lines

From: Thomas Monjalon <[email protected]>

Quoted 12 lines

20/04/2020 19:37, Wang, Haiyue:

Quoted 11 lines

From: Thomas Monjalon <[email protected]>

Quoted 10 lines

20/04/2020 19:02, Wang, Haiyue:

Quoted 8 lines

From: David Marchand <[email protected]>

Quoted 2 lines

I had a look at the CI, I can see we are still missing bits to handle the ABI failure on rte_vfio_setup_device.

Yes, not handle it now.

If 'rte_vfio_setup_device' can be internal, not official DPDK API, then __rte_internal is the best way to handle ABI issue.

Quoted 8 lines
Quoted 4 lines
Quoted 1 line

Please could you help finishing integration of __rte_internal?

I thought it should be Neil ? "Yes, I'll get back to this today" ;-)
http://inbox.dpdk.org/dev/CAJFAV8ydLkV0afEHqbh6KeA3Box0Yxb3N0MNGtMD4S9bmSgT0A@mail.gmail.com/

It did not happen after several months.
If you want to unblock your patches, I think it is safer to finish yourself.

Unfortunately, this __rte_internal will still make the ci fail to run when move the function to INTERNAL session:

[...]

Quoted 10 lines

devtools/libabigail.abignore EXPERIMENTAL { · line 347

Quoted 6 lines
devtools/libabigail.abignore +6 -0
+INTERNAL {
+       global:
+
+       # added in 20.05
+       rte_vfio_setup_device;
+};

Why do you need to move the symbol explicitly in .map?
The tool should ignore symbols moving to internal, as an exception until 20.11.

If not move the symbol explicitly in .map, another kind of error happened.

./devtools/check-abi.sh old_abi new_abi
Functions changes summary: 0 Removed, 1 Changed, 0 Added function Variables changes summary: 0 Removed, 0 Changed, 0 Added variable

1 function with some indirect sub-type change:

[C] 'function int rte_vfio_setup_device(const char*, const char*, int*, vfio_device_info*)' at eal_vfio.c:704:1 has some indirect sub-type changes:
parameter 5 of type 'unsigned char*' was added

Error: ABI issue reported for 'abidiff --suppr ./devtools/libabigail.abignore --no-added-syms --headers-dir1 old_abi/include --headers-dir2 new_abi/include old_abi/dump/librte_eal.dump new_abi/dump/librte_eal.dump'

#44 lore raw

Re: [dpdk-dev] [PATCH v8 0/2] support for VFIO-PCI VF token interface

21/04/2020 04:52, Wang, Haiyue:

Quoted 43 lines

From: Thomas Monjalon <[email protected]>

Quoted 26 lines

21/04/2020 03:38, Wang, Haiyue:

Quoted 24 lines

From: Thomas Monjalon <[email protected]>

Quoted 12 lines

20/04/2020 19:37, Wang, Haiyue:

Quoted 11 lines

From: Thomas Monjalon <[email protected]>

Quoted 10 lines

20/04/2020 19:02, Wang, Haiyue:

Quoted 8 lines

From: David Marchand <[email protected]>

Quoted 2 lines

I had a look at the CI, I can see we are still missing bits to handle the ABI failure on rte_vfio_setup_device.

Yes, not handle it now.

If 'rte_vfio_setup_device' can be internal, not official DPDK API, then __rte_internal is the best way to handle ABI issue.

Quoted 8 lines
Quoted 4 lines
Quoted 1 line

Please could you help finishing integration of __rte_internal?

I thought it should be Neil ? "Yes, I'll get back to this today" ;-)
http://inbox.dpdk.org/dev/CAJFAV8ydLkV0afEHqbh6KeA3Box0Yxb3N0MNGtMD4S9bmSgT0A@mail.gmail.com/

It did not happen after several months.
If you want to unblock your patches, I think it is safer to finish yourself.

Unfortunately, this __rte_internal will still make the ci fail to run when move the function to INTERNAL session:

[...]

Quoted 9 lines

devtools/libabigail.abignore EXPERIMENTAL { · line 347

Quoted 6 lines
devtools/libabigail.abignore +6 -0
+INTERNAL {
+       global:
+
+       # added in 20.05
+       rte_vfio_setup_device;
+};

Why do you need to move the symbol explicitly in .map?
The tool should ignore symbols moving to internal, as an exception until 20.11.

If not move the symbol explicitly in .map, another kind of error happened.

./devtools/check-abi.sh old_abi new_abi
Functions changes summary: 0 Removed, 1 Changed, 0 Added function Variables changes summary: 0 Removed, 0 Changed, 0 Added variable

Quoted 6 lines

1 function with some indirect sub-type change:

[C] 'function int rte_vfio_setup_device(const char*, const char*, int*, vfio_device_info*)' at eal_vfio.c:704:1 has some indirect sub-type changes:
parameter 5 of type 'unsigned char*' was added

Error: ABI issue reported for 'abidiff --suppr ./devtools/libabigail.abignore --no-added-syms --headers-dir1 old_abi/include --headers-dir2 new_abi/include old_abi/dump/librte_eal.dump new_abi/dump/librte_eal.dump'

This is what I said: you need to add a rule to ignore internal symbols.

#45 lore raw

Re: [dpdk-dev] [PATCH v8 0/2] support for VFIO-PCI VF token interface

Quoted 58 lines

-----Original Message-----
From: Thomas Monjalon <[email protected]>
Sent: Tuesday, April 21, 2020 16:48
To: David Marchand <[email protected]>; Wang, Haiyue <[email protected]>
Cc: Neil Horman <[email protected]>; dev <[email protected]>; Burakov, Anatoly <[email protected]>; Vamsi Attunuru <[email protected]>; Jerin Jacob Kollanukkaran <[email protected]>; Yigit, Ferruh <[email protected]>; Richardson, Bruce <[email protected]>; Kinsella, Ray <[email protected]>
Subject: Re: [PATCH v8 0/2] support for VFIO-PCI VF token interface

21/04/2020 04:52, Wang, Haiyue:

Quoted 19 lines

From: Thomas Monjalon <[email protected]>

Quoted 18 lines

21/04/2020 03:38, Wang, Haiyue:

Quoted 17 lines

From: Thomas Monjalon <[email protected]>

Quoted 16 lines

20/04/2020 19:37, Wang, Haiyue:

Quoted 15 lines

From: Thomas Monjalon <[email protected]>

Quoted 11 lines

20/04/2020 19:02, Wang, Haiyue:

Quoted 8 lines

From: David Marchand <[email protected]>

Quoted 2 lines

I had a look at the CI, I can see we are still missing bits to handle the ABI failure on rte_vfio_setup_device.

Yes, not handle it now.

If 'rte_vfio_setup_device' can be internal, not official DPDK API, then __rte_internal is the best way to handle ABI issue.

Please could you help finishing integration of __rte_internal?

I thought it should be Neil ? "Yes, I'll get back to this today" ;-)

http://inbox.dpdk.org/dev/CAJFAV8ydLkV0afEHqbh6KeA3Box0Yxb3N0MNGtMD4S9bmSgT0A@mail.gmail.com/

Quoted 20 lines
Quoted 8 lines
Quoted 7 lines
Quoted 4 lines

It did not happen after several months.
If you want to unblock your patches, I think it is safer to finish yourself.

Unfortunately, this __rte_internal will still make the ci fail to run when move the function to INTERNAL session:

[...]

Quoted 9 lines

devtools/libabigail.abignore EXPERIMENTAL { · line 347

Quoted 6 lines
devtools/libabigail.abignore +6 -0
+INTERNAL {
+       global:
+
+       # added in 20.05
+       rte_vfio_setup_device;
+};

Why do you need to move the symbol explicitly in .map?
The tool should ignore symbols moving to internal, as an exception until 20.11.

If not move the symbol explicitly in .map, another kind of error happened.

Quoted 7 lines

./devtools/check-abi.sh old_abi new_abi
Functions changes summary: 0 Removed, 1 Changed, 0 Added function Variables changes summary: 0 Removed, 0 Changed, 0 Added variable

1 function with some indirect sub-type change:

[C] 'function int rte_vfio_setup_device(const char*, const char*, int*, vfio_device_info*)' at

Quoted 9 lines

eal_vfio.c:704:1 has some indirect sub-type changes:

Quoted 3 lines

parameter 5 of type 'unsigned char*' was added

Error: ABI issue reported for 'abidiff --suppr ./devtools/libabigail.abignore --no-added-syms --

headers-dir1 old_abi/include --headers-dir2 new_abi/include old_abi/dump/librte_eal.dump new_abi/dump/librte_eal.dump'

This is what I said: you need to add a rule to ignore internal symbols.

Got the kind reply from Dodji, the maintainer of libabigail, his suggestion meets what we have done for 'EXPERIMENTAL'. So it seems that have to mark all needed functions as INTERNAL firstly, which obviously will break the CI as the public functions are removed....

"
__PROJECT_INTERNAL_USE_ONLY_VERSION__ {
global: funA
};
...
Then, once you have all your internal functions marked in the ELF binary with the proper ELF version string, you can tell libabigail to suppress all functions that have that version by writing a suppression specification file that has this content:

[suppress_function]
symbol_version = __PROJECT_INTERNAL_USE_ONLY_VERSION__
"

#46 lore raw

v9 cover support for VFIO-PCI VF token interface

In reply to [email protected]

v9: Rewrite the document.

v8: Update the document.

    https://patchwork.dpdk.org/cover/68859/

v7: Add the Fixes tag in uuid, the release note and help

    document.
    https://patchwork.dpdk.org/cover/68845/

v6: Drop the Fixes tag in uuid, since the file has been
moved to another place, not suitable to apply on stable. And this is not a bug, just some kind of enhancement.
https://patchwork.dpdk.org/cover/68367/

v5: 1. Add the VF token parse error handling.
2. Split into two patches for different logic module.
3. Add more comments into the code for explaining the design.
4. Drop the ABI change workaround, this patch set focuses on code review.
https://patchwork.dpdk.org/cover/68364/

v4: 1. Ignore rte_vfio_setup_device ABI check since it is
for Linux driver use.
https://patchwork.dpdk.org/patch/68255/

v3: Fix the Travis build failed:
(1). rte_uuid.h:97:55: error: unknown type name ‘size_t’ (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’
https://patchwork.dpdk.org/patch/68254/

v2: Fix the FreeBSD build error.

         https://patchwork.dpdk.org/patch/68240/

v1: Update the commit message.

        https://patchwork.dpdk.org/patch/68237/

RFC v2: https://patchwork.dpdk.org/patch/68114/
Based on Vamsi's RFC v1, and Alex's patch for Qemu
[https://lore.kernel.org/lkml/[email protected]/]:
Use the devarg to pass-down the VF token.

RFC v1: https://patchwork.dpdk.org/patch/66281/ by Vamsi.

Haiyue Wang (2):
eal: add uuid dependent header files explicitly
eal: support for VFIO-PCI VF token

 devtools/libabigail.abignore           |  2 +
 doc/guides/linux_gsg/linux_drivers.rst | 41 +++++++++++++-
 doc/guides/rel_notes/release_20_05.rst |  5 ++
 drivers/bus/pci/linux/pci_vfio.c       | 74 +++++++++++++++++++++++++-
 lib/librte_eal/freebsd/eal.c           |  3 +-
 lib/librte_eal/include/rte_uuid.h      |  2 +
 lib/librte_eal/include/rte_vfio.h      | 24 ++++++++-
 lib/librte_eal/linux/eal_vfio.c        | 20 +++++--
 8 files changed, 163 insertions(+), 8 deletions(-)
Signature · 1 line
2.26.2
#47 lore raw

PATCH v9 1/2 eal: add uuid dependent header files explicitly

Add the dependent header files explicitly, so that the user just needs to include the 'rte_uuid.h' header file directly to avoid compile error:
(1). rte_uuid.h:97:55: error: unknown type name ‘size_t’ (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’

Fixes 6bc67c497a51 ("eal: add uuid API")
---
 lib/librte_eal/include/rte_uuid.h | 2 ++
 1 file changed, 2 insertions(+)
lib/librte_eal/include/rte_uuid.h +2 -0
index 044afbdfa..8b42e070a 100644
@@ -15,6 +15,8 @@ extern "C" {
 #endif
 
 #include <stdbool.h>
+#include <stddef.h>
+#include <string.h>
 
 /**
  * Struct describing a Universal Unique Identifier
Signature · 1 line
2.26.2
#48 lore raw

PATCH v9 2/2 eal: support for VFIO-PCI VF token

In reply to Haiyue Wang

The kernel module vfio-pci introduces the VF token to enable SR-IOV support since 5.7.

The VF token can be set by a vfio-pci based PF driver and must be known by the vfio-pci based VF driver in order to gain access to the device.

---
 devtools/libabigail.abignore           |  2 +
 doc/guides/linux_gsg/linux_drivers.rst | 41 +++++++++++++-
 doc/guides/rel_notes/release_20_05.rst |  5 ++
 drivers/bus/pci/linux/pci_vfio.c       | 74 +++++++++++++++++++++++++-
 lib/librte_eal/freebsd/eal.c           |  3 +-
 lib/librte_eal/include/rte_vfio.h      | 24 ++++++++-
 lib/librte_eal/linux/eal_vfio.c        | 20 +++++--
 7 files changed, 161 insertions(+), 8 deletions(-)
devtools/libabigail.abignore +2 -0
index cd86d89ca..01d987a1e 100644
@@ -6,6 +6,8 @@
 ; Explicit ignore for driver-only ABI
 [suppress_type]
         name = rte_cryptodev_ops
+[suppress_function]
+	name = rte_vfio_setup_device
 ; Ignore this enum update as it is part of an experimental API
 [suppress_type]
         type_kind = enum
doc/guides/linux_gsg/linux_drivers.rst +40 -1

51 lines not on this page · raw message

doc/guides/rel_notes/release_20_05.rst +5 -0
index 709372e5e..9460e1eb2 100644
@@ -97,6 +97,11 @@ New Features
   by making use of the event device capabilities. The event mode currently supports
   only inline IPsec protocol offload.
 
+* **Added the support for vfio-pci new VF token interface.**
+
+  Since Linux version 5.7, vfio-pci supports a shared VF token (UUID) to represent
+  the trust between SR-IOV PF and the created VFs. Update the method to gain access
+  to the device by appending the VF token.
 
 Removed Items
 -------------
drivers/bus/pci/linux/pci_vfio.c +72 -2

112 lines not on this page · raw message

lib/librte_eal/freebsd/eal.c +2 -1
index 80dc9aa78..86d5a5f49 100644
@@ -995,7 +995,8 @@ rte_eal_vfio_intr_mode(void)
 int rte_vfio_setup_device(__rte_unused const char *sysfs_base,
 		      __rte_unused const char *dev_addr,
 		      __rte_unused int *vfio_dev_fd,
-		      __rte_unused struct vfio_device_info *device_info)
+		      __rte_unused struct vfio_device_info *device_info,
+		      __rte_unused rte_uuid_t vf_token)
 {
 	return -1;
 }
lib/librte_eal/include/rte_vfio.h +23 -1
index 20ed8c45a..28d918cde 100644
@@ -16,6 +16,8 @@ extern "C" {
 
 #include <stdint.h>
 
+#include <rte_uuid.h>
+
 /*
  * determine if VFIO is present on the system
  */
@@ -102,13 +104,33 @@ struct vfio_device_info;
  * @param device_info
  *   Device information.
  *
+ * @param vf_token
+ *   Before linux 5.7, the PF bound to vfio-pci doesn't support SR-IOV to
+ *   create VFs for security reason. Now the VF token is introduced to work
+ *   as some degree of trust or collaboration between PF and VFs.
+ *
+ *   A). as VF device, if the PF is a vfio device and it is bound to the
+ *   vfio-pci driver, the user needs to provide a VF token to access the
+ *   device, in the form of appending a vf_token to the device name, for
+ *   example:
+ *     "-w 04:10.0,vf_token=bd8d9d2b-5a5f-4f5a-a211-f591514ba1f3"
+ *
+ *   B). as PF device, When presented with a PF which has VFs in use, the
+ *   user must also provide the current VF token to prove collaboration with
+ *   existing VF users.  If VFs are not in use, the VF token provided for the
+ *   PF device will act to set the VF token.
+ *
+ *   The vf_token can be zero uuid, which will be ignored to pass into the
+ *   vfio-pci module.
+ *
  * @return
  *   0 on success.
  *   <0 on failure.
  *   >1 if the device cannot be managed this way.
  */
 int rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
-		int *vfio_dev_fd, struct vfio_device_info *device_info);
+		int *vfio_dev_fd, struct vfio_device_info *device_info,
+		rte_uuid_t vf_token);
 
 /**
  * Release a device mapped to a VFIO-managed I/O MMU group.
lib/librte_eal/linux/eal_vfio.c +17 -3
index d26e1649a..e8d7cbda5 100644
@@ -702,7 +702,8 @@ rte_vfio_clear_group(int vfio_group_fd)
 
 int
 rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
-		int *vfio_dev_fd, struct vfio_device_info *device_info)
+		int *vfio_dev_fd, struct vfio_device_info *device_info,
+		rte_uuid_t vf_token)
 {
 	struct vfio_group_status group_status = {
 			.argsz = sizeof(group_status)
@@ -712,6 +713,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 	int vfio_container_fd;
 	int vfio_group_fd;
 	int iommu_group_num;
+	char dev[PATH_MAX];
 	int i, ret;
 
 	/* get group number */
@@ -895,8 +897,19 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 				t->type_id, t->name);
 	}
 
+	if (!rte_uuid_is_null(vf_token)) {
+		char vf_token_str[RTE_UUID_STRLEN];
+
+		rte_uuid_unparse(vf_token, vf_token_str, sizeof(vf_token_str));
+		snprintf(dev, sizeof(dev),
+			 "%s vf_token=%s", dev_addr, vf_token_str);
+	} else {
+		snprintf(dev, sizeof(dev),
+			 "%s", dev_addr);
+	}
+
 	/* get a file descriptor for the device */
-	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev_addr);
+	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev);
 	if (*vfio_dev_fd < 0) {
 		/* if we cannot get a device fd, this implies a problem with
 		 * the VFIO group or the container not having IOMMU configured.
@@ -2083,7 +2096,8 @@ int
 rte_vfio_setup_device(__rte_unused const char *sysfs_base,
 		__rte_unused const char *dev_addr,
 		__rte_unused int *vfio_dev_fd,
-		__rte_unused struct vfio_device_info *device_info)
+		__rte_unused struct vfio_device_info *device_info,
+		__rte_unused rte_uuid_t vf_token)
 {
 	return -1;
 }
Signature · 1 line
2.26.2
#49 lore raw

Re: [dpdk-dev] [PATCH v9 2/2] eal: support for VFIO-PCI VF token

22/04/2020 07:08, Haiyue Wang:

devtools/libabigail.abignore line 10

Quoted 4 lines
devtools/libabigail.abignore +2 -0
+[suppress_function]
+	name = rte_vfio_setup_device

There is no comment to justify ignoring this symbol.

The current policy is to reject any change in libabigail.abignore which could be better handled with (unfinished) rte_internal mark.

#50 lore raw

Re: [dpdk-dev] [PATCH v9 2/2] eal: support for VFIO-PCI VF token

Quoted 16 lines

-----Original Message-----
From: Thomas Monjalon <[email protected]>
Sent: Wednesday, April 22, 2020 16:24
To: Wang, Haiyue <[email protected]>
Cc: [email protected]; Burakov, Anatoly <[email protected]>; [email protected]; [email protected]; [email protected]; [email protected]
Subject: Re: [PATCH v9 2/2] eal: support for VFIO-PCI VF token

22/04/2020 07:08, Haiyue Wang:

devtools/libabigail.abignore line 10

Quoted 4 lines
devtools/libabigail.abignore +2 -0
+[suppress_function]
+	name = rte_vfio_setup_device

There is no comment to justify ignoring this symbol.

Quoted 4 lines

The current policy is to reject any change in libabigail.abignore which could be better handled with (unfinished) rte_internal mark.

While updating the vfio patch, also doing the ABI things. Found that:
1). [suppress_function]
name = rte_vfio_setup_device
can make CI happy, but not scale.

2). rte_internal is beautiful, but make CI cry : https://patchwork.dpdk.org/patch/69109/
1 Removed function:

[D] 'function int rte_vfio_setup_device(const char*, const char*, int*, vfio_device_info*)' {rte_vfio_setup_device@@DPDK_20.0}

#51 lore raw

v10 cover support for VFIO-PCI VF token interface

In reply to [email protected]

v10: Use the __rte_internal to mark the internal API changing.

v9: Rewrite the document.

v8: Update the document.

v7: Add the Fixes tag in uuid, the release note and help
document.

v6: Drop the Fixes tag in uuid, since the file has been
moved to another place, not suitable to apply on stable. And this is not a bug, just some kind of enhancement.

v5: 1. Add the VF token parse error handling.
2. Split into two patches for different logic module.
3. Add more comments into the code for explaining the design.
4. Drop the ABI change workaround, this patch set focuses on code review.

v4: 1. Ignore rte_vfio_setup_device ABI check since it is
for Linux driver use.

v3: Fix the Travis build failed:
(1). rte_uuid.h:97:55: error: unknown type name ‘size_t’ (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’

v2: Fix the FreeBSD build error.

v1: Update the commit message.

RFC v2:
Based on Vamsi's RFC v1, and Alex's patch for Qemu
[https://lore.kernel.org/lkml/[email protected]/]:
Use the devarg to pass-down the VF token.

RFC v1: https://patchwork.dpdk.org/patch/66281/ by Vamsi.

Haiyue Wang (2):
eal: add uuid dependent header files explicitly
eal: support for VFIO-PCI VF token

 doc/guides/linux_gsg/linux_drivers.rst | 41 +++++++++++++-
 doc/guides/rel_notes/release_20_05.rst |  6 +++
 drivers/bus/pci/linux/pci_vfio.c       | 74 +++++++++++++++++++++++++-
 lib/librte_eal/freebsd/eal.c           |  3 +-
 lib/librte_eal/include/rte_uuid.h      |  2 +
 lib/librte_eal/include/rte_vfio.h      | 26 ++++++++-
 lib/librte_eal/linux/eal_vfio.c        | 20 +++++--
 lib/librte_eal/rte_eal_version.map     |  8 ++-
 8 files changed, 171 insertions(+), 9 deletions(-)
Signature · 1 line
2.26.2
#52 lore raw

PATCH v10 1/2 eal: add uuid dependent header files explicitly

Add the dependent header files explicitly, so that the user just needs to include the 'rte_uuid.h' header file directly to avoid compile error:
(1). rte_uuid.h:97:55: error: unknown type name ‘size_t’ (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’

Fixes 6bc67c497a51 ("eal: add uuid API")
---
 lib/librte_eal/include/rte_uuid.h | 2 ++
 1 file changed, 2 insertions(+)
lib/librte_eal/include/rte_uuid.h +2 -0
index 044afbdfa..8b42e070a 100644
@@ -15,6 +15,8 @@ extern "C" {
 #endif
 
 #include <stdbool.h>
+#include <stddef.h>
+#include <string.h>
 
 /**
  * Struct describing a Universal Unique Identifier
Signature · 1 line
2.26.2
#53 lore raw

PATCH v10 2/2 eal: support for VFIO-PCI VF token

In reply to Haiyue Wang

The kernel module vfio-pci introduces the VF token to enable SR-IOV support since 5.7.

The VF token can be set by a vfio-pci based PF driver and must be known by the vfio-pci based VF driver in order to gain access to the device.

---
 doc/guides/linux_gsg/linux_drivers.rst | 41 +++++++++++++-
 doc/guides/rel_notes/release_20_05.rst |  6 +++
 drivers/bus/pci/linux/pci_vfio.c       | 74 +++++++++++++++++++++++++-
 lib/librte_eal/freebsd/eal.c           |  3 +-
 lib/librte_eal/include/rte_vfio.h      | 26 ++++++++-
 lib/librte_eal/linux/eal_vfio.c        | 20 +++++--
 lib/librte_eal/rte_eal_version.map     |  8 ++-
 7 files changed, 169 insertions(+), 9 deletions(-)
doc/guides/linux_gsg/linux_drivers.rst +40 -1
index 238f3e900..b42fd708b 100644
@@ -72,11 +72,50 @@ Note that in order to use VFIO, your kernel must support it.
 VFIO kernel modules have been included in the Linux kernel since version 3.6.0 and are usually present by default,
 however please consult your distributions documentation to make sure that is the case.
 
+The ``vfio-pci`` module since Linux version 5.7 supports the creation of virtual
+functions. After the PF is bound to vfio-pci module, the user can create the VFs
+by sysfs interface, and these VFs are bound to vfio-pci module automatically.
+
+When the PF is bound to vfio-pci, it has initial VF token generated by random. For
+security reason, this token is write only, the user can't read it from the kernel
+directly. For accessing the VF, the user needs to start the PF with token parameter
+to setup a VF token (uuid format), then the VF can be accessed with this new known
+VF token.
+
+Also if the DPDK application running on the PF device exits, the user wants to start
+the PF with another different VF token value, it has no issue if no application like
+DPDK or KVM runs on VFs. Otherwise, the PF will fail to start until all VFs are free
+to use, after that, the user can select a new VF token to start the PF device.
+
+DPDK will use the keyword ``vf_token`` as the device argument to pass the VF token
+value to PF and its related VFs, the PMD should not use it, and this argument will
+be pruned from the device argument list, so the PMD can parse its own valid device
+arguments successfully.
+
+.. code-block:: console
+
+    1. Generate the VF token by uuid command
+        14d63f20-8445-11ea-8900-1f9ce7d5650d
+
+    2. sudo modprobe vfio-pci enable_sriov=1
+
+    2. ./usertools/dpdk-devbind.py -b vfio-pci 0000:86:00.0
+
+    3. echo 2 > /sys/bus/pci/devices/0000:86:00.0/sriov_numvfs
+
+    4. Start the PF:
+        ./x86_64-native-linux-gcc/app/testpmd -l 22-25 -n 4 \
+         -w 86:00.0,vf_token=14d63f20-8445-11ea-8900-1f9ce7d5650d --file-prefix=pf -- -i
+
+    5. Start the VF:
+        ./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 \
+         -w 86:02.0,vf_token=14d63f20-8445-11ea-8900-1f9ce7d5650d --file-prefix=vf0 -- -i
+
 Also, to use VFIO, both kernel and BIOS must support and be configured to use IO virtualization (such as Intel® VT-d).
 
 .. note::
 
-    ``vfio-pci`` module doesn't support the creation of virtual functions.
+    ``vfio-pci`` module doesn't support the creation of virtual functions before Linux version 5.7.
 
 For proper operation of VFIO when running DPDK applications as a non-privileged user, correct permissions should also be set up.
 This can be done by using the DPDK setup script (called dpdk-setup.sh and located in the usertools directory).
doc/guides/rel_notes/release_20_05.rst +6 -0
index b124c3f28..722c61e67 100644
@@ -93,6 +93,12 @@ New Features
   * Added new query: ``rte_flow_get_aged_flows`` to get the aged-out flows
     contexts from the port.
 
+* **Added the support for vfio-pci new VF token interface.**
+
+  Since Linux version 5.7, vfio-pci supports a shared VF token (UUID) to represent
+  the trust between SR-IOV PF and the created VFs. Update the method to gain access
+  to the PF and VFs devices by appending the VF token parameter.
+
 * **Updated Amazon ena driver.**
 
   Updated ena PMD with new features and improvements, including:
drivers/bus/pci/linux/pci_vfio.c +72 -2

112 lines not on this page · raw message

lib/librte_eal/freebsd/eal.c +2 -1
index 540b7d38c..e42d3e12e 100644
@@ -1005,7 +1005,8 @@ rte_eal_vfio_intr_mode(void)
 int rte_vfio_setup_device(__rte_unused const char *sysfs_base,
 		      __rte_unused const char *dev_addr,
 		      __rte_unused int *vfio_dev_fd,
-		      __rte_unused struct vfio_device_info *device_info)
+		      __rte_unused struct vfio_device_info *device_info,
+		      __rte_unused rte_uuid_t vf_token)
 {
 	return -1;
 }
lib/librte_eal/include/rte_vfio.h +25 -1
index 20ed8c45a..430f98770 100644
@@ -16,6 +16,9 @@ extern "C" {
 
 #include <stdint.h>
 
+#include <rte_compat.h>
+#include <rte_uuid.h>
+
 /*
  * determine if VFIO is present on the system
  */
@@ -102,13 +105,34 @@ struct vfio_device_info;
  * @param device_info
  *   Device information.
  *
+ * @param vf_token
+ *   Before linux 5.7, the PF bound to vfio-pci doesn't support SR-IOV to
+ *   create VFs for security reason. Now the VF token is introduced to work
+ *   as some degree of trust or collaboration between PF and VFs.
+ *
+ *   A). as VF device, if the PF is a vfio device and it is bound to the
+ *   vfio-pci driver, the user needs to provide a VF token to access the
+ *   device, in the form of appending a vf_token to the device name, for
+ *   example:
+ *     "-w 04:10.0,vf_token=bd8d9d2b-5a5f-4f5a-a211-f591514ba1f3"
+ *
+ *   B). as PF device, When presented with a PF which has VFs in use, the
+ *   user must also provide the current VF token to prove collaboration with
+ *   existing VF users.  If VFs are not in use, the VF token provided for the
+ *   PF device will act to set the VF token.
+ *
+ *   The vf_token can be zero uuid, which will be ignored to pass into the
+ *   vfio-pci module.
+ *
  * @return
  *   0 on success.
  *   <0 on failure.
  *   >1 if the device cannot be managed this way.
  */
+__rte_internal
 int rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
-		int *vfio_dev_fd, struct vfio_device_info *device_info);
+		int *vfio_dev_fd, struct vfio_device_info *device_info,
+		rte_uuid_t vf_token);
 
 /**
  * Release a device mapped to a VFIO-managed I/O MMU group.
lib/librte_eal/linux/eal_vfio.c +17 -3
index d26e1649a..e8d7cbda5 100644
@@ -702,7 +702,8 @@ rte_vfio_clear_group(int vfio_group_fd)
 
 int
 rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
-		int *vfio_dev_fd, struct vfio_device_info *device_info)
+		int *vfio_dev_fd, struct vfio_device_info *device_info,
+		rte_uuid_t vf_token)
 {
 	struct vfio_group_status group_status = {
 			.argsz = sizeof(group_status)
@@ -712,6 +713,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 	int vfio_container_fd;
 	int vfio_group_fd;
 	int iommu_group_num;
+	char dev[PATH_MAX];
 	int i, ret;
 
 	/* get group number */
@@ -895,8 +897,19 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 				t->type_id, t->name);
 	}
 
+	if (!rte_uuid_is_null(vf_token)) {
+		char vf_token_str[RTE_UUID_STRLEN];
+
+		rte_uuid_unparse(vf_token, vf_token_str, sizeof(vf_token_str));
+		snprintf(dev, sizeof(dev),
+			 "%s vf_token=%s", dev_addr, vf_token_str);
+	} else {
+		snprintf(dev, sizeof(dev),
+			 "%s", dev_addr);
+	}
+
 	/* get a file descriptor for the device */
-	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev_addr);
+	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev);
 	if (*vfio_dev_fd < 0) {
 		/* if we cannot get a device fd, this implies a problem with
 		 * the VFIO group or the container not having IOMMU configured.
@@ -2083,7 +2096,8 @@ int
 rte_vfio_setup_device(__rte_unused const char *sysfs_base,
 		__rte_unused const char *dev_addr,
 		__rte_unused int *vfio_dev_fd,
-		__rte_unused struct vfio_device_info *device_info)
+		__rte_unused struct vfio_device_info *device_info,
+		__rte_unused rte_uuid_t vf_token)
 {
 	return -1;
 }
lib/librte_eal/rte_eal_version.map +7 -1
index 6088e7f6c..5bcdccae6 100644
@@ -213,7 +213,6 @@ DPDK_20.0 {
 	rte_vfio_is_enabled;
 	rte_vfio_noiommu_is_enabled;
 	rte_vfio_release_device;
-	rte_vfio_setup_device;
 	rte_vlog;
 	rte_zmalloc;
 	rte_zmalloc_socket;
@@ -388,3 +387,10 @@ EXPERIMENTAL {
 	rte_trace_regexp;
 	rte_trace_save;
 };
+
+INTERNAL {
+	global:
+
+	# added in 20.05
+	rte_vfio_setup_device;
+};
Signature · 1 line
2.26.2
Acked-by on v10 2/2
#54 lore raw

Re: [dpdk-dev] [PATCH v10 2/2] eal: support for VFIO-PCI VF token

Quoted 10 lines · Haiyue Wang

The kernel module vfio-pci introduces the VF token to enable SR-IOV support since 5.7.

The VF token can be set by a vfio-pci based PF driver and must be known by the vfio-pci based VF driver in order to gain access to the device.

---

LGTM

Signature · 2 lines
Thanks,
Anatoly
Acked-by on v10 2/2
#55 lore raw

Re: [dpdk-dev] [PATCH v10 2/2] eal: support for VFIO-PCI VF token

In reply to Haiyue Wang
Quoted 9 lines · Haiyue Wang

The kernel module vfio-pci introduces the VF token to enable SR-IOV support since 5.7.

The VF token can be set by a vfio-pci based PF driver and must be known by the vfio-pci based VF driver in order to gain access to the device.

Few points below, anyway:

[snip]

Quoted 7 lines
doc/guides/linux_gsg/linux_drivers.rst +0 -0
index 238f3e900..b42fd708b 100644
@@ -72,11 +72,50 @@ Note that in order to use VFIO, your kernel must support it.
 VFIO kernel modules have been included in the Linux kernel since version 3.6.0 and are usually present by default,
 however please consult your distributions documentation to make sure that is the case.

doc/guides/linux_gsg/linux_drivers.rst Note that in order to use VFIO, your kernel must support it. · line 83

Quoted 10 lines
doc/guides/linux_gsg/linux_drivers.rst +9 -0
 
+The ``vfio-pci`` module since Linux version 5.7 supports the creation of virtual
+functions. After the PF is bound to vfio-pci module, the user can create the VFs
+by sysfs interface, and these VFs are bound to vfio-pci module automatically.
+
+When the PF is bound to vfio-pci, it has initial VF token generated by random. For
+security reason, this token is write only, the user can't read it from the kernel
+directly. For accessing the VF, the user needs to start the PF with token parameter
+to setup a VF token (uuid format), then the VF can be accessed with this new known
+VF token.

If token is write-only in kernel sysfs, shouldn't we make it invisible in ps output (procfs)? I.e. substitute with something like xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx.
However, it does not look easy. Just a thought.

[snip]

Quoted 19 lines
drivers/bus/pci/linux/pci_vfio.c +4 -0
index 64cd84a68..efb64e2ba 100644
@@ -11,6 +11,7 @@
 #include <sys/mman.h>
 #include <stdbool.h>
 
+#include <rte_devargs.h>
 #include <rte_log.h>
 #include <rte_pci.h>
 #include <rte_bus_pci.h>
@@ -644,12 +645,72 @@ pci_vfio_msix_is_mappable(int vfio_dev_fd, int msix_region)
 	return ret;
 }
 
+static int
+vfio_pci_vf_token_arg(struct rte_devargs *devargs, rte_uuid_t uuid)
+{

drivers/bus/pci/linux/pci_vfio.c pci_vfio_msix_is_mappable(int vfio_dev_fd, int msix_region) · line 660

Quoted 10 lines
drivers/bus/pci/linux/pci_vfio.c +10 -0
+#define VF_TOKEN_ARG "vf_token="
+	char c, *p, *vf_token;
+
+	memset(uuid, 0, sizeof(rte_uuid_t));
+
+	if (devargs == NULL)
+		return 0;
+
+	p = strstr(devargs->args, VF_TOKEN_ARG);
+	if (!p)

Please, p == NULL as you do few lines above.

[snip]

#56 lore raw

v11 cover support for VFIO-PCI VF token interface

In reply to [email protected]

v11: Use the eal parameter to pass the VF token, then not every PCI
device needs to be specified with this token. Also no ABI issue now.

v10: Use the __rte_internal to mark the internal API changing.

v9: Rewrite the document.

v8: Update the document.

v7: Add the Fixes tag in uuid, the release note and help
document.

v6: Drop the Fixes tag in uuid, since the file has been
moved to another place, not suitable to apply on stable. And this is not a bug, just some kind of enhancement.

v5: 1. Add the VF token parse error handling.
2. Split into two patches for different logic module.
3. Add more comments into the code for explaining the design.
4. Drop the ABI change workaround, this patch set focuses on code review.

v4: 1. Ignore rte_vfio_setup_device ABI check since it is
for Linux driver use.

v3: Fix the Travis build failed:
(1). rte_uuid.h:97:55: error: unknown type name ‘size_t’ (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’

v2: Fix the FreeBSD build error.

v1: Update the commit message.

RFC v2:
Based on Vamsi's RFC v1, and Alex's patch for Qemu
[https://lore.kernel.org/lkml/[email protected]/]:
Use the devarg to pass-down the VF token.

RFC v1: https://patchwork.dpdk.org/patch/66281/ by Vamsi.

Haiyue Wang (2):
eal: add uuid dependent header files explicitly
eal: support for VFIO-PCI VF token

 doc/guides/linux_gsg/linux_drivers.rst        | 35 ++++++++++++++++++-
 doc/guides/linux_gsg/linux_eal_parameters.rst |  4 +++
 doc/guides/rel_notes/release_20_05.rst        |  6 ++++
 lib/librte_eal/common/eal_common_options.c    |  2 ++
 lib/librte_eal/common/eal_internal_cfg.h      |  2 ++
 lib/librte_eal/common/eal_options.h           |  2 ++
 lib/librte_eal/freebsd/eal.c                  |  5 +++
 lib/librte_eal/include/rte_eal.h              | 12 +++++++
 lib/librte_eal/include/rte_uuid.h             |  2 ++
 lib/librte_eal/linux/eal.c                    | 29 +++++++++++++++
 lib/librte_eal/linux/eal_vfio.c               | 16 ++++++++-
 lib/librte_eal/rte_eal_version.map            |  1 +
 12 files changed, 114 insertions(+), 2 deletions(-)
Signature · 1 line
2.26.2
#57 lore raw

PATCH v11 1/2 eal: add uuid dependent header files explicitly

Add the dependent header files explicitly, so that the user just needs to include the 'rte_uuid.h' header file directly to avoid compile error:
(1). rte_uuid.h:97:55: error: unknown type name ‘size_t’ (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’

Fixes 6bc67c497a51 ("eal: add uuid API")
---
 lib/librte_eal/include/rte_uuid.h | 2 ++
 1 file changed, 2 insertions(+)
lib/librte_eal/include/rte_uuid.h +2 -0
index 044afbdfa..8b42e070a 100644
@@ -15,6 +15,8 @@ extern "C" {
 #endif
 
 #include <stdbool.h>
+#include <stddef.h>
+#include <string.h>
 
 /**
  * Struct describing a Universal Unique Identifier
Signature · 1 line
2.26.2
#58 lore raw

PATCH v11 2/2 eal: support for VFIO-PCI VF token

In reply to Haiyue Wang

The kernel module vfio-pci introduces the VF token to enable SR-IOV support since 5.7.

The VF token can be set by a vfio-pci based PF driver and must be known by the vfio-pci based VF driver in order to gain access to the device.

---
 doc/guides/linux_gsg/linux_drivers.rst        | 35 ++++++++++++++++++-
 doc/guides/linux_gsg/linux_eal_parameters.rst |  4 +++
 doc/guides/rel_notes/release_20_05.rst        |  6 ++++
 lib/librte_eal/common/eal_common_options.c    |  2 ++
 lib/librte_eal/common/eal_internal_cfg.h      |  2 ++
 lib/librte_eal/common/eal_options.h           |  2 ++
 lib/librte_eal/freebsd/eal.c                  |  5 +++
 lib/librte_eal/include/rte_eal.h              | 12 +++++++
 lib/librte_eal/linux/eal.c                    | 29 +++++++++++++++
 lib/librte_eal/linux/eal_vfio.c               | 16 ++++++++-
 lib/librte_eal/rte_eal_version.map            |  1 +
 11 files changed, 112 insertions(+), 2 deletions(-)
doc/guides/linux_gsg/linux_drivers.rst +34 -1
index 238f3e900..910397243 100644
@@ -72,11 +72,44 @@ Note that in order to use VFIO, your kernel must support it.
 VFIO kernel modules have been included in the Linux kernel since version 3.6.0 and are usually present by default,
 however please consult your distributions documentation to make sure that is the case.
 
+The ``vfio-pci`` module since Linux version 5.7 supports the creation of virtual
+functions. After the PF is bound to vfio-pci module, the user can create the VFs
+by sysfs interface, and these VFs are bound to vfio-pci module automatically.
+
+When the PF is bound to vfio-pci, it has initial VF token generated by random. For
+security reason, this token is write only, the user can't read it from the kernel
+directly. For accessing the VF, the user needs to start the PF with token parameter
+to setup a VF token (uuid format), then the VF can be accessed with this new known
+VF token.
+
+DPDK will use the EAL parameter ``--vfio-vf-token`` to specify the VF token value to
+PF and its related VFs, this VF token will be shared in all VFIO devices, including
+the different PFs.
+
+.. code-block:: console
+
+    1. Generate the VF token by uuid command
+        14d63f20-8445-11ea-8900-1f9ce7d5650d
+
+    2. sudo modprobe vfio-pci enable_sriov=1
+
+    2. ./usertools/dpdk-devbind.py -b vfio-pci 0000:86:00.0
+
+    3. echo 2 > /sys/bus/pci/devices/0000:86:00.0/sriov_numvfs
+
+    4. Start the PF:
+        ./x86_64-native-linux-gcc/app/testpmd -l 22-25 -n 4 -w 86:00.0 \
+         --vfio-vf-token=14d63f20-8445-11ea-8900-1f9ce7d5650d --file-prefix=pf -- -i
+
+    5. Start the VF:
+        ./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 -w 86:02.0 \
+         --vfio-vf-token=14d63f20-8445-11ea-8900-1f9ce7d5650d --file-prefix=vf0 -- -i
+
 Also, to use VFIO, both kernel and BIOS must support and be configured to use IO virtualization (such as Intel® VT-d).
 
 .. note::
 
-    ``vfio-pci`` module doesn't support the creation of virtual functions.
+    ``vfio-pci`` module doesn't support the creation of virtual functions before Linux version 5.7.
 
 For proper operation of VFIO when running DPDK applications as a non-privileged user, correct permissions should also be set up.
 This can be done by using the DPDK setup script (called dpdk-setup.sh and located in the usertools directory).
doc/guides/linux_gsg/linux_eal_parameters.rst +4 -0
index b2cc60e44..bd3977cb3 100644
@@ -40,6 +40,10 @@ Device-related options
 
     Use specified interrupt mode for devices bound to VFIO kernel driver.
 
+*   ``--vfio-vf-token <uuid>``
+
+    Use specified VF token for devices bound to VFIO kernel driver.
+
 Multiprocessing-related options
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
doc/guides/rel_notes/release_20_05.rst +6 -0
index ab20a7d02..c81deb06c 100644
@@ -93,6 +93,12 @@ New Features
   * Added new query: ``rte_flow_get_aged_flows`` to get the aged-out flows
     contexts from the port.
 
+* **Added the support for vfio-pci new VF token interface.**
+
+  Since Linux version 5.7, vfio-pci supports a shared VF token (UUID) to represent
+  the trust between SR-IOV PF and the created VFs. Update the method to gain access
+  to the PF and VFs devices by appending the VF token parameter.
+
 * **Updated Amazon ena driver.**
 
   Updated ena PMD with new features and improvements, including:
lib/librte_eal/common/eal_common_options.c +2 -0
index 418731ca4..dbce919ad 100644
@@ -89,6 +89,7 @@ eal_long_options[] = {
 	{OPT_SYSLOG,            1, NULL, OPT_SYSLOG_NUM           },
 	{OPT_VDEV,              1, NULL, OPT_VDEV_NUM             },
 	{OPT_VFIO_INTR,         1, NULL, OPT_VFIO_INTR_NUM        },
+	{OPT_VFIO_VF_TOKEN,     1, NULL, OPT_VFIO_VF_TOKEN_NUM    },
 	{OPT_VMWARE_TSC_MAP,    0, NULL, OPT_VMWARE_TSC_MAP_NUM   },
 	{OPT_LEGACY_MEM,        0, NULL, OPT_LEGACY_MEM_NUM       },
 	{OPT_SINGLE_FILE_SEGMENTS, 0, NULL, OPT_SINGLE_FILE_SEGMENTS_NUM},
@@ -222,6 +223,7 @@ eal_reset_internal_config(struct internal_config *internal_cfg)
 
 	/* if set to NONE, interrupt mode is determined automatically */
 	internal_cfg->vfio_intr_mode = RTE_INTR_MODE_NONE;
+	memset(internal_cfg->vfio_vf_token, 0, sizeof(rte_uuid_t));
 
 #ifdef RTE_LIBEAL_USE_HPET
 	internal_cfg->no_hpet = 0;
lib/librte_eal/common/eal_internal_cfg.h +2 -0
index a42f34923..28154f0ef 100644
@@ -72,6 +72,8 @@ struct internal_config {
 	volatile int syslog_facility;	  /**< facility passed to openlog() */
 	/** default interrupt mode for VFIO */
 	volatile enum rte_intr_mode vfio_intr_mode;
+	/** the shared VF token for VFIO-PCI bound PF and VFs devices */
+	rte_uuid_t vfio_vf_token;
 	char *hugefile_prefix;      /**< the base filename of hugetlbfs files */
 	char *hugepage_dir;         /**< specific hugetlbfs directory to use */
 	char *user_mbuf_pool_ops_name;
lib/librte_eal/common/eal_options.h +2 -0
index 90ead1b7c..cd5fd01f2 100644
@@ -67,6 +67,8 @@ enum {
 	OPT_VDEV_NUM,
 #define OPT_VFIO_INTR         "vfio-intr"
 	OPT_VFIO_INTR_NUM,
+#define OPT_VFIO_VF_TOKEN     "vfio-vf-token"
+	OPT_VFIO_VF_TOKEN_NUM,
 #define OPT_VMWARE_TSC_MAP    "vmware-tsc-map"
 	OPT_VMWARE_TSC_MAP_NUM,
 #define OPT_LEGACY_MEM    "legacy-mem"
lib/librte_eal/freebsd/eal.c +5 -0
index 540b7d38c..00b02855e 100644
@@ -1002,6 +1002,11 @@ rte_eal_vfio_intr_mode(void)
 	return RTE_INTR_MODE_NONE;
 }
 
+void rte_eal_vfio_vf_token(rte_uuid_t vf_token)
+{
+	memset(vf_token, 0, sizeof(rte_uuid_t));
+}
+
 int rte_vfio_setup_device(__rte_unused const char *sysfs_base,
 		      __rte_unused const char *dev_addr,
 		      __rte_unused int *vfio_dev_fd,
lib/librte_eal/include/rte_eal.h +12 -0
index 2f9ed298d..d9040ce14 100644
@@ -21,6 +21,7 @@
 #include <rte_bus.h>
 
 #include <rte_pci_dev_feature_defs.h>
+#include <rte_uuid.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -438,6 +439,17 @@ int rte_eal_create_uio_dev(void);
  */
 enum rte_intr_mode rte_eal_vfio_intr_mode(void);
 
+
+/**
+ * The user-configured vfio VF token.
+ *
+ * @param vf_token
+ *   vfio VF token configured with the command line is copied
+ *   into this parameter, zero uuid by default.
+ */
+__rte_experimental
+void rte_eal_vfio_vf_token(rte_uuid_t vf_token);
+
 /**
  * A wrap API for syscall gettid.
  *
lib/librte_eal/linux/eal.c +29 -0
index aa72d3650..05cd6e497 100644
@@ -558,6 +558,7 @@ eal_usage(const char *prgname)
 	       "  --"OPT_FILE_PREFIX"       Prefix for hugepage filenames\n"
 	       "  --"OPT_CREATE_UIO_DEV"    Create /dev/uioX (usually done by hotplug)\n"
 	       "  --"OPT_VFIO_INTR"         Interrupt mode for VFIO (legacy|msi|msix)\n"
+	       "  --"OPT_VFIO_VF_TOKEN"     VF token (UUID) shared between SR-IOV PF and VFs\n"
 	       "  --"OPT_LEGACY_MEM"        Legacy memory mode (no dynamic allocation, contiguous segments)\n"
 	       "  --"OPT_SINGLE_FILE_SEGMENTS" Put all hugepage memory in single files\n"
 	       "  --"OPT_MATCH_ALLOCATIONS" Free hugepages exactly as allocated\n"
@@ -649,6 +650,19 @@ eal_parse_vfio_intr(const char *mode)
 	return -1;
 }
 
+static int
+eal_parse_vfio_vf_token(const char *vf_token)
+{
+	rte_uuid_t uuid;
+
+	if (!rte_uuid_parse(vf_token, uuid)) {
+		rte_uuid_copy(internal_config.vfio_vf_token, uuid);
+		return 0;
+	}
+
+	return -1;
+}
+
 /* Parse the arguments for --log-level only */
 static void
 eal_log_level_parse(int argc, char **argv)
@@ -795,6 +809,16 @@ eal_parse_args(int argc, char **argv)
 			}
 			break;
 
+		case OPT_VFIO_VF_TOKEN_NUM:
+			if (eal_parse_vfio_vf_token(optarg) < 0) {
+				RTE_LOG(ERR, EAL, "invalid parameters for --"
+						OPT_VFIO_VF_TOKEN "\n");
+				eal_usage(prgname);
+				ret = -1;
+				goto out;
+			}
+			break;
+
 		case OPT_CREATE_UIO_DEV_NUM:
 			internal_config.create_uio_dev = 1;
 			break;
@@ -1367,6 +1391,11 @@ rte_eal_vfio_intr_mode(void)
 	return internal_config.vfio_intr_mode;
 }
 
+void rte_eal_vfio_vf_token(rte_uuid_t vf_token)
+{
+	rte_uuid_copy(vf_token, internal_config.vfio_vf_token);
+}
+
 int
 rte_eal_check_module(const char *module_name)
 {
lib/librte_eal/linux/eal_vfio.c +15 -1
index d26e1649a..e8a264bea 100644
@@ -712,6 +712,8 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 	int vfio_container_fd;
 	int vfio_group_fd;
 	int iommu_group_num;
+	rte_uuid_t vf_token;
+	char dev[PATH_MAX];
 	int i, ret;
 
 	/* get group number */
@@ -895,8 +897,20 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 				t->type_id, t->name);
 	}
 
+	rte_eal_vfio_vf_token(vf_token);
+	if (!rte_uuid_is_null(vf_token)) {
+		char vf_token_str[RTE_UUID_STRLEN];
+
+		rte_uuid_unparse(vf_token, vf_token_str, sizeof(vf_token_str));
+		snprintf(dev, sizeof(dev),
+			 "%s vf_token=%s", dev_addr, vf_token_str);
+	} else {
+		snprintf(dev, sizeof(dev),
+			 "%s", dev_addr);
+	}
+
 	/* get a file descriptor for the device */
-	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev_addr);
+	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev);
 	if (*vfio_dev_fd < 0) {
 		/* if we cannot get a device fd, this implies a problem with
 		 * the VFIO group or the container not having IOMMU configured.
lib/librte_eal/rte_eal_version.map +1 -0
index 6088e7f6c..5b7dd3e81 100644
@@ -373,6 +373,7 @@ EXPERIMENTAL {
 	__rte_trace_point_register;
 	per_lcore_trace_mem;
 	per_lcore_trace_point_sz;
+	rte_eal_vfio_vf_token;
 	rte_log_can_log;
 	rte_thread_getname;
 	rte_trace_dump;
Signature · 1 line
2.26.2
#59 lore raw

v12 cover support for VFIO-PCI VF token interface

In reply to [email protected]

v12: support to vfio devices with VF token and no token.

v11: Use the eal parameter to pass the VF token, then not every PCI
device needs to be specified with this token. Also no ABI issue now.

v10: Use the __rte_internal to mark the internal API changing.

v9: Rewrite the document.

v8: Update the document.

v7: Add the Fixes tag in uuid, the release note and help
document.

v6: Drop the Fixes tag in uuid, since the file has been
moved to another place, not suitable to apply on stable. And this is not a bug, just some kind of enhancement.

v5: 1. Add the VF token parse error handling.
2. Split into two patches for different logic module.
3. Add more comments into the code for explaining the design.
4. Drop the ABI change workaround, this patch set focuses on code review.

v4: 1. Ignore rte_vfio_setup_device ABI check since it is
for Linux driver use.

v3: Fix the Travis build failed:
(1). rte_uuid.h:97:55: error: unknown type name ‘size_t’ (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’

v2: Fix the FreeBSD build error.

v1: Update the commit message.

RFC v2:
Based on Vamsi's RFC v1, and Alex's patch for Qemu
[https://lore.kernel.org/lkml/[email protected]/]:
Use the devarg to pass-down the VF token.

RFC v1: https://patchwork.dpdk.org/patch/66281/ by Vamsi.

Haiyue Wang (2):
eal: add uuid dependent header files explicitly
eal: support for VFIO-PCI VF token

 doc/guides/linux_gsg/linux_drivers.rst        | 35 ++++++++++++++++++-
 doc/guides/linux_gsg/linux_eal_parameters.rst |  4 +++
 doc/guides/rel_notes/release_20_05.rst        |  6 ++++
 lib/librte_eal/common/eal_common_options.c    |  2 ++
 lib/librte_eal/common/eal_internal_cfg.h      |  2 ++
 lib/librte_eal/common/eal_options.h           |  2 ++
 lib/librte_eal/freebsd/eal.c                  |  5 +++
 lib/librte_eal/include/rte_eal.h              | 12 +++++++
 lib/librte_eal/include/rte_uuid.h             |  2 ++
 lib/librte_eal/linux/eal.c                    | 29 +++++++++++++++
 lib/librte_eal/linux/eal_vfio.c               | 19 ++++++++++
 lib/librte_eal/rte_eal_version.map            |  1 +
 12 files changed, 118 insertions(+), 1 deletion(-)
Signature · 1 line
2.26.2
#60 lore raw

PATCH v12 1/2 eal: add uuid dependent header files explicitly

Add the dependent header files explicitly, so that the user just needs to include the 'rte_uuid.h' header file directly to avoid compile error:
(1). rte_uuid.h:97:55: error: unknown type name ‘size_t’ (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’

Fixes 6bc67c497a51 ("eal: add uuid API")
---
 lib/librte_eal/include/rte_uuid.h | 2 ++
 1 file changed, 2 insertions(+)
lib/librte_eal/include/rte_uuid.h +2 -0
index 044afbdfa..8b42e070a 100644
@@ -15,6 +15,8 @@ extern "C" {
 #endif
 
 #include <stdbool.h>
+#include <stddef.h>
+#include <string.h>
 
 /**
  * Struct describing a Universal Unique Identifier
Signature · 1 line
2.26.2
#61 lore raw

PATCH v12 2/2 eal: support for VFIO-PCI VF token

In reply to Haiyue Wang

The kernel module vfio-pci introduces the VF token to enable SR-IOV support since 5.7.

The VF token can be set by a vfio-pci based PF driver and must be known by the vfio-pci based VF driver in order to gain access to the device.

---
 doc/guides/linux_gsg/linux_drivers.rst        | 35 ++++++++++++++++++-
 doc/guides/linux_gsg/linux_eal_parameters.rst |  4 +++
 doc/guides/rel_notes/release_20_05.rst        |  6 ++++
 lib/librte_eal/common/eal_common_options.c    |  2 ++
 lib/librte_eal/common/eal_internal_cfg.h      |  2 ++
 lib/librte_eal/common/eal_options.h           |  2 ++
 lib/librte_eal/freebsd/eal.c                  |  5 +++
 lib/librte_eal/include/rte_eal.h              | 12 +++++++
 lib/librte_eal/linux/eal.c                    | 29 +++++++++++++++
 lib/librte_eal/linux/eal_vfio.c               | 19 ++++++++++
 lib/librte_eal/rte_eal_version.map            |  1 +
 11 files changed, 116 insertions(+), 1 deletion(-)
doc/guides/linux_gsg/linux_drivers.rst +34 -1
index 238f3e900..910397243 100644
@@ -72,11 +72,44 @@ Note that in order to use VFIO, your kernel must support it.
 VFIO kernel modules have been included in the Linux kernel since version 3.6.0 and are usually present by default,
 however please consult your distributions documentation to make sure that is the case.
 
+The ``vfio-pci`` module since Linux version 5.7 supports the creation of virtual
+functions. After the PF is bound to vfio-pci module, the user can create the VFs
+by sysfs interface, and these VFs are bound to vfio-pci module automatically.
+
+When the PF is bound to vfio-pci, it has initial VF token generated by random. For
+security reason, this token is write only, the user can't read it from the kernel
+directly. For accessing the VF, the user needs to start the PF with token parameter
+to setup a VF token (uuid format), then the VF can be accessed with this new known
+VF token.
+
+DPDK will use the EAL parameter ``--vfio-vf-token`` to specify the VF token value to
+PF and its related VFs, this VF token will be shared in all VFIO devices, including
+the different PFs.
+
+.. code-block:: console
+
+    1. Generate the VF token by uuid command
+        14d63f20-8445-11ea-8900-1f9ce7d5650d
+
+    2. sudo modprobe vfio-pci enable_sriov=1
+
+    2. ./usertools/dpdk-devbind.py -b vfio-pci 0000:86:00.0
+
+    3. echo 2 > /sys/bus/pci/devices/0000:86:00.0/sriov_numvfs
+
+    4. Start the PF:
+        ./x86_64-native-linux-gcc/app/testpmd -l 22-25 -n 4 -w 86:00.0 \
+         --vfio-vf-token=14d63f20-8445-11ea-8900-1f9ce7d5650d --file-prefix=pf -- -i
+
+    5. Start the VF:
+        ./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 -w 86:02.0 \
+         --vfio-vf-token=14d63f20-8445-11ea-8900-1f9ce7d5650d --file-prefix=vf0 -- -i
+
 Also, to use VFIO, both kernel and BIOS must support and be configured to use IO virtualization (such as Intel® VT-d).
 
 .. note::
 
-    ``vfio-pci`` module doesn't support the creation of virtual functions.
+    ``vfio-pci`` module doesn't support the creation of virtual functions before Linux version 5.7.
 
 For proper operation of VFIO when running DPDK applications as a non-privileged user, correct permissions should also be set up.
 This can be done by using the DPDK setup script (called dpdk-setup.sh and located in the usertools directory).
doc/guides/linux_gsg/linux_eal_parameters.rst +4 -0
index b2cc60e44..bd3977cb3 100644
@@ -40,6 +40,10 @@ Device-related options
 
     Use specified interrupt mode for devices bound to VFIO kernel driver.
 
+*   ``--vfio-vf-token <uuid>``
+
+    Use specified VF token for devices bound to VFIO kernel driver.
+
 Multiprocessing-related options
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
doc/guides/rel_notes/release_20_05.rst +6 -0
index ab20a7d02..c81deb06c 100644
@@ -93,6 +93,12 @@ New Features
   * Added new query: ``rte_flow_get_aged_flows`` to get the aged-out flows
     contexts from the port.
 
+* **Added the support for vfio-pci new VF token interface.**
+
+  Since Linux version 5.7, vfio-pci supports a shared VF token (UUID) to represent
+  the trust between SR-IOV PF and the created VFs. Update the method to gain access
+  to the PF and VFs devices by appending the VF token parameter.
+
 * **Updated Amazon ena driver.**
 
   Updated ena PMD with new features and improvements, including:
lib/librte_eal/common/eal_common_options.c +2 -0
index 418731ca4..dbce919ad 100644
@@ -89,6 +89,7 @@ eal_long_options[] = {
 	{OPT_SYSLOG,            1, NULL, OPT_SYSLOG_NUM           },
 	{OPT_VDEV,              1, NULL, OPT_VDEV_NUM             },
 	{OPT_VFIO_INTR,         1, NULL, OPT_VFIO_INTR_NUM        },
+	{OPT_VFIO_VF_TOKEN,     1, NULL, OPT_VFIO_VF_TOKEN_NUM    },
 	{OPT_VMWARE_TSC_MAP,    0, NULL, OPT_VMWARE_TSC_MAP_NUM   },
 	{OPT_LEGACY_MEM,        0, NULL, OPT_LEGACY_MEM_NUM       },
 	{OPT_SINGLE_FILE_SEGMENTS, 0, NULL, OPT_SINGLE_FILE_SEGMENTS_NUM},
@@ -222,6 +223,7 @@ eal_reset_internal_config(struct internal_config *internal_cfg)
 
 	/* if set to NONE, interrupt mode is determined automatically */
 	internal_cfg->vfio_intr_mode = RTE_INTR_MODE_NONE;
+	memset(internal_cfg->vfio_vf_token, 0, sizeof(rte_uuid_t));
 
 #ifdef RTE_LIBEAL_USE_HPET
 	internal_cfg->no_hpet = 0;
lib/librte_eal/common/eal_internal_cfg.h +2 -0
index a42f34923..28154f0ef 100644
@@ -72,6 +72,8 @@ struct internal_config {
 	volatile int syslog_facility;	  /**< facility passed to openlog() */
 	/** default interrupt mode for VFIO */
 	volatile enum rte_intr_mode vfio_intr_mode;
+	/** the shared VF token for VFIO-PCI bound PF and VFs devices */
+	rte_uuid_t vfio_vf_token;
 	char *hugefile_prefix;      /**< the base filename of hugetlbfs files */
 	char *hugepage_dir;         /**< specific hugetlbfs directory to use */
 	char *user_mbuf_pool_ops_name;
lib/librte_eal/common/eal_options.h +2 -0
index 90ead1b7c..cd5fd01f2 100644
@@ -67,6 +67,8 @@ enum {
 	OPT_VDEV_NUM,
 #define OPT_VFIO_INTR         "vfio-intr"
 	OPT_VFIO_INTR_NUM,
+#define OPT_VFIO_VF_TOKEN     "vfio-vf-token"
+	OPT_VFIO_VF_TOKEN_NUM,
 #define OPT_VMWARE_TSC_MAP    "vmware-tsc-map"
 	OPT_VMWARE_TSC_MAP_NUM,
 #define OPT_LEGACY_MEM    "legacy-mem"
lib/librte_eal/freebsd/eal.c +5 -0
index 540b7d38c..00b02855e 100644
@@ -1002,6 +1002,11 @@ rte_eal_vfio_intr_mode(void)
 	return RTE_INTR_MODE_NONE;
 }
 
+void rte_eal_vfio_vf_token(rte_uuid_t vf_token)
+{
+	memset(vf_token, 0, sizeof(rte_uuid_t));
+}
+
 int rte_vfio_setup_device(__rte_unused const char *sysfs_base,
 		      __rte_unused const char *dev_addr,
 		      __rte_unused int *vfio_dev_fd,
lib/librte_eal/include/rte_eal.h +12 -0
index 2f9ed298d..d9040ce14 100644
@@ -21,6 +21,7 @@
 #include <rte_bus.h>
 
 #include <rte_pci_dev_feature_defs.h>
+#include <rte_uuid.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -438,6 +439,17 @@ int rte_eal_create_uio_dev(void);
  */
 enum rte_intr_mode rte_eal_vfio_intr_mode(void);
 
+
+/**
+ * The user-configured vfio VF token.
+ *
+ * @param vf_token
+ *   vfio VF token configured with the command line is copied
+ *   into this parameter, zero uuid by default.
+ */
+__rte_experimental
+void rte_eal_vfio_vf_token(rte_uuid_t vf_token);
+
 /**
  * A wrap API for syscall gettid.
  *
lib/librte_eal/linux/eal.c +29 -0
index aa72d3650..05cd6e497 100644
@@ -558,6 +558,7 @@ eal_usage(const char *prgname)
 	       "  --"OPT_FILE_PREFIX"       Prefix for hugepage filenames\n"
 	       "  --"OPT_CREATE_UIO_DEV"    Create /dev/uioX (usually done by hotplug)\n"
 	       "  --"OPT_VFIO_INTR"         Interrupt mode for VFIO (legacy|msi|msix)\n"
+	       "  --"OPT_VFIO_VF_TOKEN"     VF token (UUID) shared between SR-IOV PF and VFs\n"
 	       "  --"OPT_LEGACY_MEM"        Legacy memory mode (no dynamic allocation, contiguous segments)\n"
 	       "  --"OPT_SINGLE_FILE_SEGMENTS" Put all hugepage memory in single files\n"
 	       "  --"OPT_MATCH_ALLOCATIONS" Free hugepages exactly as allocated\n"
@@ -649,6 +650,19 @@ eal_parse_vfio_intr(const char *mode)
 	return -1;
 }
 
+static int
+eal_parse_vfio_vf_token(const char *vf_token)
+{
+	rte_uuid_t uuid;
+
+	if (!rte_uuid_parse(vf_token, uuid)) {
+		rte_uuid_copy(internal_config.vfio_vf_token, uuid);
+		return 0;
+	}
+
+	return -1;
+}
+
 /* Parse the arguments for --log-level only */
 static void
 eal_log_level_parse(int argc, char **argv)
@@ -795,6 +809,16 @@ eal_parse_args(int argc, char **argv)
 			}
 			break;
 
+		case OPT_VFIO_VF_TOKEN_NUM:
+			if (eal_parse_vfio_vf_token(optarg) < 0) {
+				RTE_LOG(ERR, EAL, "invalid parameters for --"
+						OPT_VFIO_VF_TOKEN "\n");
+				eal_usage(prgname);
+				ret = -1;
+				goto out;
+			}
+			break;
+
 		case OPT_CREATE_UIO_DEV_NUM:
 			internal_config.create_uio_dev = 1;
 			break;
@@ -1367,6 +1391,11 @@ rte_eal_vfio_intr_mode(void)
 	return internal_config.vfio_intr_mode;
 }
 
+void rte_eal_vfio_vf_token(rte_uuid_t vf_token)
+{
+	rte_uuid_copy(vf_token, internal_config.vfio_vf_token);
+}
+
 int
 rte_eal_check_module(const char *module_name)
 {
lib/librte_eal/linux/eal_vfio.c +19 -0
index d26e1649a..6b8496b52 100644
@@ -712,6 +712,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 	int vfio_container_fd;
 	int vfio_group_fd;
 	int iommu_group_num;
+	rte_uuid_t vf_token;
 	int i, ret;
 
 	/* get group number */
@@ -895,6 +896,23 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 				t->type_id, t->name);
 	}
 
+	rte_eal_vfio_vf_token(vf_token);
+
+	/* get a file descriptor for the device with VF token firstly */
+	if (!rte_uuid_is_null(vf_token)) {
+		char vf_token_str[RTE_UUID_STRLEN];
+		char dev[PATH_MAX];
+
+		rte_uuid_unparse(vf_token, vf_token_str, sizeof(vf_token_str));
+		snprintf(dev, sizeof(dev),
+			 "%s vf_token=%s", dev_addr, vf_token_str);
+
+		*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD,
+				     dev);
+		if (*vfio_dev_fd >= 0)
+			goto dev_get_info;
+	}
+
 	/* get a file descriptor for the device */
 	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev_addr);
 	if (*vfio_dev_fd < 0) {
@@ -909,6 +927,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 		return -1;
 	}
 
+dev_get_info:
 	/* test and setup the device */
 	ret = ioctl(*vfio_dev_fd, VFIO_DEVICE_GET_INFO, device_info);
 	if (ret) {
lib/librte_eal/rte_eal_version.map +1 -0
index 6088e7f6c..5b7dd3e81 100644
@@ -373,6 +373,7 @@ EXPERIMENTAL {
 	__rte_trace_point_register;
 	per_lcore_trace_mem;
 	per_lcore_trace_point_sz;
+	rte_eal_vfio_vf_token;
 	rte_log_can_log;
 	rte_thread_getname;
 	rte_trace_dump;
Signature · 1 line
2.26.2
Acked-by on v12 2/2
#62 lore raw

Re: [dpdk-dev] [PATCH v12 2/2] eal: support for VFIO-PCI VF token

Quoted 8 lines · Haiyue Wang

The kernel module vfio-pci introduces the VF token to enable SR-IOV support since 5.7.

The VF token can be set by a vfio-pci based PF driver and must be known by the vfio-pci based VF driver in order to gain access to the device.

---

<snip>

lib/librte_eal/freebsd/eal.c rte_eal_vfio_intr_mode(void) · line 1008

Quoted 12 lines
lib/librte_eal/freebsd/eal.c +4 -0
index 540b7d38c..00b02855e 100644
@@ -1002,6 +1002,11 @@ rte_eal_vfio_intr_mode(void)
  	return RTE_INTR_MODE_NONE;
  }
 
+void rte_eal_vfio_vf_token(rte_uuid_t vf_token)
+{
+	memset(vf_token, 0, sizeof(rte_uuid_t));
+}

What's the purpose of memset(0) here? Presumably, if the API is not supposed to be supported, the function should have no effect?

Quoted 17 lines
lib/librte_eal/freebsd/eal.c +1 -0
+
  int rte_vfio_setup_device(__rte_unused const char *sysfs_base,
  		      __rte_unused const char *dev_addr,
  		      __rte_unused int *vfio_dev_fd,
lib/librte_eal/include/rte_eal.h +1 -0
index 2f9ed298d..d9040ce14 100644
@@ -21,6 +21,7 @@
  #include <rte_bus.h>
 
  #include <rte_pci_dev_feature_defs.h>
+#include <rte_uuid.h>
 
  #ifdef __cplusplus
  extern "C" {

lib/librte_eal/include/rte_eal.h int rte_eal_create_uio_dev(void); · line 444

Quoted 6 lines
lib/librte_eal/include/rte_eal.h +3 -0
@@ -438,6 +439,17 @@ int rte_eal_create_uio_dev(void);
   */
  enum rte_intr_mode rte_eal_vfio_intr_mode(void);
 
+
+/**
+ * The user-configured vfio VF token.

The description is unclear. Suggested rewording:

Copy the user-configured vfio VF token.

lib/librte_eal/include/rte_eal.h int rte_eal_create_uio_dev(void); · line 451

Quoted 7 lines
lib/librte_eal/include/rte_eal.h +7 -0
+ *
+ * @param vf_token
+ *   vfio VF token configured with the command line is copied
+ *   into this parameter, zero uuid by default.
+ */
+__rte_experimental
+void rte_eal_vfio_vf_token(rte_uuid_t vf_token);

Maybe rte_eal_vfio_get_vf_token()? I imagine this would be a more descriptive name for this API.

Once all of the above is addressed (or explained),

Signature · 2 lines
Thanks,
Anatoly
#63 lore raw

Re: [dpdk-dev] [PATCH v12 2/2] eal: support for VFIO-PCI VF token

Hi Anatoly,

Quoted 27 lines

-----Original Message-----
From: Burakov, Anatoly <[email protected]>
Sent: Wednesday, May 6, 2020 18:27
To: Wang, Haiyue <[email protected]>; [email protected]; [email protected]; [email protected]; [email protected]
Subject: Re: [PATCH v12 2/2] eal: support for VFIO-PCI VF token

Quoted 8 lines · Haiyue Wang

The kernel module vfio-pci introduces the VF token to enable SR-IOV support since 5.7.

The VF token can be set by a vfio-pci based PF driver and must be known by the vfio-pci based VF driver in order to gain access to the device.

---

<snip>

Quoted 8 lines
lib/librte_eal/freebsd/eal.c +0 -0
index 540b7d38c..00b02855e 100644
@@ -1002,6 +1002,11 @@ rte_eal_vfio_intr_mode(void)
  	return RTE_INTR_MODE_NONE;
  }
 
Quoted 8 lines

lib/librte_eal/freebsd/eal.c rte_eal_vfio_intr_mode(void) · line 1008

Quoted 4 lines
lib/librte_eal/freebsd/eal.c +4 -0
+void rte_eal_vfio_vf_token(rte_uuid_t vf_token)
+{
+	memset(vf_token, 0, sizeof(rte_uuid_t));
+}

What's the purpose of memset(0) here? Presumably, if the API is not supposed to be supported, the function should have no effect?

Yes, originally I meant to no parse the input, then return zero UUID. I changed it to the same as Linux:

	rte_uuid_copy(vf_token, internal_config.vfio_vf_token);

Since it is inited to zero.

Quoted 20 lines
Quoted 20 lines
lib/librte_eal/freebsd/eal.c +1 -0
+
  int rte_vfio_setup_device(__rte_unused const char *sysfs_base,
  		      __rte_unused const char *dev_addr,
  		      __rte_unused int *vfio_dev_fd,
lib/librte_eal/include/rte_eal.h +1 -0
index 2f9ed298d..d9040ce14 100644
@@ -21,6 +21,7 @@
  #include <rte_bus.h>
 
  #include <rte_pci_dev_feature_defs.h>
+#include <rte_uuid.h>
 
  #ifdef __cplusplus
  extern "C" {
@@ -438,6 +439,17 @@ int rte_eal_create_uio_dev(void);
   */
  enum rte_intr_mode rte_eal_vfio_intr_mode(void);
 
Quoted 8 lines

lib/librte_eal/include/rte_eal.h int rte_eal_create_uio_dev(void); · line 444

Quoted 3 lines
lib/librte_eal/include/rte_eal.h +3 -0
+
+/**
+ * The user-configured vfio VF token.

The description is unclear. Suggested rewording:

Copy the user-configured vfio VF token.

+1.

Quoted 11 lines

lib/librte_eal/include/rte_eal.h int rte_eal_create_uio_dev(void); · line 451

Quoted 7 lines
lib/librte_eal/include/rte_eal.h +7 -0
+ *
+ * @param vf_token
+ *   vfio VF token configured with the command line is copied
+ *   into this parameter, zero uuid by default.
+ */
+__rte_experimental
+void rte_eal_vfio_vf_token(rte_uuid_t vf_token);

Maybe rte_eal_vfio_get_vf_token()? I imagine this would be a more descriptive name for this API.

Yes, this looks better. Fixed.

Quoted 7 lines

Once all of the above is addressed (or explained),

--
Thanks,
Anatoly

#64 lore raw

Re: [dpdk-dev] [PATCH v12 2/2] eal: support for VFIO-PCI VF token

Quoted 38 lines · Haiyue

Hi Anatoly,

Quoted 27 lines

-----Original Message-----
From: Burakov, Anatoly <[email protected]>
Sent: Wednesday, May 6, 2020 18:27
To: Wang, Haiyue <[email protected]>; [email protected]; [email protected]; [email protected]; [email protected]
Subject: Re: [PATCH v12 2/2] eal: support for VFIO-PCI VF token

Quoted 8 lines · Haiyue Wang

The kernel module vfio-pci introduces the VF token to enable SR-IOV support since 5.7.

The VF token can be set by a vfio-pci based PF driver and must be known by the vfio-pci based VF driver in order to gain access to the device.

---

<snip>

Quoted 8 lines
lib/librte_eal/freebsd/eal.c +0 -0
index 540b7d38c..00b02855e 100644
@@ -1002,6 +1002,11 @@ rte_eal_vfio_intr_mode(void)
   	return RTE_INTR_MODE_NONE;
   }
 
Quoted 8 lines

lib/librte_eal/freebsd/eal.c rte_eal_vfio_intr_mode(void) · line 1008

Quoted 4 lines
lib/librte_eal/freebsd/eal.c +4 -0
+void rte_eal_vfio_vf_token(rte_uuid_t vf_token)
+{
+	memset(vf_token, 0, sizeof(rte_uuid_t));
+}

What's the purpose of memset(0) here? Presumably, if the API is not supposed to be supported, the function should have no effect?

Quoted 4 lines

Yes, originally I meant to no parse the input, then return zero UUID. I changed it to the same as Linux:

	rte_uuid_copy(vf_token, internal_config.vfio_vf_token);

Since it is inited to zero.

Yes but on Linux it has effect because Linux supports it. On FreeBSD there's no VFIO, so the function should IMO have no effect. However, i don't have a strong opinion on this, so i'm fine either way.

Signature · 2 lines
Thanks,
Anatoly
#65 lore raw

Re: [dpdk-dev] [PATCH v12 2/2] eal: support for VFIO-PCI VF token

Quoted 46 lines

-----Original Message-----
From: Burakov, Anatoly <[email protected]>
Sent: Wednesday, May 6, 2020 19:40
To: Wang, Haiyue <[email protected]>; [email protected]; [email protected]; [email protected]; [email protected]
Subject: Re: [PATCH v12 2/2] eal: support for VFIO-PCI VF token

Quoted 38 lines · Haiyue

Hi Anatoly,

Quoted 35 lines

-----Original Message-----
From: Burakov, Anatoly <[email protected]>
Sent: Wednesday, May 6, 2020 18:27
To: Wang, Haiyue <[email protected]>; [email protected]; [email protected]; [email protected]; [email protected]
Subject: Re: [PATCH v12 2/2] eal: support for VFIO-PCI VF token

Quoted 8 lines · Haiyue Wang

The kernel module vfio-pci introduces the VF token to enable SR-IOV support since 5.7.

The VF token can be set by a vfio-pci based PF driver and must be known by the vfio-pci based VF driver in order to gain access to the device.

---

<snip>

lib/librte_eal/freebsd/eal.c rte_eal_vfio_intr_mode(void) · line 1008

Quoted 12 lines
lib/librte_eal/freebsd/eal.c +4 -0
index 540b7d38c..00b02855e 100644
@@ -1002,6 +1002,11 @@ rte_eal_vfio_intr_mode(void)
   	return RTE_INTR_MODE_NONE;
   }
 
+void rte_eal_vfio_vf_token(rte_uuid_t vf_token)
+{
+	memset(vf_token, 0, sizeof(rte_uuid_t));
+}

What's the purpose of memset(0) here? Presumably, if the API is not supposed to be supported, the function should have no effect?

Quoted 9 lines
Quoted 4 lines

Yes, originally I meant to no parse the input, then return zero UUID. I changed it to the same as Linux:

	rte_uuid_copy(vf_token, internal_config.vfio_vf_token);

Since it is inited to zero.

Yes but on Linux it has effect because Linux supports it. On FreeBSD there's no VFIO, so the function should IMO have no effect. However, i don't have a strong opinion on this, so i'm fine either way.

Uh, seems empty function should be good one. I will change it. ;-)

Quoted 3 lines

--
Thanks,
Anatoly

#66 lore raw

v13 cover support for VFIO-PCI VF token interface

In reply to [email protected]

v13: Rename the EAL get VF token function, and leave the freebsd type as empty.

v12: support to vfio devices with VF token and no token.

v11: Use the eal parameter to pass the VF token, then not every PCI
device needs to be specified with this token. Also no ABI issue now.

v10: Use the __rte_internal to mark the internal API changing.

v9: Rewrite the document.

v8: Update the document.

v7: Add the Fixes tag in uuid, the release note and help
document.

v6: Drop the Fixes tag in uuid, since the file has been
moved to another place, not suitable to apply on stable. And this is not a bug, just some kind of enhancement.

v5: 1. Add the VF token parse error handling.
2. Split into two patches for different logic module.
3. Add more comments into the code for explaining the design.
4. Drop the ABI change workaround, this patch set focuses on code review.

v4: 1. Ignore rte_vfio_setup_device ABI check since it is
for Linux driver use.

v3: Fix the Travis build failed:
(1). rte_uuid.h:97:55: error: unknown type name ‘size_t’ (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’

v2: Fix the FreeBSD build error.

v1: Update the commit message.

RFC v2:
Based on Vamsi's RFC v1, and Alex's patch for Qemu
[https://lore.kernel.org/lkml/[email protected]/]:
Use the devarg to pass-down the VF token.

RFC v1: https://patchwork.dpdk.org/patch/66281/ by Vamsi.

Haiyue Wang (2):
eal: add uuid dependent header files explicitly
eal: support for VFIO-PCI VF token

 doc/guides/linux_gsg/linux_drivers.rst        | 35 ++++++++++++++++++-
 doc/guides/linux_gsg/linux_eal_parameters.rst |  4 +++
 doc/guides/rel_notes/release_20_05.rst        |  6 ++++
 lib/librte_eal/common/eal_common_options.c    |  2 ++
 lib/librte_eal/common/eal_internal_cfg.h      |  2 ++
 lib/librte_eal/common/eal_options.h           |  2 ++
 lib/librte_eal/freebsd/eal.c                  |  4 +++
 lib/librte_eal/include/rte_eal.h              | 12 +++++++
 lib/librte_eal/include/rte_uuid.h             |  2 ++
 lib/librte_eal/linux/eal.c                    | 29 +++++++++++++++
 lib/librte_eal/linux/eal_vfio.c               | 19 ++++++++++
 lib/librte_eal/rte_eal_version.map            |  1 +
 12 files changed, 117 insertions(+), 1 deletion(-)
Signature · 1 line
2.26.2
#67 lore raw

PATCH v13 1/2 eal: add uuid dependent header files explicitly

Add the dependent header files explicitly, so that the user just needs to include the 'rte_uuid.h' header file directly to avoid compile error:
(1). rte_uuid.h:97:55: error: unknown type name ‘size_t’ (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’

Fixes 6bc67c497a51 ("eal: add uuid API")
---
 lib/librte_eal/include/rte_uuid.h | 2 ++
 1 file changed, 2 insertions(+)
lib/librte_eal/include/rte_uuid.h +2 -0
index 044afbdfa..8b42e070a 100644
@@ -15,6 +15,8 @@ extern "C" {
 #endif
 
 #include <stdbool.h>
+#include <stddef.h>
+#include <string.h>
 
 /**
  * Struct describing a Universal Unique Identifier
Signature · 1 line
2.26.2
#68 lore raw

PATCH v13 2/2 eal: support for VFIO-PCI VF token

In reply to Haiyue Wang

The kernel module vfio-pci introduces the VF token to enable SR-IOV support since 5.7.

The VF token can be set by a vfio-pci based PF driver and must be known by the vfio-pci based VF driver in order to gain access to the device.

---
 doc/guides/linux_gsg/linux_drivers.rst        | 35 ++++++++++++++++++-
 doc/guides/linux_gsg/linux_eal_parameters.rst |  4 +++
 doc/guides/rel_notes/release_20_05.rst        |  6 ++++
 lib/librte_eal/common/eal_common_options.c    |  2 ++
 lib/librte_eal/common/eal_internal_cfg.h      |  2 ++
 lib/librte_eal/common/eal_options.h           |  2 ++
 lib/librte_eal/freebsd/eal.c                  |  4 +++
 lib/librte_eal/include/rte_eal.h              | 12 +++++++
 lib/librte_eal/linux/eal.c                    | 29 +++++++++++++++
 lib/librte_eal/linux/eal_vfio.c               | 19 ++++++++++
 lib/librte_eal/rte_eal_version.map            |  1 +
 11 files changed, 115 insertions(+), 1 deletion(-)
doc/guides/linux_gsg/linux_drivers.rst +34 -1
index 238f3e900..910397243 100644
@@ -72,11 +72,44 @@ Note that in order to use VFIO, your kernel must support it.
 VFIO kernel modules have been included in the Linux kernel since version 3.6.0 and are usually present by default,
 however please consult your distributions documentation to make sure that is the case.
 
+The ``vfio-pci`` module since Linux version 5.7 supports the creation of virtual
+functions. After the PF is bound to vfio-pci module, the user can create the VFs
+by sysfs interface, and these VFs are bound to vfio-pci module automatically.
+
+When the PF is bound to vfio-pci, it has initial VF token generated by random. For
+security reason, this token is write only, the user can't read it from the kernel
+directly. For accessing the VF, the user needs to start the PF with token parameter
+to setup a VF token (uuid format), then the VF can be accessed with this new known
+VF token.
+
+DPDK will use the EAL parameter ``--vfio-vf-token`` to specify the VF token value to
+PF and its related VFs, this VF token will be shared in all VFIO devices, including
+the different PFs.
+
+.. code-block:: console
+
+    1. Generate the VF token by uuid command
+        14d63f20-8445-11ea-8900-1f9ce7d5650d
+
+    2. sudo modprobe vfio-pci enable_sriov=1
+
+    2. ./usertools/dpdk-devbind.py -b vfio-pci 0000:86:00.0
+
+    3. echo 2 > /sys/bus/pci/devices/0000:86:00.0/sriov_numvfs
+
+    4. Start the PF:
+        ./x86_64-native-linux-gcc/app/testpmd -l 22-25 -n 4 -w 86:00.0 \
+         --vfio-vf-token=14d63f20-8445-11ea-8900-1f9ce7d5650d --file-prefix=pf -- -i
+
+    5. Start the VF:
+        ./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 -w 86:02.0 \
+         --vfio-vf-token=14d63f20-8445-11ea-8900-1f9ce7d5650d --file-prefix=vf0 -- -i
+
 Also, to use VFIO, both kernel and BIOS must support and be configured to use IO virtualization (such as Intel® VT-d).
 
 .. note::
 
-    ``vfio-pci`` module doesn't support the creation of virtual functions.
+    ``vfio-pci`` module doesn't support the creation of virtual functions before Linux version 5.7.
 
 For proper operation of VFIO when running DPDK applications as a non-privileged user, correct permissions should also be set up.
 This can be done by using the DPDK setup script (called dpdk-setup.sh and located in the usertools directory).
doc/guides/linux_gsg/linux_eal_parameters.rst +4 -0
index b2cc60e44..bd3977cb3 100644
@@ -40,6 +40,10 @@ Device-related options
 
     Use specified interrupt mode for devices bound to VFIO kernel driver.
 
+*   ``--vfio-vf-token <uuid>``
+
+    Use specified VF token for devices bound to VFIO kernel driver.
+
 Multiprocessing-related options
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
doc/guides/rel_notes/release_20_05.rst +6 -0
index c287cb48a..b19b03f83 100644
@@ -93,6 +93,12 @@ New Features
   * Added new query: ``rte_flow_get_aged_flows`` to get the aged-out flows
     contexts from the port.
 
+* **Added the support for vfio-pci new VF token interface.**
+
+  Since Linux version 5.7, vfio-pci supports a shared VF token (UUID) to represent
+  the trust between SR-IOV PF and the created VFs. Update the method to gain access
+  to the PF and VFs devices by appending the VF token parameter.
+
 * **Updated Amazon ena driver.**
 
   Updated ena PMD with new features and improvements, including:
lib/librte_eal/common/eal_common_options.c +2 -0
index 418731ca4..dbce919ad 100644
@@ -89,6 +89,7 @@ eal_long_options[] = {
 	{OPT_SYSLOG,            1, NULL, OPT_SYSLOG_NUM           },
 	{OPT_VDEV,              1, NULL, OPT_VDEV_NUM             },
 	{OPT_VFIO_INTR,         1, NULL, OPT_VFIO_INTR_NUM        },
+	{OPT_VFIO_VF_TOKEN,     1, NULL, OPT_VFIO_VF_TOKEN_NUM    },
 	{OPT_VMWARE_TSC_MAP,    0, NULL, OPT_VMWARE_TSC_MAP_NUM   },
 	{OPT_LEGACY_MEM,        0, NULL, OPT_LEGACY_MEM_NUM       },
 	{OPT_SINGLE_FILE_SEGMENTS, 0, NULL, OPT_SINGLE_FILE_SEGMENTS_NUM},
@@ -222,6 +223,7 @@ eal_reset_internal_config(struct internal_config *internal_cfg)
 
 	/* if set to NONE, interrupt mode is determined automatically */
 	internal_cfg->vfio_intr_mode = RTE_INTR_MODE_NONE;
+	memset(internal_cfg->vfio_vf_token, 0, sizeof(rte_uuid_t));
 
 #ifdef RTE_LIBEAL_USE_HPET
 	internal_cfg->no_hpet = 0;
lib/librte_eal/common/eal_internal_cfg.h +2 -0
index a42f34923..28154f0ef 100644
@@ -72,6 +72,8 @@ struct internal_config {
 	volatile int syslog_facility;	  /**< facility passed to openlog() */
 	/** default interrupt mode for VFIO */
 	volatile enum rte_intr_mode vfio_intr_mode;
+	/** the shared VF token for VFIO-PCI bound PF and VFs devices */
+	rte_uuid_t vfio_vf_token;
 	char *hugefile_prefix;      /**< the base filename of hugetlbfs files */
 	char *hugepage_dir;         /**< specific hugetlbfs directory to use */
 	char *user_mbuf_pool_ops_name;
lib/librte_eal/common/eal_options.h +2 -0
index 90ead1b7c..cd5fd01f2 100644
@@ -67,6 +67,8 @@ enum {
 	OPT_VDEV_NUM,
 #define OPT_VFIO_INTR         "vfio-intr"
 	OPT_VFIO_INTR_NUM,
+#define OPT_VFIO_VF_TOKEN     "vfio-vf-token"
+	OPT_VFIO_VF_TOKEN_NUM,
 #define OPT_VMWARE_TSC_MAP    "vmware-tsc-map"
 	OPT_VMWARE_TSC_MAP_NUM,
 #define OPT_LEGACY_MEM    "legacy-mem"
lib/librte_eal/freebsd/eal.c +4 -0
index 540b7d38c..07f17f182 100644
@@ -1002,6 +1002,10 @@ rte_eal_vfio_intr_mode(void)
 	return RTE_INTR_MODE_NONE;
 }
 
+void rte_eal_vfio_get_vf_token(__rte_unused rte_uuid_t vf_token)
+{
+}
+
 int rte_vfio_setup_device(__rte_unused const char *sysfs_base,
 		      __rte_unused const char *dev_addr,
 		      __rte_unused int *vfio_dev_fd,
lib/librte_eal/include/rte_eal.h +12 -0
index 2f9ed298d..744370013 100644
@@ -21,6 +21,7 @@
 #include <rte_bus.h>
 
 #include <rte_pci_dev_feature_defs.h>
+#include <rte_uuid.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -438,6 +439,17 @@ int rte_eal_create_uio_dev(void);
  */
 enum rte_intr_mode rte_eal_vfio_intr_mode(void);
 
+
+/**
+ * Copy the user-configured vfio VF token.
+ *
+ * @param vf_token
+ *   vfio VF token configured with the command line is copied
+ *   into this parameter, zero uuid by default.
+ */
+__rte_experimental
+void rte_eal_vfio_get_vf_token(rte_uuid_t vf_token);
+
 /**
  * A wrap API for syscall gettid.
  *
lib/librte_eal/linux/eal.c +29 -0
index aa72d3650..f394c281c 100644
@@ -558,6 +558,7 @@ eal_usage(const char *prgname)
 	       "  --"OPT_FILE_PREFIX"       Prefix for hugepage filenames\n"
 	       "  --"OPT_CREATE_UIO_DEV"    Create /dev/uioX (usually done by hotplug)\n"
 	       "  --"OPT_VFIO_INTR"         Interrupt mode for VFIO (legacy|msi|msix)\n"
+	       "  --"OPT_VFIO_VF_TOKEN"     VF token (UUID) shared between SR-IOV PF and VFs\n"
 	       "  --"OPT_LEGACY_MEM"        Legacy memory mode (no dynamic allocation, contiguous segments)\n"
 	       "  --"OPT_SINGLE_FILE_SEGMENTS" Put all hugepage memory in single files\n"
 	       "  --"OPT_MATCH_ALLOCATIONS" Free hugepages exactly as allocated\n"
@@ -649,6 +650,19 @@ eal_parse_vfio_intr(const char *mode)
 	return -1;
 }
 
+static int
+eal_parse_vfio_vf_token(const char *vf_token)
+{
+	rte_uuid_t uuid;
+
+	if (!rte_uuid_parse(vf_token, uuid)) {
+		rte_uuid_copy(internal_config.vfio_vf_token, uuid);
+		return 0;
+	}
+
+	return -1;
+}
+
 /* Parse the arguments for --log-level only */
 static void
 eal_log_level_parse(int argc, char **argv)
@@ -795,6 +809,16 @@ eal_parse_args(int argc, char **argv)
 			}
 			break;
 
+		case OPT_VFIO_VF_TOKEN_NUM:
+			if (eal_parse_vfio_vf_token(optarg) < 0) {
+				RTE_LOG(ERR, EAL, "invalid parameters for --"
+						OPT_VFIO_VF_TOKEN "\n");
+				eal_usage(prgname);
+				ret = -1;
+				goto out;
+			}
+			break;
+
 		case OPT_CREATE_UIO_DEV_NUM:
 			internal_config.create_uio_dev = 1;
 			break;
@@ -1367,6 +1391,11 @@ rte_eal_vfio_intr_mode(void)
 	return internal_config.vfio_intr_mode;
 }
 
+void rte_eal_vfio_get_vf_token(rte_uuid_t vf_token)
+{
+	rte_uuid_copy(vf_token, internal_config.vfio_vf_token);
+}
+
 int
 rte_eal_check_module(const char *module_name)
 {
lib/librte_eal/linux/eal_vfio.c +19 -0
index d26e1649a..5b2f6b305 100644
@@ -712,6 +712,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 	int vfio_container_fd;
 	int vfio_group_fd;
 	int iommu_group_num;
+	rte_uuid_t vf_token;
 	int i, ret;
 
 	/* get group number */
@@ -895,6 +896,23 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 				t->type_id, t->name);
 	}
 
+	rte_eal_vfio_get_vf_token(vf_token);
+
+	/* get a file descriptor for the device with VF token firstly */
+	if (!rte_uuid_is_null(vf_token)) {
+		char vf_token_str[RTE_UUID_STRLEN];
+		char dev[PATH_MAX];
+
+		rte_uuid_unparse(vf_token, vf_token_str, sizeof(vf_token_str));
+		snprintf(dev, sizeof(dev),
+			 "%s vf_token=%s", dev_addr, vf_token_str);
+
+		*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD,
+				     dev);
+		if (*vfio_dev_fd >= 0)
+			goto dev_get_info;
+	}
+
 	/* get a file descriptor for the device */
 	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev_addr);
 	if (*vfio_dev_fd < 0) {
@@ -909,6 +927,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 		return -1;
 	}
 
+dev_get_info:
 	/* test and setup the device */
 	ret = ioctl(*vfio_dev_fd, VFIO_DEVICE_GET_INFO, device_info);
 	if (ret) {
lib/librte_eal/rte_eal_version.map +1 -0
index 6088e7f6c..f7043af69 100644
@@ -373,6 +373,7 @@ EXPERIMENTAL {
 	__rte_trace_point_register;
 	per_lcore_trace_mem;
 	per_lcore_trace_point_sz;
+	rte_eal_vfio_get_vf_token;
 	rte_log_can_log;
 	rte_thread_getname;
 	rte_trace_dump;
Signature · 1 line
2.26.2
Acked-by on v13 2/2
#69 lore raw

Re: [dpdk-dev] [PATCH v13 2/2] eal: support for VFIO-PCI VF token

Quoted 8 lines · Haiyue Wang

The kernel module vfio-pci introduces the VF token to enable SR-IOV support since 5.7.

The VF token can be set by a vfio-pci based PF driver and must be known by the vfio-pci based VF driver in order to gain access to the device.

Sorry, lost from my view new versions of the patch series.

Quoted 7 lines
doc/guides/linux_gsg/linux_drivers.rst +0 -0
index 238f3e900..910397243 100644
@@ -72,11 +72,44 @@ Note that in order to use VFIO, your kernel must support it.
 VFIO kernel modules have been included in the Linux kernel since version 3.6.0 and are usually present by default,
 however please consult your distributions documentation to make sure that is the case.

doc/guides/linux_gsg/linux_drivers.rst Note that in order to use VFIO, your kernel must support it. · line 83

Quoted 10 lines
doc/guides/linux_gsg/linux_drivers.rst +9 -0
 
+The ``vfio-pci`` module since Linux version 5.7 supports the creation of virtual
+functions. After the PF is bound to vfio-pci module, the user can create the VFs
+by sysfs interface, and these VFs are bound to vfio-pci module automatically.
+
+When the PF is bound to vfio-pci, it has initial VF token generated by random. For
+security reason, this token is write only, the user can't read it from the kernel
+directly. For accessing the VF, the user needs to start the PF with token parameter
+to setup a VF token (uuid format), then the VF can be accessed with this new known
+VF token.

If token is write-only in kernel sysfs, shouldn't we make it invisible in ps output? I.e. substitute with something like xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx.
It is a bit easier with the new design. Just a thought.

[snip]

#70 lore raw

Re: [dpdk-dev] [PATCH v13 2/2] eal: support for VFIO-PCI VF token

Quoted 30 lines

-----Original Message-----
From: Andrew Rybchenko <[email protected]>
Sent: Thursday, May 7, 2020 00:51
To: Wang, Haiyue <[email protected]>; [email protected]; Burakov, Anatoly <[email protected]>; [email protected]; [email protected]; [email protected]
Subject: Re: [dpdk-dev] [PATCH v13 2/2] eal: support for VFIO-PCI VF token

Quoted 8 lines · Haiyue Wang

The kernel module vfio-pci introduces the VF token to enable SR-IOV support since 5.7.

The VF token can be set by a vfio-pci based PF driver and must be known by the vfio-pci based VF driver in order to gain access to the device.

Sorry, lost from my view new versions of the patch series.

doc/guides/linux_gsg/linux_drivers.rst Note that in order to use VFIO, your kernel must support it. · line 72

Quoted 6 lines
doc/guides/linux_gsg/linux_drivers.rst +0 -0
index 238f3e900..910397243 100644
@@ -72,11 +72,44 @@ Note that in order to use VFIO, your kernel must support it.
 VFIO kernel modules have been included in the Linux kernel since version 3.6.0 and are usually

present by default,

Quoted 2 lines
doc/guides/linux_gsg/linux_drivers.rst +0 -0
 however please consult your distributions documentation to make sure that is the case.
 
Quoted 15 lines

doc/guides/linux_gsg/linux_drivers.rst Note that in order to use VFIO, your kernel must support it. · line 83

Quoted 9 lines
doc/guides/linux_gsg/linux_drivers.rst +9 -0
+The ``vfio-pci`` module since Linux version 5.7 supports the creation of virtual
+functions. After the PF is bound to vfio-pci module, the user can create the VFs
+by sysfs interface, and these VFs are bound to vfio-pci module automatically.
+
+When the PF is bound to vfio-pci, it has initial VF token generated by random. For
+security reason, this token is write only, the user can't read it from the kernel
+directly. For accessing the VF, the user needs to start the PF with token parameter
+to setup a VF token (uuid format), then the VF can be accessed with this new known
+VF token.

If token is write-only in kernel sysfs, shouldn't we make it invisible in ps output? I.e. substitute with something like xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx.
It is a bit easier with the new design. Just a thought.

In fact, no sysfs for VF token, just write-only IOCTL. ;-)

Quoted 1 line

[snip]

#71 lore raw

Re: [dpdk-dev] [PATCH v13 2/2] eal: support for VFIO-PCI VF token

Quoted 46 lines · Haiyue
Quoted 30 lines

-----Original Message-----
From: Andrew Rybchenko <[email protected]>
Sent: Thursday, May 7, 2020 00:51
To: Wang, Haiyue <[email protected]>; [email protected]; Burakov, Anatoly <[email protected]>; [email protected]; [email protected]; [email protected]
Subject: Re: [dpdk-dev] [PATCH v13 2/2] eal: support for VFIO-PCI VF token

Quoted 8 lines · Haiyue Wang

The kernel module vfio-pci introduces the VF token to enable SR-IOV support since 5.7.

The VF token can be set by a vfio-pci based PF driver and must be known by the vfio-pci based VF driver in order to gain access to the device.

Sorry, lost from my view new versions of the patch series.

doc/guides/linux_gsg/linux_drivers.rst Note that in order to use VFIO, your kernel must support it. · line 72

Quoted 6 lines
doc/guides/linux_gsg/linux_drivers.rst +0 -0
index 238f3e900..910397243 100644
@@ -72,11 +72,44 @@ Note that in order to use VFIO, your kernel must support it.
 VFIO kernel modules have been included in the Linux kernel since version 3.6.0 and are usually

present by default,

Quoted 2 lines
doc/guides/linux_gsg/linux_drivers.rst +0 -0
 however please consult your distributions documentation to make sure that is the case.
 
Quoted 15 lines

doc/guides/linux_gsg/linux_drivers.rst Note that in order to use VFIO, your kernel must support it. · line 83

Quoted 9 lines
doc/guides/linux_gsg/linux_drivers.rst +9 -0
+The ``vfio-pci`` module since Linux version 5.7 supports the creation of virtual
+functions. After the PF is bound to vfio-pci module, the user can create the VFs
+by sysfs interface, and these VFs are bound to vfio-pci module automatically.
+
+When the PF is bound to vfio-pci, it has initial VF token generated by random. For
+security reason, this token is write only, the user can't read it from the kernel
+directly. For accessing the VF, the user needs to start the PF with token parameter
+to setup a VF token (uuid format), then the VF can be accessed with this new known
+VF token.

If token is write-only in kernel sysfs, shouldn't we make it invisible in ps output? I.e. substitute with something like xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx.
It is a bit easier with the new design. Just a thought.

Quoted 1 line

In fact, no sysfs for VF token, just write-only IOCTL. ;-)

OK, got it. The question remains anyway. Should it be treated as a secret with at least minimal security precaution?

#72 lore raw

Re: [dpdk-dev] [PATCH v13 2/2] eal: support for VFIO-PCI VF token

+Alex

Quoted 55 lines

-----Original Message-----
From: Andrew Rybchenko <[email protected]>
Sent: Thursday, May 7, 2020 00:59
To: Wang, Haiyue <[email protected]>; [email protected]; Burakov, Anatoly <[email protected]>; [email protected]; [email protected]; [email protected]
Subject: Re: [dpdk-dev] [PATCH v13 2/2] eal: support for VFIO-PCI VF token

Quoted 4 lines · Haiyue
Quoted 4 lines

-----Original Message-----
From: Andrew Rybchenko <[email protected]>
Sent: Thursday, May 7, 2020 00:51
To: Wang, Haiyue <[email protected]>; [email protected]; Burakov, Anatoly

<[email protected]>;

Quoted 42 lines
Quoted 41 lines

[email protected]; [email protected]; [email protected]
Subject: Re: [dpdk-dev] [PATCH v13 2/2] eal: support for VFIO-PCI VF token

Quoted 8 lines · Haiyue Wang

The kernel module vfio-pci introduces the VF token to enable SR-IOV support since 5.7.

The VF token can be set by a vfio-pci based PF driver and must be known by the vfio-pci based VF driver in order to gain access to the device.

Sorry, lost from my view new versions of the patch series.

doc/guides/linux_gsg/linux_drivers.rst Note that in order to use VFIO, your kernel must support it. · line 72

Quoted 6 lines
doc/guides/linux_gsg/linux_drivers.rst +0 -0
index 238f3e900..910397243 100644
@@ -72,11 +72,44 @@ Note that in order to use VFIO, your kernel must support it.
 VFIO kernel modules have been included in the Linux kernel since version 3.6.0 and are usually

present by default,

doc/guides/linux_gsg/linux_drivers.rst Note that in order to use VFIO, your kernel must support it. · line 83

Quoted 11 lines
doc/guides/linux_gsg/linux_drivers.rst +9 -0
 however please consult your distributions documentation to make sure that is the case.
 
+The ``vfio-pci`` module since Linux version 5.7 supports the creation of virtual
+functions. After the PF is bound to vfio-pci module, the user can create the VFs
+by sysfs interface, and these VFs are bound to vfio-pci module automatically.
+
+When the PF is bound to vfio-pci, it has initial VF token generated by random. For
+security reason, this token is write only, the user can't read it from the kernel
+directly. For accessing the VF, the user needs to start the PF with token parameter
+to setup a VF token (uuid format), then the VF can be accessed with this new known
+VF token.

If token is write-only in kernel sysfs, shouldn't we make it invisible in ps output? I.e. substitute with something like xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx.
It is a bit easier with the new design. Just a thought.

Quoted 5 lines
Quoted 1 line

In fact, no sysfs for VF token, just write-only IOCTL. ;-)

OK, got it. The question remains anyway. Should it be treated as a secret with at least minimal security precaution?

Sounds yes, and also it looks like be more friendly for user to check whether this PF/VF have a VF token required or not by cat /sys/...

@Alex may consider this design.

#73 lore raw

v14 cover support for VFIO-PCI VF token interface

In reply to [email protected]

v14: Rebase the patch for 20.08 release note.

v13: Rename the EAL get VF token function, and leave the freebsd type as empty.

v12: support to vfio devices with VF token and no token.

v11: Use the eal parameter to pass the VF token, then not every PCI
device needs to be specified with this token. Also no ABI issue now.

v10: Use the __rte_internal to mark the internal API changing.

v9: Rewrite the document.

v8: Update the document.

v7: Add the Fixes tag in uuid, the release note and help
document.

v6: Drop the Fixes tag in uuid, since the file has been
moved to another place, not suitable to apply on stable. And this is not a bug, just some kind of enhancement.

v5: 1. Add the VF token parse error handling.
2. Split into two patches for different logic module.
3. Add more comments into the code for explaining the design.
4. Drop the ABI change workaround, this patch set focuses on code review.

v4: 1. Ignore rte_vfio_setup_device ABI check since it is
for Linux driver use.

v3: Fix the Travis build failed:
(1). rte_uuid.h:97:55: error: unknown type name ‘size_t’ (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’

v2: Fix the FreeBSD build error.

v1: Update the commit message.

RFC v2:
Based on Vamsi's RFC v1, and Alex's patch for Qemu
[https://lore.kernel.org/lkml/[email protected]/]:
Use the devarg to pass-down the VF token.

RFC v1: https://patchwork.dpdk.org/patch/66281/ by Vamsi.

Haiyue Wang (2):
eal: add uuid dependent header files explicitly
eal: support for VFIO-PCI VF token

 doc/guides/linux_gsg/linux_drivers.rst        | 35 ++++++++++++++++++-
 doc/guides/linux_gsg/linux_eal_parameters.rst |  4 +++
 doc/guides/rel_notes/release_20_08.rst        |  5 +++
 lib/librte_eal/common/eal_common_options.c    |  2 ++
 lib/librte_eal/common/eal_internal_cfg.h      |  2 ++
 lib/librte_eal/common/eal_options.h           |  2 ++
 lib/librte_eal/freebsd/eal.c                  |  4 +++
 lib/librte_eal/include/rte_eal.h              | 12 +++++++
 lib/librte_eal/include/rte_uuid.h             |  2 ++
 lib/librte_eal/linux/eal.c                    | 29 +++++++++++++++
 lib/librte_eal/linux/eal_vfio.c               | 19 ++++++++++
 lib/librte_eal/rte_eal_version.map            |  1 +
 12 files changed, 116 insertions(+), 1 deletion(-)
Signature · 1 line
2.26.2
#74 lore raw

PATCH v14 1/2 eal: add uuid dependent header files explicitly

Add the dependent header files explicitly, so that the user just needs to include the 'rte_uuid.h' header file directly to avoid compile error:
(1). rte_uuid.h:97:55: error: unknown type name ‘size_t’ (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’

Fixes 6bc67c497a51 ("eal: add uuid API")
---
 lib/librte_eal/include/rte_uuid.h | 2 ++
 1 file changed, 2 insertions(+)
lib/librte_eal/include/rte_uuid.h +2 -0
index 044afbdfa..8b42e070a 100644
@@ -15,6 +15,8 @@ extern "C" {
 #endif
 
 #include <stdbool.h>
+#include <stddef.h>
+#include <string.h>
 
 /**
  * Struct describing a Universal Unique Identifier
Signature · 1 line
2.26.2
#75 lore raw

PATCH v14 2/2 eal: support for VFIO-PCI VF token

In reply to Haiyue Wang

The kernel module vfio-pci introduces the VF token to enable SR-IOV support since 5.7.

The VF token can be set by a vfio-pci based PF driver and must be known by the vfio-pci based VF driver in order to gain access to the device.

---
 doc/guides/linux_gsg/linux_drivers.rst        | 35 ++++++++++++++++++-
 doc/guides/linux_gsg/linux_eal_parameters.rst |  4 +++
 doc/guides/rel_notes/release_20_08.rst        |  5 +++
 lib/librte_eal/common/eal_common_options.c    |  2 ++
 lib/librte_eal/common/eal_internal_cfg.h      |  2 ++
 lib/librte_eal/common/eal_options.h           |  2 ++
 lib/librte_eal/freebsd/eal.c                  |  4 +++
 lib/librte_eal/include/rte_eal.h              | 12 +++++++
 lib/librte_eal/linux/eal.c                    | 29 +++++++++++++++
 lib/librte_eal/linux/eal_vfio.c               | 19 ++++++++++
 lib/librte_eal/rte_eal_version.map            |  1 +
 11 files changed, 114 insertions(+), 1 deletion(-)
doc/guides/linux_gsg/linux_drivers.rst +34 -1
index d40b495c1..02fba7d1e 100644
@@ -78,11 +78,44 @@ Note that in order to use VFIO, your kernel must support it.
 VFIO kernel modules have been included in the Linux kernel since version 3.6.0 and are usually present by default,
 however please consult your distributions documentation to make sure that is the case.
 
+The ``vfio-pci`` module since Linux version 5.7 supports the creation of virtual
+functions. After the PF is bound to vfio-pci module, the user can create the VFs
+by sysfs interface, and these VFs are bound to vfio-pci module automatically.
+
+When the PF is bound to vfio-pci, it has initial VF token generated by random. For
+security reason, this token is write only, the user can't read it from the kernel
+directly. For accessing the VF, the user needs to start the PF with token parameter
+to setup a VF token (uuid format), then the VF can be accessed with this new known
+VF token.
+
+DPDK will use the EAL parameter ``--vfio-vf-token`` to specify the VF token value to
+PF and its related VFs, this VF token will be shared in all VFIO devices, including
+the different PFs.
+
+.. code-block:: console
+
+    1. Generate the VF token by uuid command
+        14d63f20-8445-11ea-8900-1f9ce7d5650d
+
+    2. sudo modprobe vfio-pci enable_sriov=1
+
+    2. ./usertools/dpdk-devbind.py -b vfio-pci 0000:86:00.0
+
+    3. echo 2 > /sys/bus/pci/devices/0000:86:00.0/sriov_numvfs
+
+    4. Start the PF:
+        ./x86_64-native-linux-gcc/app/testpmd -l 22-25 -n 4 -w 86:00.0 \
+         --vfio-vf-token=14d63f20-8445-11ea-8900-1f9ce7d5650d --file-prefix=pf -- -i
+
+    5. Start the VF:
+        ./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 -w 86:02.0 \
+         --vfio-vf-token=14d63f20-8445-11ea-8900-1f9ce7d5650d --file-prefix=vf0 -- -i
+
 Also, to use VFIO, both kernel and BIOS must support and be configured to use IO virtualization (such as Intel® VT-d).
 
 .. note::
 
-    ``vfio-pci`` module doesn't support the creation of virtual functions.
+    ``vfio-pci`` module doesn't support the creation of virtual functions before Linux version 5.7.
 
 For proper operation of VFIO when running DPDK applications as a non-privileged user, correct permissions should also be set up.
 This can be done by using the DPDK setup script (called dpdk-setup.sh and located in the usertools directory).
doc/guides/linux_gsg/linux_eal_parameters.rst +4 -0
index b2cc60e44..bd3977cb3 100644
@@ -40,6 +40,10 @@ Device-related options
 
     Use specified interrupt mode for devices bound to VFIO kernel driver.
 
+*   ``--vfio-vf-token <uuid>``
+
+    Use specified VF token for devices bound to VFIO kernel driver.
+
 Multiprocessing-related options
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
doc/guides/rel_notes/release_20_08.rst +5 -0
index 39064afbe..dfce08b89 100644
@@ -56,6 +56,11 @@ New Features
      Also, make sure to start the actual text at the margin.
      =========================================================
 
+* **Added the support for vfio-pci new VF token interface.**
+
+  Since Linux version 5.7, vfio-pci supports a shared VF token (UUID) to represent
+  the trust between SR-IOV PF and the created VFs. Update the method to gain access
+  to the PF and VFs devices by appending the VF token parameter.
 
 Removed Items
 -------------
lib/librte_eal/common/eal_common_options.c +2 -0
index 8f2cbd1c6..e7b643a99 100644
@@ -92,6 +92,7 @@ eal_long_options[] = {
 	{OPT_SYSLOG,            1, NULL, OPT_SYSLOG_NUM           },
 	{OPT_VDEV,              1, NULL, OPT_VDEV_NUM             },
 	{OPT_VFIO_INTR,         1, NULL, OPT_VFIO_INTR_NUM        },
+	{OPT_VFIO_VF_TOKEN,     1, NULL, OPT_VFIO_VF_TOKEN_NUM    },
 	{OPT_VMWARE_TSC_MAP,    0, NULL, OPT_VMWARE_TSC_MAP_NUM   },
 	{OPT_LEGACY_MEM,        0, NULL, OPT_LEGACY_MEM_NUM       },
 	{OPT_SINGLE_FILE_SEGMENTS, 0, NULL, OPT_SINGLE_FILE_SEGMENTS_NUM},
@@ -297,6 +298,7 @@ eal_reset_internal_config(struct internal_config *internal_cfg)
 
 	/* if set to NONE, interrupt mode is determined automatically */
 	internal_cfg->vfio_intr_mode = RTE_INTR_MODE_NONE;
+	memset(internal_cfg->vfio_vf_token, 0, sizeof(rte_uuid_t));
 
 #ifdef RTE_LIBEAL_USE_HPET
 	internal_cfg->no_hpet = 0;
lib/librte_eal/common/eal_internal_cfg.h +2 -0
index c650bc081..43095b7a2 100644
@@ -72,6 +72,8 @@ struct internal_config {
 	volatile int syslog_facility;	  /**< facility passed to openlog() */
 	/** default interrupt mode for VFIO */
 	volatile enum rte_intr_mode vfio_intr_mode;
+	/** the shared VF token for VFIO-PCI bound PF and VFs devices */
+	rte_uuid_t vfio_vf_token;
 	char *hugefile_prefix;      /**< the base filename of hugetlbfs files */
 	char *hugepage_dir;         /**< specific hugetlbfs directory to use */
 	char *user_mbuf_pool_ops_name;
lib/librte_eal/common/eal_options.h +2 -0
index 18e6da9ab..89769d48b 100644
@@ -69,6 +69,8 @@ enum {
 	OPT_VDEV_NUM,
 #define OPT_VFIO_INTR         "vfio-intr"
 	OPT_VFIO_INTR_NUM,
+#define OPT_VFIO_VF_TOKEN     "vfio-vf-token"
+	OPT_VFIO_VF_TOKEN_NUM,
 #define OPT_VMWARE_TSC_MAP    "vmware-tsc-map"
 	OPT_VMWARE_TSC_MAP_NUM,
 #define OPT_LEGACY_MEM    "legacy-mem"
lib/librte_eal/freebsd/eal.c +4 -0
index c41f265fa..4ff5028d8 100644
@@ -1005,6 +1005,10 @@ rte_eal_vfio_intr_mode(void)
 	return RTE_INTR_MODE_NONE;
 }
 
+void rte_eal_vfio_get_vf_token(__rte_unused rte_uuid_t vf_token)
+{
+}
+
 int rte_vfio_setup_device(__rte_unused const char *sysfs_base,
 		      __rte_unused const char *dev_addr,
 		      __rte_unused int *vfio_dev_fd,
lib/librte_eal/include/rte_eal.h +12 -0
index 2f9ed298d..744370013 100644
@@ -21,6 +21,7 @@
 #include <rte_bus.h>
 
 #include <rte_pci_dev_feature_defs.h>
+#include <rte_uuid.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -438,6 +439,17 @@ int rte_eal_create_uio_dev(void);
  */
 enum rte_intr_mode rte_eal_vfio_intr_mode(void);
 
+
+/**
+ * Copy the user-configured vfio VF token.
+ *
+ * @param vf_token
+ *   vfio VF token configured with the command line is copied
+ *   into this parameter, zero uuid by default.
+ */
+__rte_experimental
+void rte_eal_vfio_get_vf_token(rte_uuid_t vf_token);
+
 /**
  * A wrap API for syscall gettid.
  *
lib/librte_eal/linux/eal.c +29 -0
index f162124a3..00f3fdced 100644
@@ -558,6 +558,7 @@ eal_usage(const char *prgname)
 	       "  --"OPT_FILE_PREFIX"       Prefix for hugepage filenames\n"
 	       "  --"OPT_CREATE_UIO_DEV"    Create /dev/uioX (usually done by hotplug)\n"
 	       "  --"OPT_VFIO_INTR"         Interrupt mode for VFIO (legacy|msi|msix)\n"
+	       "  --"OPT_VFIO_VF_TOKEN"     VF token (UUID) shared between SR-IOV PF and VFs\n"
 	       "  --"OPT_LEGACY_MEM"        Legacy memory mode (no dynamic allocation, contiguous segments)\n"
 	       "  --"OPT_SINGLE_FILE_SEGMENTS" Put all hugepage memory in single files\n"
 	       "  --"OPT_MATCH_ALLOCATIONS" Free hugepages exactly as allocated\n"
@@ -649,6 +650,19 @@ eal_parse_vfio_intr(const char *mode)
 	return -1;
 }
 
+static int
+eal_parse_vfio_vf_token(const char *vf_token)
+{
+	rte_uuid_t uuid;
+
+	if (!rte_uuid_parse(vf_token, uuid)) {
+		rte_uuid_copy(internal_config.vfio_vf_token, uuid);
+		return 0;
+	}
+
+	return -1;
+}
+
 /* Parse the arguments for --log-level only */
 static void
 eal_log_level_parse(int argc, char **argv)
@@ -787,6 +801,16 @@ eal_parse_args(int argc, char **argv)
 			}
 			break;
 
+		case OPT_VFIO_VF_TOKEN_NUM:
+			if (eal_parse_vfio_vf_token(optarg) < 0) {
+				RTE_LOG(ERR, EAL, "invalid parameters for --"
+						OPT_VFIO_VF_TOKEN "\n");
+				eal_usage(prgname);
+				ret = -1;
+				goto out;
+			}
+			break;
+
 		case OPT_CREATE_UIO_DEV_NUM:
 			internal_config.create_uio_dev = 1;
 			break;
@@ -1370,6 +1394,11 @@ rte_eal_vfio_intr_mode(void)
 	return internal_config.vfio_intr_mode;
 }
 
+void rte_eal_vfio_get_vf_token(rte_uuid_t vf_token)
+{
+	rte_uuid_copy(vf_token, internal_config.vfio_vf_token);
+}
+
 int
 rte_eal_check_module(const char *module_name)
 {
lib/librte_eal/linux/eal_vfio.c +19 -0
index d26e1649a..5b2f6b305 100644
@@ -712,6 +712,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 	int vfio_container_fd;
 	int vfio_group_fd;
 	int iommu_group_num;
+	rte_uuid_t vf_token;
 	int i, ret;
 
 	/* get group number */
@@ -895,6 +896,23 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 				t->type_id, t->name);
 	}
 
+	rte_eal_vfio_get_vf_token(vf_token);
+
+	/* get a file descriptor for the device with VF token firstly */
+	if (!rte_uuid_is_null(vf_token)) {
+		char vf_token_str[RTE_UUID_STRLEN];
+		char dev[PATH_MAX];
+
+		rte_uuid_unparse(vf_token, vf_token_str, sizeof(vf_token_str));
+		snprintf(dev, sizeof(dev),
+			 "%s vf_token=%s", dev_addr, vf_token_str);
+
+		*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD,
+				     dev);
+		if (*vfio_dev_fd >= 0)
+			goto dev_get_info;
+	}
+
 	/* get a file descriptor for the device */
 	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev_addr);
 	if (*vfio_dev_fd < 0) {
@@ -909,6 +927,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 		return -1;
 	}
 
+dev_get_info:
 	/* test and setup the device */
 	ret = ioctl(*vfio_dev_fd, VFIO_DEVICE_GET_INFO, device_info);
 	if (ret) {
lib/librte_eal/rte_eal_version.map +1 -0
index d8038749a..93a4d30f8 100644
@@ -372,6 +372,7 @@ EXPERIMENTAL {
 	__rte_trace_point_register;
 	per_lcore_trace_mem;
 	per_lcore_trace_point_sz;
+	rte_eal_vfio_get_vf_token;
 	rte_log_can_log;
 	rte_thread_getname;
 	rte_trace_dump;
Signature · 1 line
2.26.2
#76 lore raw

Re: [dpdk-dev] [PATCH v14 2/2] eal: support for VFIO-PCI VF token

Quoted 10 lines · Haiyue Wang

The kernel module vfio-pci introduces the VF token to enable SR-IOV support since 5.7.

The VF token can be set by a vfio-pci based PF driver and must be known by the vfio-pci based VF driver in order to gain access to the device.

---

[snip]

lib/librte_eal/include/rte_eal.h int rte_eal_create_uio_dev(void); · line 452

Quoted 11 lines
lib/librte_eal/include/rte_eal.h +11 -0
+
+/**
+ * Copy the user-configured vfio VF token.
+ *
+ * @param vf_token
+ *   vfio VF token configured with the command line is copied
+ *   into this parameter, zero uuid by default.
+ */
+__rte_experimental
+void rte_eal_vfio_get_vf_token(rte_uuid_t vf_token);
+

Minor nit, missing the EXPERIMENTAL warning int the comment.

* @warning
* @b EXPERIMENTAL: this API may change without prior notice *

#77 lore raw

Re: [dpdk-dev] [PATCH v14 2/2] eal: support for VFIO-PCI VF token

Quoted 22 lines

-----Original Message-----
From: Ye, Xiaolong <[email protected]>
Sent: Friday, May 29, 2020 09:26
To: Wang, Haiyue <[email protected]>
Cc: [email protected]; Burakov, Anatoly <[email protected]>; [email protected]; [email protected]; [email protected]; [email protected]
Subject: Re: [dpdk-dev] [PATCH v14 2/2] eal: support for VFIO-PCI VF token

Quoted 10 lines · Haiyue Wang

The kernel module vfio-pci introduces the VF token to enable SR-IOV support since 5.7.

The VF token can be set by a vfio-pci based PF driver and must be known by the vfio-pci based VF driver in order to gain access to the device.

---

[snip]

Quoted 14 lines

lib/librte_eal/include/rte_eal.h int rte_eal_create_uio_dev(void); · line 452

Quoted 11 lines
lib/librte_eal/include/rte_eal.h +11 -0
+
+/**
+ * Copy the user-configured vfio VF token.
+ *
+ * @param vf_token
+ *   vfio VF token configured with the command line is copied
+ *   into this parameter, zero uuid by default.
+ */
+__rte_experimental
+void rte_eal_vfio_get_vf_token(rte_uuid_t vf_token);
+

Minor nit, missing the EXPERIMENTAL warning int the comment.

Yes, missed, it is good for kindly reminding. ;-)

Quoted 3 lines

* @warning
* @b EXPERIMENTAL: this API may change without prior notice *

#78 lore raw

v15 cover support for VFIO-PCI VF token interface

In reply to [email protected]

v15: Add the missed EXPERIMENTAL warning for API doxgen.

v14: Rebase the patch for 20.08 release note.

v13: Rename the EAL get VF token function, and leave the freebsd type as empty.

v12: support to vfio devices with VF token and no token.

v11: Use the eal parameter to pass the VF token, then not every PCI
device needs to be specified with this token. Also no ABI issue now.

v10: Use the __rte_internal to mark the internal API changing.

v9: Rewrite the document.

v8: Update the document.

v7: Add the Fixes tag in uuid, the release note and help
document.

v6: Drop the Fixes tag in uuid, since the file has been
moved to another place, not suitable to apply on stable. And this is not a bug, just some kind of enhancement.

v5: 1. Add the VF token parse error handling.
2. Split into two patches for different logic module.
3. Add more comments into the code for explaining the design.
4. Drop the ABI change workaround, this patch set focuses on code review.

v4: 1. Ignore rte_vfio_setup_device ABI check since it is
for Linux driver use.

v3: Fix the Travis build failed:
(1). rte_uuid.h:97:55: error: unknown type name ‘size_t’ (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’

v2: Fix the FreeBSD build error.

v1: Update the commit message.

RFC v2:
Based on Vamsi's RFC v1, and Alex's patch for Qemu
[https://lore.kernel.org/lkml/[email protected]/]:
Use the devarg to pass-down the VF token.

RFC v1: https://patchwork.dpdk.org/patch/66281/ by Vamsi.

Haiyue Wang (2):
eal: add uuid dependent header files explicitly
eal: support for VFIO-PCI VF token

 doc/guides/linux_gsg/linux_drivers.rst        | 35 ++++++++++++++++++-
 doc/guides/linux_gsg/linux_eal_parameters.rst |  4 +++
 doc/guides/rel_notes/release_20_08.rst        |  5 +++
 lib/librte_eal/common/eal_common_options.c    |  2 ++
 lib/librte_eal/common/eal_internal_cfg.h      |  2 ++
 lib/librte_eal/common/eal_options.h           |  2 ++
 lib/librte_eal/freebsd/eal.c                  |  4 +++
 lib/librte_eal/include/rte_eal.h              | 15 ++++++++
 lib/librte_eal/include/rte_uuid.h             |  2 ++
 lib/librte_eal/linux/eal.c                    | 29 +++++++++++++++
 lib/librte_eal/linux/eal_vfio.c               | 19 ++++++++++
 lib/librte_eal/rte_eal_version.map            |  1 +
 12 files changed, 119 insertions(+), 1 deletion(-)
Signature · 1 line
2.26.2
#79 lore raw

PATCH v15 1/2 eal: add uuid dependent header files explicitly

Add the dependent header files explicitly, so that the user just needs to include the 'rte_uuid.h' header file directly to avoid compile error:
(1). rte_uuid.h:97:55: error: unknown type name ‘size_t’ (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’

Fixes 6bc67c497a51 ("eal: add uuid API")
---
 lib/librte_eal/include/rte_uuid.h | 2 ++
 1 file changed, 2 insertions(+)
lib/librte_eal/include/rte_uuid.h +2 -0
index 044afbdfa..8b42e070a 100644
@@ -15,6 +15,8 @@ extern "C" {
 #endif
 
 #include <stdbool.h>
+#include <stddef.h>
+#include <string.h>
 
 /**
  * Struct describing a Universal Unique Identifier
Signature · 1 line
2.26.2
#80 lore raw

PATCH v15 2/2 eal: support for VFIO-PCI VF token

In reply to Haiyue Wang

The kernel module vfio-pci introduces the VF token to enable SR-IOV support since 5.7.

The VF token can be set by a vfio-pci based PF driver and must be known by the vfio-pci based VF driver in order to gain access to the device.

---
 doc/guides/linux_gsg/linux_drivers.rst        | 35 ++++++++++++++++++-
 doc/guides/linux_gsg/linux_eal_parameters.rst |  4 +++
 doc/guides/rel_notes/release_20_08.rst        |  5 +++
 lib/librte_eal/common/eal_common_options.c    |  2 ++
 lib/librte_eal/common/eal_internal_cfg.h      |  2 ++
 lib/librte_eal/common/eal_options.h           |  2 ++
 lib/librte_eal/freebsd/eal.c                  |  4 +++
 lib/librte_eal/include/rte_eal.h              | 15 ++++++++
 lib/librte_eal/linux/eal.c                    | 29 +++++++++++++++
 lib/librte_eal/linux/eal_vfio.c               | 19 ++++++++++
 lib/librte_eal/rte_eal_version.map            |  1 +
 11 files changed, 117 insertions(+), 1 deletion(-)
doc/guides/linux_gsg/linux_drivers.rst +34 -1
index d40b495c1..02fba7d1e 100644
@@ -78,11 +78,44 @@ Note that in order to use VFIO, your kernel must support it.
 VFIO kernel modules have been included in the Linux kernel since version 3.6.0 and are usually present by default,
 however please consult your distributions documentation to make sure that is the case.
 
+The ``vfio-pci`` module since Linux version 5.7 supports the creation of virtual
+functions. After the PF is bound to vfio-pci module, the user can create the VFs
+by sysfs interface, and these VFs are bound to vfio-pci module automatically.
+
+When the PF is bound to vfio-pci, it has initial VF token generated by random. For
+security reason, this token is write only, the user can't read it from the kernel
+directly. For accessing the VF, the user needs to start the PF with token parameter
+to setup a VF token (uuid format), then the VF can be accessed with this new known
+VF token.
+
+DPDK will use the EAL parameter ``--vfio-vf-token`` to specify the VF token value to
+PF and its related VFs, this VF token will be shared in all VFIO devices, including
+the different PFs.
+
+.. code-block:: console
+
+    1. Generate the VF token by uuid command
+        14d63f20-8445-11ea-8900-1f9ce7d5650d
+
+    2. sudo modprobe vfio-pci enable_sriov=1
+
+    2. ./usertools/dpdk-devbind.py -b vfio-pci 0000:86:00.0
+
+    3. echo 2 > /sys/bus/pci/devices/0000:86:00.0/sriov_numvfs
+
+    4. Start the PF:
+        ./x86_64-native-linux-gcc/app/testpmd -l 22-25 -n 4 -w 86:00.0 \
+         --vfio-vf-token=14d63f20-8445-11ea-8900-1f9ce7d5650d --file-prefix=pf -- -i
+
+    5. Start the VF:
+        ./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 -w 86:02.0 \
+         --vfio-vf-token=14d63f20-8445-11ea-8900-1f9ce7d5650d --file-prefix=vf0 -- -i
+
 Also, to use VFIO, both kernel and BIOS must support and be configured to use IO virtualization (such as Intel® VT-d).
 
 .. note::
 
-    ``vfio-pci`` module doesn't support the creation of virtual functions.
+    ``vfio-pci`` module doesn't support the creation of virtual functions before Linux version 5.7.
 
 For proper operation of VFIO when running DPDK applications as a non-privileged user, correct permissions should also be set up.
 This can be done by using the DPDK setup script (called dpdk-setup.sh and located in the usertools directory).
doc/guides/linux_gsg/linux_eal_parameters.rst +4 -0
index b2cc60e44..bd3977cb3 100644
@@ -40,6 +40,10 @@ Device-related options
 
     Use specified interrupt mode for devices bound to VFIO kernel driver.
 
+*   ``--vfio-vf-token <uuid>``
+
+    Use specified VF token for devices bound to VFIO kernel driver.
+
 Multiprocessing-related options
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
doc/guides/rel_notes/release_20_08.rst +5 -0
index 39064afbe..dfce08b89 100644
@@ -56,6 +56,11 @@ New Features
      Also, make sure to start the actual text at the margin.
      =========================================================
 
+* **Added the support for vfio-pci new VF token interface.**
+
+  Since Linux version 5.7, vfio-pci supports a shared VF token (UUID) to represent
+  the trust between SR-IOV PF and the created VFs. Update the method to gain access
+  to the PF and VFs devices by appending the VF token parameter.
 
 Removed Items
 -------------
lib/librte_eal/common/eal_common_options.c +2 -0
index 8f2cbd1c6..e7b643a99 100644
@@ -92,6 +92,7 @@ eal_long_options[] = {
 	{OPT_SYSLOG,            1, NULL, OPT_SYSLOG_NUM           },
 	{OPT_VDEV,              1, NULL, OPT_VDEV_NUM             },
 	{OPT_VFIO_INTR,         1, NULL, OPT_VFIO_INTR_NUM        },
+	{OPT_VFIO_VF_TOKEN,     1, NULL, OPT_VFIO_VF_TOKEN_NUM    },
 	{OPT_VMWARE_TSC_MAP,    0, NULL, OPT_VMWARE_TSC_MAP_NUM   },
 	{OPT_LEGACY_MEM,        0, NULL, OPT_LEGACY_MEM_NUM       },
 	{OPT_SINGLE_FILE_SEGMENTS, 0, NULL, OPT_SINGLE_FILE_SEGMENTS_NUM},
@@ -297,6 +298,7 @@ eal_reset_internal_config(struct internal_config *internal_cfg)
 
 	/* if set to NONE, interrupt mode is determined automatically */
 	internal_cfg->vfio_intr_mode = RTE_INTR_MODE_NONE;
+	memset(internal_cfg->vfio_vf_token, 0, sizeof(rte_uuid_t));
 
 #ifdef RTE_LIBEAL_USE_HPET
 	internal_cfg->no_hpet = 0;
lib/librte_eal/common/eal_internal_cfg.h +2 -0
index c650bc081..43095b7a2 100644
@@ -72,6 +72,8 @@ struct internal_config {
 	volatile int syslog_facility;	  /**< facility passed to openlog() */
 	/** default interrupt mode for VFIO */
 	volatile enum rte_intr_mode vfio_intr_mode;
+	/** the shared VF token for VFIO-PCI bound PF and VFs devices */
+	rte_uuid_t vfio_vf_token;
 	char *hugefile_prefix;      /**< the base filename of hugetlbfs files */
 	char *hugepage_dir;         /**< specific hugetlbfs directory to use */
 	char *user_mbuf_pool_ops_name;
lib/librte_eal/common/eal_options.h +2 -0
index 18e6da9ab..89769d48b 100644
@@ -69,6 +69,8 @@ enum {
 	OPT_VDEV_NUM,
 #define OPT_VFIO_INTR         "vfio-intr"
 	OPT_VFIO_INTR_NUM,
+#define OPT_VFIO_VF_TOKEN     "vfio-vf-token"
+	OPT_VFIO_VF_TOKEN_NUM,
 #define OPT_VMWARE_TSC_MAP    "vmware-tsc-map"
 	OPT_VMWARE_TSC_MAP_NUM,
 #define OPT_LEGACY_MEM    "legacy-mem"
lib/librte_eal/freebsd/eal.c +4 -0
index c41f265fa..4ff5028d8 100644
@@ -1005,6 +1005,10 @@ rte_eal_vfio_intr_mode(void)
 	return RTE_INTR_MODE_NONE;
 }
 
+void rte_eal_vfio_get_vf_token(__rte_unused rte_uuid_t vf_token)
+{
+}
+
 int rte_vfio_setup_device(__rte_unused const char *sysfs_base,
 		      __rte_unused const char *dev_addr,
 		      __rte_unused int *vfio_dev_fd,
lib/librte_eal/include/rte_eal.h +15 -0
index 2f9ed298d..2f47347dc 100644
@@ -21,6 +21,7 @@
 #include <rte_bus.h>
 
 #include <rte_pci_dev_feature_defs.h>
+#include <rte_uuid.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -438,6 +439,20 @@ int rte_eal_create_uio_dev(void);
  */
 enum rte_intr_mode rte_eal_vfio_intr_mode(void);
 
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Copy the user-configured vfio VF token.
+ *
+ * @param vf_token
+ *   vfio VF token configured with the command line is copied
+ *   into this parameter, zero uuid by default.
+ */
+__rte_experimental
+void rte_eal_vfio_get_vf_token(rte_uuid_t vf_token);
+
 /**
  * A wrap API for syscall gettid.
  *
lib/librte_eal/linux/eal.c +29 -0
index f162124a3..00f3fdced 100644
@@ -558,6 +558,7 @@ eal_usage(const char *prgname)
 	       "  --"OPT_FILE_PREFIX"       Prefix for hugepage filenames\n"
 	       "  --"OPT_CREATE_UIO_DEV"    Create /dev/uioX (usually done by hotplug)\n"
 	       "  --"OPT_VFIO_INTR"         Interrupt mode for VFIO (legacy|msi|msix)\n"
+	       "  --"OPT_VFIO_VF_TOKEN"     VF token (UUID) shared between SR-IOV PF and VFs\n"
 	       "  --"OPT_LEGACY_MEM"        Legacy memory mode (no dynamic allocation, contiguous segments)\n"
 	       "  --"OPT_SINGLE_FILE_SEGMENTS" Put all hugepage memory in single files\n"
 	       "  --"OPT_MATCH_ALLOCATIONS" Free hugepages exactly as allocated\n"
@@ -649,6 +650,19 @@ eal_parse_vfio_intr(const char *mode)
 	return -1;
 }
 
+static int
+eal_parse_vfio_vf_token(const char *vf_token)
+{
+	rte_uuid_t uuid;
+
+	if (!rte_uuid_parse(vf_token, uuid)) {
+		rte_uuid_copy(internal_config.vfio_vf_token, uuid);
+		return 0;
+	}
+
+	return -1;
+}
+
 /* Parse the arguments for --log-level only */
 static void
 eal_log_level_parse(int argc, char **argv)
@@ -787,6 +801,16 @@ eal_parse_args(int argc, char **argv)
 			}
 			break;
 
+		case OPT_VFIO_VF_TOKEN_NUM:
+			if (eal_parse_vfio_vf_token(optarg) < 0) {
+				RTE_LOG(ERR, EAL, "invalid parameters for --"
+						OPT_VFIO_VF_TOKEN "\n");
+				eal_usage(prgname);
+				ret = -1;
+				goto out;
+			}
+			break;
+
 		case OPT_CREATE_UIO_DEV_NUM:
 			internal_config.create_uio_dev = 1;
 			break;
@@ -1370,6 +1394,11 @@ rte_eal_vfio_intr_mode(void)
 	return internal_config.vfio_intr_mode;
 }
 
+void rte_eal_vfio_get_vf_token(rte_uuid_t vf_token)
+{
+	rte_uuid_copy(vf_token, internal_config.vfio_vf_token);
+}
+
 int
 rte_eal_check_module(const char *module_name)
 {
lib/librte_eal/linux/eal_vfio.c +19 -0
index d26e1649a..5b2f6b305 100644
@@ -712,6 +712,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 	int vfio_container_fd;
 	int vfio_group_fd;
 	int iommu_group_num;
+	rte_uuid_t vf_token;
 	int i, ret;
 
 	/* get group number */
@@ -895,6 +896,23 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 				t->type_id, t->name);
 	}
 
+	rte_eal_vfio_get_vf_token(vf_token);
+
+	/* get a file descriptor for the device with VF token firstly */
+	if (!rte_uuid_is_null(vf_token)) {
+		char vf_token_str[RTE_UUID_STRLEN];
+		char dev[PATH_MAX];
+
+		rte_uuid_unparse(vf_token, vf_token_str, sizeof(vf_token_str));
+		snprintf(dev, sizeof(dev),
+			 "%s vf_token=%s", dev_addr, vf_token_str);
+
+		*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD,
+				     dev);
+		if (*vfio_dev_fd >= 0)
+			goto dev_get_info;
+	}
+
 	/* get a file descriptor for the device */
 	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev_addr);
 	if (*vfio_dev_fd < 0) {
@@ -909,6 +927,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 		return -1;
 	}
 
+dev_get_info:
 	/* test and setup the device */
 	ret = ioctl(*vfio_dev_fd, VFIO_DEVICE_GET_INFO, device_info);
 	if (ret) {
lib/librte_eal/rte_eal_version.map +1 -0
index d8038749a..93a4d30f8 100644
@@ -372,6 +372,7 @@ EXPERIMENTAL {
 	__rte_trace_point_register;
 	per_lcore_trace_mem;
 	per_lcore_trace_point_sz;
+	rte_eal_vfio_get_vf_token;
 	rte_log_can_log;
 	rte_thread_getname;
 	rte_trace_dump;
Signature · 1 line
2.26.2
#81 lore raw

v16 cover support for VFIO-PCI VF token interface

In reply to [email protected]

v16: Rebase the patch for 20.08 release note.

v15: Add the missed EXPERIMENTAL warning for API doxgen.

v14: Rebase the patch for 20.08 release note.

v13: Rename the EAL get VF token function, and leave the freebsd type as empty.

v12: support to vfio devices with VF token and no token.

v11: Use the eal parameter to pass the VF token, then not every PCI
device needs to be specified with this token. Also no ABI issue now.

v10: Use the __rte_internal to mark the internal API changing.

v9: Rewrite the document.

v8: Update the document.

v7: Add the Fixes tag in uuid, the release note and help
document.

v6: Drop the Fixes tag in uuid, since the file has been
moved to another place, not suitable to apply on stable. And this is not a bug, just some kind of enhancement.

v5: 1. Add the VF token parse error handling.
2. Split into two patches for different logic module.
3. Add more comments into the code for explaining the design.
4. Drop the ABI change workaround, this patch set focuses on code review.

v4: 1. Ignore rte_vfio_setup_device ABI check since it is
for Linux driver use.

v3: Fix the Travis build failed:
(1). rte_uuid.h:97:55: error: unknown type name ‘size_t’ (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’

v2: Fix the FreeBSD build error.

v1: Update the commit message.

RFC v2:
Based on Vamsi's RFC v1, and Alex's patch for Qemu
[https://lore.kernel.org/lkml/[email protected]/]:
Use the devarg to pass-down the VF token.

RFC v1: https://patchwork.dpdk.org/patch/66281/ by Vamsi.

Haiyue Wang (2):
eal: add uuid dependent header files explicitly
eal: support for VFIO-PCI VF token

 doc/guides/linux_gsg/linux_drivers.rst        | 35 ++++++++++++++++++-
 doc/guides/linux_gsg/linux_eal_parameters.rst |  4 +++
 doc/guides/rel_notes/release_20_08.rst        |  5 +++
 lib/librte_eal/common/eal_common_options.c    |  2 ++
 lib/librte_eal/common/eal_internal_cfg.h      |  2 ++
 lib/librte_eal/common/eal_options.h           |  2 ++
 lib/librte_eal/freebsd/eal.c                  |  4 +++
 lib/librte_eal/include/rte_eal.h              | 15 ++++++++
 lib/librte_eal/include/rte_uuid.h             |  2 ++
 lib/librte_eal/linux/eal.c                    | 29 +++++++++++++++
 lib/librte_eal/linux/eal_vfio.c               | 19 ++++++++++
 lib/librte_eal/rte_eal_version.map            |  1 +
 12 files changed, 119 insertions(+), 1 deletion(-)
Signature · 1 line
2.27.0
#82 lore raw

PATCH v16 1/2 eal: add uuid dependent header files explicitly

Add the dependent header files explicitly, so that the user just needs to include the 'rte_uuid.h' header file directly to avoid compile error:
(1). rte_uuid.h:97:55: error: unknown type name ‘size_t’ (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’

Fixes 6bc67c497a51 ("eal: add uuid API")
---
 lib/librte_eal/include/rte_uuid.h | 2 ++
 1 file changed, 2 insertions(+)
lib/librte_eal/include/rte_uuid.h +2 -0
index 044afbdfa..8b42e070a 100644
@@ -15,6 +15,8 @@ extern "C" {
 #endif
 
 #include <stdbool.h>
+#include <stddef.h>
+#include <string.h>
 
 /**
  * Struct describing a Universal Unique Identifier
Signature · 1 line
2.27.0
#83 lore raw

PATCH v16 2/2 eal: support for VFIO-PCI VF token

In reply to Haiyue Wang

The kernel module vfio-pci introduces the VF token to enable SR-IOV support since 5.7.

The VF token can be set by a vfio-pci based PF driver and must be known by the vfio-pci based VF driver in order to gain access to the device.

---
 doc/guides/linux_gsg/linux_drivers.rst        | 35 ++++++++++++++++++-
 doc/guides/linux_gsg/linux_eal_parameters.rst |  4 +++
 doc/guides/rel_notes/release_20_08.rst        |  5 +++
 lib/librte_eal/common/eal_common_options.c    |  2 ++
 lib/librte_eal/common/eal_internal_cfg.h      |  2 ++
 lib/librte_eal/common/eal_options.h           |  2 ++
 lib/librte_eal/freebsd/eal.c                  |  4 +++
 lib/librte_eal/include/rte_eal.h              | 15 ++++++++
 lib/librte_eal/linux/eal.c                    | 29 +++++++++++++++
 lib/librte_eal/linux/eal_vfio.c               | 19 ++++++++++
 lib/librte_eal/rte_eal_version.map            |  1 +
 11 files changed, 117 insertions(+), 1 deletion(-)
doc/guides/linux_gsg/linux_drivers.rst +34 -1
index d40b495c1..02fba7d1e 100644
@@ -78,11 +78,44 @@ Note that in order to use VFIO, your kernel must support it.
 VFIO kernel modules have been included in the Linux kernel since version 3.6.0 and are usually present by default,
 however please consult your distributions documentation to make sure that is the case.
 
+The ``vfio-pci`` module since Linux version 5.7 supports the creation of virtual
+functions. After the PF is bound to vfio-pci module, the user can create the VFs
+by sysfs interface, and these VFs are bound to vfio-pci module automatically.
+
+When the PF is bound to vfio-pci, it has initial VF token generated by random. For
+security reason, this token is write only, the user can't read it from the kernel
+directly. For accessing the VF, the user needs to start the PF with token parameter
+to setup a VF token (uuid format), then the VF can be accessed with this new known
+VF token.
+
+DPDK will use the EAL parameter ``--vfio-vf-token`` to specify the VF token value to
+PF and its related VFs, this VF token will be shared in all VFIO devices, including
+the different PFs.
+
+.. code-block:: console
+
+    1. Generate the VF token by uuid command
+        14d63f20-8445-11ea-8900-1f9ce7d5650d
+
+    2. sudo modprobe vfio-pci enable_sriov=1
+
+    2. ./usertools/dpdk-devbind.py -b vfio-pci 0000:86:00.0
+
+    3. echo 2 > /sys/bus/pci/devices/0000:86:00.0/sriov_numvfs
+
+    4. Start the PF:
+        ./x86_64-native-linux-gcc/app/testpmd -l 22-25 -n 4 -w 86:00.0 \
+         --vfio-vf-token=14d63f20-8445-11ea-8900-1f9ce7d5650d --file-prefix=pf -- -i
+
+    5. Start the VF:
+        ./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 -w 86:02.0 \
+         --vfio-vf-token=14d63f20-8445-11ea-8900-1f9ce7d5650d --file-prefix=vf0 -- -i
+
 Also, to use VFIO, both kernel and BIOS must support and be configured to use IO virtualization (such as Intel® VT-d).
 
 .. note::
 
-    ``vfio-pci`` module doesn't support the creation of virtual functions.
+    ``vfio-pci`` module doesn't support the creation of virtual functions before Linux version 5.7.
 
 For proper operation of VFIO when running DPDK applications as a non-privileged user, correct permissions should also be set up.
 This can be done by using the DPDK setup script (called dpdk-setup.sh and located in the usertools directory).
doc/guides/linux_gsg/linux_eal_parameters.rst +4 -0
index b2cc60e44..bd3977cb3 100644
@@ -40,6 +40,10 @@ Device-related options
 
     Use specified interrupt mode for devices bound to VFIO kernel driver.
 
+*   ``--vfio-vf-token <uuid>``
+
+    Use specified VF token for devices bound to VFIO kernel driver.
+
 Multiprocessing-related options
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
doc/guides/rel_notes/release_20_08.rst +5 -0
index 86d240213..ca7fd521b 100644
@@ -62,6 +62,11 @@ New Features
 
   * Added new PMD devarg ``reclaim_mem_mode``.
 
+* **Added the support for vfio-pci new VF token interface.**
+
+  Since Linux version 5.7, vfio-pci supports a shared VF token (UUID) to represent
+  the trust between SR-IOV PF and the created VFs. Update the method to gain access
+  to the PF and VFs devices by appending the VF token parameter.
 
 Removed Items
 -------------
lib/librte_eal/common/eal_common_options.c +2 -0
index 0546beb3a..fc889309c 100644
@@ -94,6 +94,7 @@ eal_long_options[] = {
 	{OPT_SYSLOG,            1, NULL, OPT_SYSLOG_NUM           },
 	{OPT_VDEV,              1, NULL, OPT_VDEV_NUM             },
 	{OPT_VFIO_INTR,         1, NULL, OPT_VFIO_INTR_NUM        },
+	{OPT_VFIO_VF_TOKEN,     1, NULL, OPT_VFIO_VF_TOKEN_NUM    },
 	{OPT_VMWARE_TSC_MAP,    0, NULL, OPT_VMWARE_TSC_MAP_NUM   },
 	{OPT_LEGACY_MEM,        0, NULL, OPT_LEGACY_MEM_NUM       },
 	{OPT_SINGLE_FILE_SEGMENTS, 0, NULL, OPT_SINGLE_FILE_SEGMENTS_NUM},
@@ -301,6 +302,7 @@ eal_reset_internal_config(struct internal_config *internal_cfg)
 
 	/* if set to NONE, interrupt mode is determined automatically */
 	internal_cfg->vfio_intr_mode = RTE_INTR_MODE_NONE;
+	memset(internal_cfg->vfio_vf_token, 0, sizeof(rte_uuid_t));
 
 #ifdef RTE_LIBEAL_USE_HPET
 	internal_cfg->no_hpet = 0;
lib/librte_eal/common/eal_internal_cfg.h +2 -0
index c650bc081..43095b7a2 100644
@@ -72,6 +72,8 @@ struct internal_config {
 	volatile int syslog_facility;	  /**< facility passed to openlog() */
 	/** default interrupt mode for VFIO */
 	volatile enum rte_intr_mode vfio_intr_mode;
+	/** the shared VF token for VFIO-PCI bound PF and VFs devices */
+	rte_uuid_t vfio_vf_token;
 	char *hugefile_prefix;      /**< the base filename of hugetlbfs files */
 	char *hugepage_dir;         /**< specific hugetlbfs directory to use */
 	char *user_mbuf_pool_ops_name;
lib/librte_eal/common/eal_options.h +2 -0
index 18e6da9ab..89769d48b 100644
@@ -69,6 +69,8 @@ enum {
 	OPT_VDEV_NUM,
 #define OPT_VFIO_INTR         "vfio-intr"
 	OPT_VFIO_INTR_NUM,
+#define OPT_VFIO_VF_TOKEN     "vfio-vf-token"
+	OPT_VFIO_VF_TOKEN_NUM,
 #define OPT_VMWARE_TSC_MAP    "vmware-tsc-map"
 	OPT_VMWARE_TSC_MAP_NUM,
 #define OPT_LEGACY_MEM    "legacy-mem"
lib/librte_eal/freebsd/eal.c +4 -0
index c41f265fa..4ff5028d8 100644
@@ -1005,6 +1005,10 @@ rte_eal_vfio_intr_mode(void)
 	return RTE_INTR_MODE_NONE;
 }
 
+void rte_eal_vfio_get_vf_token(__rte_unused rte_uuid_t vf_token)
+{
+}
+
 int rte_vfio_setup_device(__rte_unused const char *sysfs_base,
 		      __rte_unused const char *dev_addr,
 		      __rte_unused int *vfio_dev_fd,
lib/librte_eal/include/rte_eal.h +15 -0
index 2f9ed298d..2f47347dc 100644
@@ -21,6 +21,7 @@
 #include <rte_bus.h>
 
 #include <rte_pci_dev_feature_defs.h>
+#include <rte_uuid.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -438,6 +439,20 @@ int rte_eal_create_uio_dev(void);
  */
 enum rte_intr_mode rte_eal_vfio_intr_mode(void);
 
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Copy the user-configured vfio VF token.
+ *
+ * @param vf_token
+ *   vfio VF token configured with the command line is copied
+ *   into this parameter, zero uuid by default.
+ */
+__rte_experimental
+void rte_eal_vfio_get_vf_token(rte_uuid_t vf_token);
+
 /**
  * A wrap API for syscall gettid.
  *
lib/librte_eal/linux/eal.c +29 -0
index f162124a3..00f3fdced 100644
@@ -558,6 +558,7 @@ eal_usage(const char *prgname)
 	       "  --"OPT_FILE_PREFIX"       Prefix for hugepage filenames\n"
 	       "  --"OPT_CREATE_UIO_DEV"    Create /dev/uioX (usually done by hotplug)\n"
 	       "  --"OPT_VFIO_INTR"         Interrupt mode for VFIO (legacy|msi|msix)\n"
+	       "  --"OPT_VFIO_VF_TOKEN"     VF token (UUID) shared between SR-IOV PF and VFs\n"
 	       "  --"OPT_LEGACY_MEM"        Legacy memory mode (no dynamic allocation, contiguous segments)\n"
 	       "  --"OPT_SINGLE_FILE_SEGMENTS" Put all hugepage memory in single files\n"
 	       "  --"OPT_MATCH_ALLOCATIONS" Free hugepages exactly as allocated\n"
@@ -649,6 +650,19 @@ eal_parse_vfio_intr(const char *mode)
 	return -1;
 }
 
+static int
+eal_parse_vfio_vf_token(const char *vf_token)
+{
+	rte_uuid_t uuid;
+
+	if (!rte_uuid_parse(vf_token, uuid)) {
+		rte_uuid_copy(internal_config.vfio_vf_token, uuid);
+		return 0;
+	}
+
+	return -1;
+}
+
 /* Parse the arguments for --log-level only */
 static void
 eal_log_level_parse(int argc, char **argv)
@@ -787,6 +801,16 @@ eal_parse_args(int argc, char **argv)
 			}
 			break;
 
+		case OPT_VFIO_VF_TOKEN_NUM:
+			if (eal_parse_vfio_vf_token(optarg) < 0) {
+				RTE_LOG(ERR, EAL, "invalid parameters for --"
+						OPT_VFIO_VF_TOKEN "\n");
+				eal_usage(prgname);
+				ret = -1;
+				goto out;
+			}
+			break;
+
 		case OPT_CREATE_UIO_DEV_NUM:
 			internal_config.create_uio_dev = 1;
 			break;
@@ -1370,6 +1394,11 @@ rte_eal_vfio_intr_mode(void)
 	return internal_config.vfio_intr_mode;
 }
 
+void rte_eal_vfio_get_vf_token(rte_uuid_t vf_token)
+{
+	rte_uuid_copy(vf_token, internal_config.vfio_vf_token);
+}
+
 int
 rte_eal_check_module(const char *module_name)
 {
lib/librte_eal/linux/eal_vfio.c +19 -0
index d26e1649a..5b2f6b305 100644
@@ -712,6 +712,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 	int vfio_container_fd;
 	int vfio_group_fd;
 	int iommu_group_num;
+	rte_uuid_t vf_token;
 	int i, ret;
 
 	/* get group number */
@@ -895,6 +896,23 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 				t->type_id, t->name);
 	}
 
+	rte_eal_vfio_get_vf_token(vf_token);
+
+	/* get a file descriptor for the device with VF token firstly */
+	if (!rte_uuid_is_null(vf_token)) {
+		char vf_token_str[RTE_UUID_STRLEN];
+		char dev[PATH_MAX];
+
+		rte_uuid_unparse(vf_token, vf_token_str, sizeof(vf_token_str));
+		snprintf(dev, sizeof(dev),
+			 "%s vf_token=%s", dev_addr, vf_token_str);
+
+		*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD,
+				     dev);
+		if (*vfio_dev_fd >= 0)
+			goto dev_get_info;
+	}
+
 	/* get a file descriptor for the device */
 	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev_addr);
 	if (*vfio_dev_fd < 0) {
@@ -909,6 +927,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 		return -1;
 	}
 
+dev_get_info:
 	/* test and setup the device */
 	ret = ioctl(*vfio_dev_fd, VFIO_DEVICE_GET_INFO, device_info);
 	if (ret) {
lib/librte_eal/rte_eal_version.map +1 -0
index 196eef5af..16c8de27e 100644
@@ -372,6 +372,7 @@ EXPERIMENTAL {
 	__rte_trace_point_register;
 	per_lcore_trace_mem;
 	per_lcore_trace_point_sz;
+	rte_eal_vfio_get_vf_token;
 	rte_log_can_log;
 	rte_thread_getname;
 	rte_trace_dump;
Signature · 1 line
2.27.0
Tested-by on v16 2/2
#84 lore raw

Re: [dpdk-dev] [PATCH v16 2/2] eal: support for VFIO-PCI VF token

Quoted 10 lines · Haiyue Wang

The kernel module vfio-pci introduces the VF token to enable SR-IOV support since 5.7.

The VF token can be set by a vfio-pci based PF driver and must be known by the vfio-pci based VF driver in order to gain access to the device.

---

Quoted 308 lines
 doc/guides/linux_gsg/linux_drivers.rst        | 35 ++++++++++++++++++-
 doc/guides/linux_gsg/linux_eal_parameters.rst |  4 +++
 doc/guides/rel_notes/release_20_08.rst        |  5 +++
 lib/librte_eal/common/eal_common_options.c    |  2 ++
 lib/librte_eal/common/eal_internal_cfg.h      |  2 ++
 lib/librte_eal/common/eal_options.h           |  2 ++
 lib/librte_eal/freebsd/eal.c                  |  4 +++
 lib/librte_eal/include/rte_eal.h              | 15 ++++++++
 lib/librte_eal/linux/eal.c                    | 29 +++++++++++++++
 lib/librte_eal/linux/eal_vfio.c               | 19 ++++++++++
 lib/librte_eal/rte_eal_version.map            |  1 +
 11 files changed, 117 insertions(+), 1 deletion(-)
doc/guides/linux_gsg/linux_drivers.rst +34 -1
index d40b495c1..02fba7d1e 100644
@@ -78,11 +78,44 @@ Note that in order to use VFIO, your kernel must support it.
 VFIO kernel modules have been included in the Linux kernel since version 3.6.0 and are usually present by default,
 however please consult your distributions documentation to make sure that is the case.
 
+The ``vfio-pci`` module since Linux version 5.7 supports the creation of virtual
+functions. After the PF is bound to vfio-pci module, the user can create the VFs
+by sysfs interface, and these VFs are bound to vfio-pci module automatically.
+
+When the PF is bound to vfio-pci, it has initial VF token generated by random. For
+security reason, this token is write only, the user can't read it from the kernel
+directly. For accessing the VF, the user needs to start the PF with token parameter
+to setup a VF token (uuid format), then the VF can be accessed with this new known
+VF token.
+
+DPDK will use the EAL parameter ``--vfio-vf-token`` to specify the VF token value to
+PF and its related VFs, this VF token will be shared in all VFIO devices, including
+the different PFs.
+
+.. code-block:: console
+
+    1. Generate the VF token by uuid command
+        14d63f20-8445-11ea-8900-1f9ce7d5650d
+
+    2. sudo modprobe vfio-pci enable_sriov=1
+
+    2. ./usertools/dpdk-devbind.py -b vfio-pci 0000:86:00.0
+
+    3. echo 2 > /sys/bus/pci/devices/0000:86:00.0/sriov_numvfs
+
+    4. Start the PF:
+        ./x86_64-native-linux-gcc/app/testpmd -l 22-25 -n 4 -w 86:00.0 \
+         --vfio-vf-token=14d63f20-8445-11ea-8900-1f9ce7d5650d --file-prefix=pf -- -i
+
+    5. Start the VF:
+        ./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 -w 86:02.0 \
+         --vfio-vf-token=14d63f20-8445-11ea-8900-1f9ce7d5650d --file-prefix=vf0 -- -i
+
 Also, to use VFIO, both kernel and BIOS must support and be configured to use IO virtualization (such as Intel® VT-d).
 
 .. note::
 
-    ``vfio-pci`` module doesn't support the creation of virtual functions.
+    ``vfio-pci`` module doesn't support the creation of virtual functions before Linux version 5.7.
 
 For proper operation of VFIO when running DPDK applications as a non-privileged user, correct permissions should also be set up.
 This can be done by using the DPDK setup script (called dpdk-setup.sh and located in the usertools directory).
doc/guides/linux_gsg/linux_eal_parameters.rst +4 -0
index b2cc60e44..bd3977cb3 100644
@@ -40,6 +40,10 @@ Device-related options
 
     Use specified interrupt mode for devices bound to VFIO kernel driver.
 
+*   ``--vfio-vf-token <uuid>``
+
+    Use specified VF token for devices bound to VFIO kernel driver.
+
 Multiprocessing-related options
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
doc/guides/rel_notes/release_20_08.rst +5 -0
index 86d240213..ca7fd521b 100644
@@ -62,6 +62,11 @@ New Features
 
   * Added new PMD devarg ``reclaim_mem_mode``.
 
+* **Added the support for vfio-pci new VF token interface.**
+
+  Since Linux version 5.7, vfio-pci supports a shared VF token (UUID) to represent
+  the trust between SR-IOV PF and the created VFs. Update the method to gain access
+  to the PF and VFs devices by appending the VF token parameter.
 
 Removed Items
 -------------
lib/librte_eal/common/eal_common_options.c +2 -0
index 0546beb3a..fc889309c 100644
@@ -94,6 +94,7 @@ eal_long_options[] = {
 	{OPT_SYSLOG,            1, NULL, OPT_SYSLOG_NUM           },
 	{OPT_VDEV,              1, NULL, OPT_VDEV_NUM             },
 	{OPT_VFIO_INTR,         1, NULL, OPT_VFIO_INTR_NUM        },
+	{OPT_VFIO_VF_TOKEN,     1, NULL, OPT_VFIO_VF_TOKEN_NUM    },
 	{OPT_VMWARE_TSC_MAP,    0, NULL, OPT_VMWARE_TSC_MAP_NUM   },
 	{OPT_LEGACY_MEM,        0, NULL, OPT_LEGACY_MEM_NUM       },
 	{OPT_SINGLE_FILE_SEGMENTS, 0, NULL, OPT_SINGLE_FILE_SEGMENTS_NUM},
@@ -301,6 +302,7 @@ eal_reset_internal_config(struct internal_config *internal_cfg)
 
 	/* if set to NONE, interrupt mode is determined automatically */
 	internal_cfg->vfio_intr_mode = RTE_INTR_MODE_NONE;
+	memset(internal_cfg->vfio_vf_token, 0, sizeof(rte_uuid_t));
 
 #ifdef RTE_LIBEAL_USE_HPET
 	internal_cfg->no_hpet = 0;
lib/librte_eal/common/eal_internal_cfg.h +2 -0
index c650bc081..43095b7a2 100644
@@ -72,6 +72,8 @@ struct internal_config {
 	volatile int syslog_facility;	  /**< facility passed to openlog() */
 	/** default interrupt mode for VFIO */
 	volatile enum rte_intr_mode vfio_intr_mode;
+	/** the shared VF token for VFIO-PCI bound PF and VFs devices */
+	rte_uuid_t vfio_vf_token;
 	char *hugefile_prefix;      /**< the base filename of hugetlbfs files */
 	char *hugepage_dir;         /**< specific hugetlbfs directory to use */
 	char *user_mbuf_pool_ops_name;
lib/librte_eal/common/eal_options.h +2 -0
index 18e6da9ab..89769d48b 100644
@@ -69,6 +69,8 @@ enum {
 	OPT_VDEV_NUM,
 #define OPT_VFIO_INTR         "vfio-intr"
 	OPT_VFIO_INTR_NUM,
+#define OPT_VFIO_VF_TOKEN     "vfio-vf-token"
+	OPT_VFIO_VF_TOKEN_NUM,
 #define OPT_VMWARE_TSC_MAP    "vmware-tsc-map"
 	OPT_VMWARE_TSC_MAP_NUM,
 #define OPT_LEGACY_MEM    "legacy-mem"
lib/librte_eal/freebsd/eal.c +4 -0
index c41f265fa..4ff5028d8 100644
@@ -1005,6 +1005,10 @@ rte_eal_vfio_intr_mode(void)
 	return RTE_INTR_MODE_NONE;
 }
 
+void rte_eal_vfio_get_vf_token(__rte_unused rte_uuid_t vf_token)
+{
+}
+
 int rte_vfio_setup_device(__rte_unused const char *sysfs_base,
 		      __rte_unused const char *dev_addr,
 		      __rte_unused int *vfio_dev_fd,
lib/librte_eal/include/rte_eal.h +15 -0
index 2f9ed298d..2f47347dc 100644
@@ -21,6 +21,7 @@
 #include <rte_bus.h>
 
 #include <rte_pci_dev_feature_defs.h>
+#include <rte_uuid.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -438,6 +439,20 @@ int rte_eal_create_uio_dev(void);
  */
 enum rte_intr_mode rte_eal_vfio_intr_mode(void);
 
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Copy the user-configured vfio VF token.
+ *
+ * @param vf_token
+ *   vfio VF token configured with the command line is copied
+ *   into this parameter, zero uuid by default.
+ */
+__rte_experimental
+void rte_eal_vfio_get_vf_token(rte_uuid_t vf_token);
+
 /**
  * A wrap API for syscall gettid.
  *
lib/librte_eal/linux/eal.c +29 -0
index f162124a3..00f3fdced 100644
@@ -558,6 +558,7 @@ eal_usage(const char *prgname)
 	       "  --"OPT_FILE_PREFIX"       Prefix for hugepage filenames\n"
 	       "  --"OPT_CREATE_UIO_DEV"    Create /dev/uioX (usually done by hotplug)\n"
 	       "  --"OPT_VFIO_INTR"         Interrupt mode for VFIO (legacy|msi|msix)\n"
+	       "  --"OPT_VFIO_VF_TOKEN"     VF token (UUID) shared between SR-IOV PF and VFs\n"
 	       "  --"OPT_LEGACY_MEM"        Legacy memory mode (no dynamic allocation, contiguous segments)\n"
 	       "  --"OPT_SINGLE_FILE_SEGMENTS" Put all hugepage memory in single files\n"
 	       "  --"OPT_MATCH_ALLOCATIONS" Free hugepages exactly as allocated\n"
@@ -649,6 +650,19 @@ eal_parse_vfio_intr(const char *mode)
 	return -1;
 }
 
+static int
+eal_parse_vfio_vf_token(const char *vf_token)
+{
+	rte_uuid_t uuid;
+
+	if (!rte_uuid_parse(vf_token, uuid)) {
+		rte_uuid_copy(internal_config.vfio_vf_token, uuid);
+		return 0;
+	}
+
+	return -1;
+}
+
 /* Parse the arguments for --log-level only */
 static void
 eal_log_level_parse(int argc, char **argv)
@@ -787,6 +801,16 @@ eal_parse_args(int argc, char **argv)
 			}
 			break;
 
+		case OPT_VFIO_VF_TOKEN_NUM:
+			if (eal_parse_vfio_vf_token(optarg) < 0) {
+				RTE_LOG(ERR, EAL, "invalid parameters for --"
+						OPT_VFIO_VF_TOKEN "\n");
+				eal_usage(prgname);
+				ret = -1;
+				goto out;
+			}
+			break;
+
 		case OPT_CREATE_UIO_DEV_NUM:
 			internal_config.create_uio_dev = 1;
 			break;
@@ -1370,6 +1394,11 @@ rte_eal_vfio_intr_mode(void)
 	return internal_config.vfio_intr_mode;
 }
 
+void rte_eal_vfio_get_vf_token(rte_uuid_t vf_token)
+{
+	rte_uuid_copy(vf_token, internal_config.vfio_vf_token);
+}
+
 int
 rte_eal_check_module(const char *module_name)
 {
lib/librte_eal/linux/eal_vfio.c +19 -0
index d26e1649a..5b2f6b305 100644
@@ -712,6 +712,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 	int vfio_container_fd;
 	int vfio_group_fd;
 	int iommu_group_num;
+	rte_uuid_t vf_token;
 	int i, ret;
 
 	/* get group number */
@@ -895,6 +896,23 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 				t->type_id, t->name);
 	}
 
+	rte_eal_vfio_get_vf_token(vf_token);
+
+	/* get a file descriptor for the device with VF token firstly */
+	if (!rte_uuid_is_null(vf_token)) {
+		char vf_token_str[RTE_UUID_STRLEN];
+		char dev[PATH_MAX];
+
+		rte_uuid_unparse(vf_token, vf_token_str, sizeof(vf_token_str));
+		snprintf(dev, sizeof(dev),
+			 "%s vf_token=%s", dev_addr, vf_token_str);
+
+		*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD,
+				     dev);
+		if (*vfio_dev_fd >= 0)
+			goto dev_get_info;
+	}
+
 	/* get a file descriptor for the device */
 	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev_addr);
 	if (*vfio_dev_fd < 0) {
@@ -909,6 +927,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 		return -1;
 	}
 
+dev_get_info:
 	/* test and setup the device */
 	ret = ioctl(*vfio_dev_fd, VFIO_DEVICE_GET_INFO, device_info);
 	if (ret) {
lib/librte_eal/rte_eal_version.map +1 -0
index 196eef5af..16c8de27e 100644
@@ -372,6 +372,7 @@ EXPERIMENTAL {
 	__rte_trace_point_register;
 	per_lcore_trace_mem;
 	per_lcore_trace_point_sz;
+	rte_eal_vfio_get_vf_token;
 	rte_log_can_log;
 	rte_thread_getname;
 	rte_trace_dump;
Signature · 1 line
2.27.0
#85 lore raw

Re: [dpdk-dev] [PATCH v16 2/2] eal: support for VFIO-PCI VF token

Quoted 8 lines · Harman Kalra
Quoted 6 lines · Haiyue Wang

The kernel module vfio-pci introduces the VF token to enable SR-IOV support since 5.7.

The VF token can be set by a vfio-pci based PF driver and must be known by the vfio-pci based VF driver in order to gain access to the device.

Quoted 6 lines
Quoted 4 lines

---

Thanks for the test Harman.

I can see no complaint on using a single token for all devices, which is the only concern I would have with the last revision. If everyone is ok with this choice, I will take this for -rc1.

Signature · 1 line
David Marchand
#86 lore raw

Re: [dpdk-dev] [PATCH v16 2/2] eal: support for VFIO-PCI VF token

Quoted 23 lines

-----Original Message-----
From: David Marchand <[email protected]>
Sent: Thursday, June 25, 2020 15:33
To: Harman Kalra <[email protected]>; Jerin Jacob Kollanukkaran <[email protected]>
Cc: Wang, Haiyue <[email protected]>; dev <[email protected]>; Burakov, Anatoly <[email protected]>; Thomas Monjalon <[email protected]>; Andrew Rybchenko <[email protected]>; Maxime Coquelin <[email protected]>
Subject: Re: [dpdk-dev] [PATCH v16 2/2] eal: support for VFIO-PCI VF token

Quoted 13 lines · Harman Kalra
Quoted 10 lines · Haiyue Wang

The kernel module vfio-pci introduces the VF token to enable SR-IOV support since 5.7.

The VF token can be set by a vfio-pci based PF driver and must be known by the vfio-pci based VF driver in order to gain access to the device.

---

Quoted 5 lines
Quoted 1 line

Thanks for the test Harman.

I can see no complaint on using a single token for all devices, which

Yeah, not the best, but may meet with the most needed: create VFs by vfio-pci ;-)

Since devarg was a private option for ALL kind of devices, and have to break the ABI policy to implement the design, so I drop this revision, choose to use it as a global option as "vfio-intr" option does.

Quoted 6 lines

is the only concern I would have with the last revision. If everyone is ok with this choice, I will take this for -rc1.

--
David Marchand

#87 lore raw

v17 cover support for VFIO-PCI VF token interface

In reply to [email protected]

v17: Rebase for new EAL config API, update the commit message and doc.

v16: Rebase the patch for 20.08 release note.

v15: Add the missed EXPERIMENTAL warning for API doxgen.

v14: Rebase the patch for 20.08 release note.

v13: Rename the EAL get VF token function, and leave the freebsd type as empty.

v12: support to vfio devices with VF token and no token.

v11: Use the eal parameter to pass the VF token, then not every PCI
device needs to be specified with this token. Also no ABI issue now.

v10: Use the __rte_internal to mark the internal API changing.

v9: Rewrite the document.

v8: Update the document.

v7: Add the Fixes tag in uuid, the release note and help
document.

v6: Drop the Fixes tag in uuid, since the file has been
moved to another place, not suitable to apply on stable. And this is not a bug, just some kind of enhancement.

v5: 1. Add the VF token parse error handling.
2. Split into two patches for different logic module.
3. Add more comments into the code for explaining the design.
4. Drop the ABI change workaround, this patch set focuses on code review.

v4: 1. Ignore rte_vfio_setup_device ABI check since it is
for Linux driver use.

v3: Fix the Travis build failed:
(1). rte_uuid.h:97:55: error: unknown type name ‘size_t’ (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’

v2: Fix the FreeBSD build error.

v1: Update the commit message.

RFC v2:
Based on Vamsi's RFC v1, and Alex's patch for Qemu
[https://lore.kernel.org/lkml/[email protected]/]:
Use the devarg to pass-down the VF token.

RFC v1: https://patchwork.dpdk.org/patch/66281/ by Vamsi.

Haiyue Wang (2):
eal: add uuid dependent header files explicitly
eal: support for VFIO-PCI VF token

 doc/guides/linux_gsg/linux_drivers.rst        | 35 ++++++++++++++++++-
 doc/guides/linux_gsg/linux_eal_parameters.rst |  4 +++
 doc/guides/rel_notes/release_20_08.rst        |  6 ++++
 lib/librte_eal/common/eal_common_options.c    |  3 ++
 lib/librte_eal/common/eal_internal_cfg.h      |  2 ++
 lib/librte_eal/common/eal_options.h           |  2 ++
 lib/librte_eal/freebsd/eal.c                  |  5 +++
 lib/librte_eal/include/rte_eal.h              | 14 ++++++++
 lib/librte_eal/include/rte_uuid.h             |  2 ++
 lib/librte_eal/linux/eal.c                    | 33 +++++++++++++++++
 lib/librte_eal/linux/eal_vfio.c               | 19 ++++++++++
 lib/librte_eal/rte_eal_version.map            |  3 ++
 12 files changed, 127 insertions(+), 1 deletion(-)
Signature · 1 line
2.27.0
#88 lore raw

PATCH v17 1/2 eal: add uuid dependent header files explicitly

Add the dependent header files explicitly, so that the user just needs to include the 'rte_uuid.h' header file directly to avoid compile error:
(1). rte_uuid.h:97:55: error: unknown type name ‘size_t’ (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’

Fixes 6bc67c497a51 ("eal: add uuid API")
---
 lib/librte_eal/include/rte_uuid.h | 2 ++
 1 file changed, 2 insertions(+)
lib/librte_eal/include/rte_uuid.h +2 -0
index 044afbdfa..8b42e070a 100644
@@ -15,6 +15,8 @@ extern "C" {
 #endif
 
 #include <stdbool.h>
+#include <stddef.h>
+#include <string.h>
 
 /**
  * Struct describing a Universal Unique Identifier
Signature · 1 line
2.27.0
Acked-by on v17 1/2
#89 lore raw

Re: [dpdk-dev] [PATCH v17 1/2] eal: add uuid dependent header files explicitly

Quoted 28 lines · Haiyue Wang

Add the dependent header files explicitly, so that the user just needs to include the 'rte_uuid.h' header file directly to avoid compile error:
(1). rte_uuid.h:97:55: error: unknown type name ‘size_t’ (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’

Fixes 6bc67c497a51 ("eal: add uuid API")
---
 lib/librte_eal/include/rte_uuid.h | 2 ++
 1 file changed, 2 insertions(+)
lib/librte_eal/include/rte_uuid.h +2 -1
index 044afbdfa..8b42e070a 100644
@@ -15,6 +15,8 @@ extern "C" {
 #endif
 
 #include <stdbool.h>
+#include <stddef.h>
+#include <string.h>
 
 /**
  * Struct describing a Universal Unique Identifier
--
Quoted 2 lines

2.27.0

Signature · 1 line
David Marchand
#90 lore raw

PATCH v17 2/2 eal: support for VFIO-PCI VF token

In reply to Haiyue Wang

The kernel module vfio-pci introduces the VF token to enable SR-IOV support since 5.7.

The VF token can be set by a vfio-pci based PF driver and must be known by the vfio-pci based VF driver in order to gain access to the device.

Since the vfio-pci module uses the VF token as internal data to provide the collaboration between SR-IOV PF and VFs, so DPDK can use the same VF token for all PF devices by specifying the related EAL option.

---
 doc/guides/linux_gsg/linux_drivers.rst        | 35 ++++++++++++++++++-
 doc/guides/linux_gsg/linux_eal_parameters.rst |  4 +++
 doc/guides/rel_notes/release_20_08.rst        |  6 ++++
 lib/librte_eal/common/eal_common_options.c    |  3 ++
 lib/librte_eal/common/eal_internal_cfg.h      |  2 ++
 lib/librte_eal/common/eal_options.h           |  2 ++
 lib/librte_eal/freebsd/eal.c                  |  5 +++
 lib/librte_eal/include/rte_eal.h              | 14 ++++++++
 lib/librte_eal/linux/eal.c                    | 33 +++++++++++++++++
 lib/librte_eal/linux/eal_vfio.c               | 19 ++++++++++
 lib/librte_eal/rte_eal_version.map            |  3 ++
 11 files changed, 125 insertions(+), 1 deletion(-)
doc/guides/linux_gsg/linux_drivers.rst +34 -1
index d40b495c1..4eda3d5bf 100644
@@ -78,11 +78,44 @@ Note that in order to use VFIO, your kernel must support it.
 VFIO kernel modules have been included in the Linux kernel since version 3.6.0 and are usually present by default,
 however please consult your distributions documentation to make sure that is the case.
 
+The ``vfio-pci`` module since Linux version 5.7 supports the creation of virtual
+functions. After the PF is bound to vfio-pci module, the user can create the VFs
+by sysfs interface, and these VFs are bound to vfio-pci module automatically.
+
+When the PF is bound to vfio-pci, it has initial VF token generated by random. For
+security reason, this token is write only, the user can't read it from the kernel
+directly. To access the VF, the user needs to start the PF with token parameter to
+setup a VF token in UUID format, then the VF can be accessed with this new token.
+
+Since the ``vfio-pci`` module uses the VF token as internal data to provide the
+collaboration between SR-IOV PF and VFs, so DPDK can use the same VF token for all
+PF devices which bound to one application. This VF token can be specified by the EAL
+parameter ``--vfio-vf-token``.
+
+.. code-block:: console
+
+    1. Generate the VF token by uuid command
+        14d63f20-8445-11ea-8900-1f9ce7d5650d
+
+    2. sudo modprobe vfio-pci enable_sriov=1
+
+    2. ./usertools/dpdk-devbind.py -b vfio-pci 0000:86:00.0
+
+    3. echo 2 > /sys/bus/pci/devices/0000:86:00.0/sriov_numvfs
+
+    4. Start the PF:
+        ./x86_64-native-linux-gcc/app/testpmd -l 22-25 -n 4 -w 86:00.0 \
+         --vfio-vf-token=14d63f20-8445-11ea-8900-1f9ce7d5650d --file-prefix=pf -- -i
+
+    5. Start the VF:
+        ./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 -w 86:02.0 \
+         --vfio-vf-token=14d63f20-8445-11ea-8900-1f9ce7d5650d --file-prefix=vf0 -- -i
+
 Also, to use VFIO, both kernel and BIOS must support and be configured to use IO virtualization (such as Intel® VT-d).
 
 .. note::
 
-    ``vfio-pci`` module doesn't support the creation of virtual functions.
+    ``vfio-pci`` module doesn't support the creation of virtual functions before Linux version 5.7.
 
 For proper operation of VFIO when running DPDK applications as a non-privileged user, correct permissions should also be set up.
 This can be done by using the DPDK setup script (called dpdk-setup.sh and located in the usertools directory).
doc/guides/linux_gsg/linux_eal_parameters.rst +4 -0
index b2cc60e44..bd3977cb3 100644
@@ -40,6 +40,10 @@ Device-related options
 
     Use specified interrupt mode for devices bound to VFIO kernel driver.
 
+*   ``--vfio-vf-token <uuid>``
+
+    Use specified VF token for devices bound to VFIO kernel driver.
+
 Multiprocessing-related options
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
doc/guides/rel_notes/release_20_08.rst +6 -0
index 5cbc4ce14..333f315ca 100644
@@ -107,6 +107,12 @@ New Features
   * Dump ``rte_flow`` memory consumption.
   * Measure packet per second forwarding.
 
+* **Added the support for vfio-pci new VF token interface.**
+
+  From Linux 5.7, vfio-pci supports to bind both SR-IOV PF and the created VFs,
+  it uses a shared VF token (UUID) to represent the collaboration between PF
+  and VFs. Update DPDK PCI driver to gain the access to the PF and VFs devices
+  by appending the VF token parameter.
 
 Removed Items
 -------------
lib/librte_eal/common/eal_common_options.c +3 -0
index 24b223ebf..15b47349d 100644
@@ -94,6 +94,7 @@ eal_long_options[] = {
 	{OPT_SYSLOG,            1, NULL, OPT_SYSLOG_NUM           },
 	{OPT_VDEV,              1, NULL, OPT_VDEV_NUM             },
 	{OPT_VFIO_INTR,         1, NULL, OPT_VFIO_INTR_NUM        },
+	{OPT_VFIO_VF_TOKEN,     1, NULL, OPT_VFIO_VF_TOKEN_NUM    },
 	{OPT_VMWARE_TSC_MAP,    0, NULL, OPT_VMWARE_TSC_MAP_NUM   },
 	{OPT_LEGACY_MEM,        0, NULL, OPT_LEGACY_MEM_NUM       },
 	{OPT_SINGLE_FILE_SEGMENTS, 0, NULL, OPT_SINGLE_FILE_SEGMENTS_NUM},
@@ -327,6 +328,8 @@ eal_reset_internal_config(struct internal_config *internal_cfg)
 
 	/* if set to NONE, interrupt mode is determined automatically */
 	internal_cfg->vfio_intr_mode = RTE_INTR_MODE_NONE;
+	memset(internal_cfg->vfio_vf_token, 0,
+			sizeof(internal_cfg->vfio_vf_token));
 
 #ifdef RTE_LIBEAL_USE_HPET
 	internal_cfg->no_hpet = 0;
lib/librte_eal/common/eal_internal_cfg.h +2 -0
index 2aaa167d4..13f93388a 100644
@@ -72,6 +72,8 @@ struct internal_config {
 	volatile int syslog_facility;	  /**< facility passed to openlog() */
 	/** default interrupt mode for VFIO */
 	volatile enum rte_intr_mode vfio_intr_mode;
+	/** the shared VF token for VFIO-PCI bound PF and VFs devices */
+	rte_uuid_t vfio_vf_token;
 	char *hugefile_prefix;      /**< the base filename of hugetlbfs files */
 	char *hugepage_dir;         /**< specific hugetlbfs directory to use */
 	char *user_mbuf_pool_ops_name;
lib/librte_eal/common/eal_options.h +2 -0
index 18e6da9ab..89769d48b 100644
@@ -69,6 +69,8 @@ enum {
 	OPT_VDEV_NUM,
 #define OPT_VFIO_INTR         "vfio-intr"
 	OPT_VFIO_INTR_NUM,
+#define OPT_VFIO_VF_TOKEN     "vfio-vf-token"
+	OPT_VFIO_VF_TOKEN_NUM,
 #define OPT_VMWARE_TSC_MAP    "vmware-tsc-map"
 	OPT_VMWARE_TSC_MAP_NUM,
 #define OPT_LEGACY_MEM    "legacy-mem"
lib/librte_eal/freebsd/eal.c +5 -0
index 8c75cba79..040b1b7ee 100644
@@ -966,6 +966,11 @@ rte_eal_vfio_intr_mode(void)
 	return RTE_INTR_MODE_NONE;
 }
 
+void
+rte_eal_vfio_get_vf_token(__rte_unused rte_uuid_t vf_token)
+{
+}
+
 int rte_vfio_setup_device(__rte_unused const char *sysfs_base,
 		      __rte_unused const char *dev_addr,
 		      __rte_unused int *vfio_dev_fd,
lib/librte_eal/include/rte_eal.h +14 -0
index 2f9ed298d..6c70632cc 100644
@@ -19,6 +19,7 @@
 #include <rte_compat.h>
 #include <rte_per_lcore.h>
 #include <rte_bus.h>
+#include <rte_uuid.h>
 
 #include <rte_pci_dev_feature_defs.h>
 
@@ -438,6 +439,19 @@ int rte_eal_create_uio_dev(void);
  */
 enum rte_intr_mode rte_eal_vfio_intr_mode(void);
 
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice
+ *
+ * Copy the user-configured vfio VF token.
+ *
+ * @param vf_token
+ *   vfio VF token configured with the command line is copied
+ *   into this parameter, zero uuid by default.
+ */
+__rte_experimental
+void rte_eal_vfio_get_vf_token(rte_uuid_t vf_token);
+
 /**
  * A wrap API for syscall gettid.
  *
lib/librte_eal/linux/eal.c +33 -0
index 3b56d14da..d6b8d8be7 100644
@@ -540,6 +540,7 @@ eal_usage(const char *prgname)
 	       "  --"OPT_FILE_PREFIX"       Prefix for hugepage filenames\n"
 	       "  --"OPT_CREATE_UIO_DEV"    Create /dev/uioX (usually done by hotplug)\n"
 	       "  --"OPT_VFIO_INTR"         Interrupt mode for VFIO (legacy|msi|msix)\n"
+	       "  --"OPT_VFIO_VF_TOKEN"     VF token (UUID) shared between SR-IOV PF and VFs\n"
 	       "  --"OPT_LEGACY_MEM"        Legacy memory mode (no dynamic allocation, contiguous segments)\n"
 	       "  --"OPT_SINGLE_FILE_SEGMENTS" Put all hugepage memory in single files\n"
 	       "  --"OPT_MATCH_ALLOCATIONS" Free hugepages exactly as allocated\n"
@@ -620,6 +621,20 @@ eal_parse_vfio_intr(const char *mode)
 	return -1;
 }
 
+static int
+eal_parse_vfio_vf_token(const char *vf_token)
+{
+	struct internal_config *cfg = eal_get_internal_configuration();
+	rte_uuid_t uuid;
+
+	if (!rte_uuid_parse(vf_token, uuid)) {
+		rte_uuid_copy(cfg->vfio_vf_token, uuid);
+		return 0;
+	}
+
+	return -1;
+}
+
 /* Parse the arguments for --log-level only */
 static void
 eal_log_level_parse(int argc, char **argv)
@@ -762,6 +777,16 @@ eal_parse_args(int argc, char **argv)
 			}
 			break;
 
+		case OPT_VFIO_VF_TOKEN_NUM:
+			if (eal_parse_vfio_vf_token(optarg) < 0) {
+				RTE_LOG(ERR, EAL, "invalid parameters for --"
+						OPT_VFIO_VF_TOKEN "\n");
+				eal_usage(prgname);
+				ret = -1;
+				goto out;
+			}
+			break;
+
 		case OPT_CREATE_UIO_DEV_NUM:
 			internal_conf->create_uio_dev = 1;
 			break;
@@ -1342,6 +1367,14 @@ rte_eal_vfio_intr_mode(void)
 	return internal_conf->vfio_intr_mode;
 }
 
+void
+rte_eal_vfio_get_vf_token(rte_uuid_t vf_token)
+{
+	struct internal_config *cfg = eal_get_internal_configuration();
+
+	rte_uuid_copy(vf_token, cfg->vfio_vf_token);
+}
+
 int
 rte_eal_check_module(const char *module_name)
 {
lib/librte_eal/linux/eal_vfio.c +19 -0
index abb12a354..d4470a34d 100644
@@ -714,6 +714,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 	int vfio_container_fd;
 	int vfio_group_fd;
 	int iommu_group_num;
+	rte_uuid_t vf_token;
 	int i, ret;
 	const struct internal_config *internal_conf =
 		eal_get_internal_configuration();
@@ -899,6 +900,23 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 				t->type_id, t->name);
 	}
 
+	rte_eal_vfio_get_vf_token(vf_token);
+
+	/* get a file descriptor for the device with VF token firstly */
+	if (!rte_uuid_is_null(vf_token)) {
+		char vf_token_str[RTE_UUID_STRLEN];
+		char dev[PATH_MAX];
+
+		rte_uuid_unparse(vf_token, vf_token_str, sizeof(vf_token_str));
+		snprintf(dev, sizeof(dev),
+			 "%s vf_token=%s", dev_addr, vf_token_str);
+
+		*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD,
+				     dev);
+		if (*vfio_dev_fd >= 0)
+			goto dev_get_info;
+	}
+
 	/* get a file descriptor for the device */
 	*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev_addr);
 	if (*vfio_dev_fd < 0) {
@@ -914,6 +932,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
 	}
 
 	/* test and setup the device */
+dev_get_info:
 	ret = ioctl(*vfio_dev_fd, VFIO_DEVICE_GET_INFO, device_info);
 	if (ret) {
 		RTE_LOG(ERR, EAL, "  %s cannot get device info, "
lib/librte_eal/rte_eal_version.map +3 -0
index 196eef5af..7923c62f2 100644
@@ -386,6 +386,9 @@ EXPERIMENTAL {
 	rte_trace_point_lookup;
 	rte_trace_regexp;
 	rte_trace_save;
+
+	# added in 20.08
+	rte_eal_vfio_get_vf_token;
 };
 
 INTERNAL {
Signature · 1 line
2.27.0
#91 lore raw

Re: [dpdk-dev] [PATCH v17 2/2] eal: support for VFIO-PCI VF token

Quoted 7 lines · Haiyue Wang

The kernel module vfio-pci introduces the VF token to enable SR-IOV support since 5.7.

The VF token can be set by a vfio-pci based PF driver and must be known by the vfio-pci based VF driver in order to gain access to the device.

Quoted 8 lines

Since the vfio-pci module uses the VF token as internal data to provide the collaboration between SR-IOV PF and VFs, so DPDK can use the same VF token for all PF devices by specifying the related EAL option.

Series applied, thanks Haiyue.

Signature · 1 line
David Marchand