vfio: set vf token and gain vf device access
RFC 1/1 vfio: set vf token and gain vf device access
From: Vamsi Attunuru <[email protected]>
vfio-pci driver enables virtual function access from the DPDK applications when those vf device's physical function is also bound to vfio driver.
Patch adds the required configuration and checks to enable DPDK applications to access both pf and it's vf devices through vfio-pci driver.
See background on vf token scheme in linux vfio driver.
http://patches.dpdk.org/cover/65915/
When a physical function is enabled with non-zero virtual functions, patch sets the UUID using VFIO_DEVICE_FEATURE ioctl from physical function's file descriptor. Same UUID is used to gain the access for the virtual functions on those physical function.
Following changes required on top of this DPDK patch
* Kernel version check for VFIO_DEVICE_FEATURE ioctl
* Use uuid gen API to generate UUID.
lib/librte_eal/linux/eal/eal_vfio.c +150 -1
@@ -12,6 +12,7 @@#include <rte_log.h> #include <rte_memory.h> #include <rte_eal_memconfig.h> +#include <rte_uuid.h> #include <rte_vfio.h> #include "eal_filesystem.h"
@@ -50,6 +51,9 @@ struct vfio_config {struct user_mem_maps mem_maps; }; +rte_uuid_t uuid_token =RTE_UUID_INIT (0xf8615163 ,0xdf3e ,0x46c5 , +0x913f ,0xf2d2f965ed0eULL ); +/* per-process VFIO config */ static struct vfio_config vfio_cfgs[VFIO_MAX_CONTAINERS ];static struct vfio_config *default_vfio_cfg = &vfio_cfgs[0 ];
@@ -657,6 +661,102 @@ rte_vfio_clear_group(int vfio_group_fd)return 0 ; } +static bool +rte_vfio_dev_is_physfn (const char *sysfs_base,const char *dev_addr) +{ +char linkname[PATH_MAX ]; +char filename[PATH_MAX ]; +int ret; + +memset (linkname,0 ,sizeof (linkname)); +memset (filename,0 ,sizeof (filename)); + +/* check if physfn directory exist for this device */ +snprintf (linkname,sizeof (linkname), +"%s/%s/physfn" , sysfs_base, dev_addr); + + ret =readlink (linkname, filename,sizeof (filename)); + +/* For PFs, physfn directory does not exist */ +if (ret <0 ) +return true; + +return false; +} + +static int +is_vf_token_required (const char *sysfs_base,const char *dev_addr) +{ +char *tok[16 ], *physfn, *physfn_drv; +char linkname[PATH_MAX ]; +char filename[PATH_MAX ]; +int ret; + +memset (linkname,0 ,sizeof (linkname)); +memset (filename,0 ,sizeof (filename)); + +snprintf (linkname,sizeof (linkname), +"%s/%s/physfn" , sysfs_base, dev_addr); + + ret =readlink (linkname, filename,sizeof (filename)); +if (ret <0 ) +return -1 ; + + ret =rte_strsplit (filename,sizeof (filename), + tok,RTE_DIM (tok),'/' ); +if (ret <=0 ) { +RTE_LOG (ERR ,EAL ," %s cannot get it's physfn\n" , dev_addr); +return -1 ; + } + + physfn = tok[ret -1 ]; + +snprintf (linkname,sizeof (linkname), +"/sys/bus/pci/devices/%s/driver" , physfn); + ret =readlink (linkname, filename,sizeof (filename)); +if (ret <0 ) +return -1 ; + + ret =rte_strsplit (filename,sizeof (filename), + tok,RTE_DIM (tok),'/' ); +if (ret <=0 ) { +RTE_LOG (ERR ,EAL ," %s cannot get it's physfn driver info\n" , + dev_addr); +return -1 ; + } + + physfn_drv = tok[ret -1 ]; + +if (strncmp (physfn_drv,"vfio-pci" ,sizeof ("vfio-pci" ))) +return 1 ; + +/* physfn is bound to vfio-pci */ +return 0 ; +} + +static bool +rte_vfio_dev_has_nonzero_numvfs (const char *sysfs_base,const char *dev_addr) +{ +char linkname[PATH_MAX ]; +unsigned long num_vfs; +int ret; + +if (!rte_vfio_dev_is_physfn (sysfs_base, dev_addr)) +return false; + +memset (linkname,0 ,sizeof (linkname)); + +snprintf (linkname,sizeof (linkname), +"%s/%s/sriov_numvfs" , sysfs_base, dev_addr); + + ret =eal_parse_sysfs_value (linkname, &num_vfs); + +if ((ret <0 ) || (num_vfs ==0 )) +return false; + +return true; +} +int rte_vfio_setup_device (const char *sysfs_base,const char *dev_addr,int *vfio_dev_fd,struct vfio_device_info *device_info)
@@ -669,6 +769,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,int vfio_container_fd;int vfio_group_fd;int iommu_group_num; +char dev[PATH_MAX ];int i, ret;/* get group number */
@@ -683,6 +784,29 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,if (ret <0 )return -1 ; +snprintf (dev,sizeof (dev),"%s" , dev_addr); + +if (!rte_vfio_dev_is_physfn (sysfs_base, dev_addr)) { +char vf_token[PATH_MAX ]; +/* +* Check if vf_token is required or not, +* vf_token is required when the VF's physfn is +* binded with vfio-pci driver +*/ + ret =is_vf_token_required (sysfs_base, dev_addr); +/* if negative, something failed */ +if (ret <0 ) +return -1 ; + +if (ret ==0 ) { +/* vf_token required to open device file descriptor */ +rte_uuid_unparse (uuid_token, + vf_token,sizeof (vf_token)); +snprintf (dev,sizeof (dev), +"%s vf_token=%s" , dev_addr, vf_token); + } + } +/* get the actual group fd */ vfio_group_fd =rte_vfio_get_group_fd (iommu_group_num);if (vfio_group_fd <0 )
@@ -853,7 +977,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr, }/* get a file descriptor for the device */ - *vfio_dev_fd =ioctl (vfio_group_fd,VFIO_GROUP_GET_DEVICE_FD , dev_addr); + *vfio_dev_fd =ioctl (vfio_group_fd,VFIO_GROUP_GET_DEVICE_FD , dev);if (*vfio_dev_fd <0 ) {/* if we cannot get a device fd, this implies a problem with * the VFIO group or the container not having IOMMU configured.
@@ -877,6 +1001,31 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,rte_vfio_clear_group (vfio_group_fd);return -1 ; } + +if (rte_vfio_dev_has_nonzero_numvfs (sysfs_base, dev_addr)) { +#define VF_TOKEN (sizeof(struct vfio_device_feature) + sizeof(rte_uuid_t)) + +struct vfio_device_feature *vf_token; +uint8_t local[VF_TOKEN ]; + +memset (local,0 ,VF_TOKEN ); + vf_token = (struct vfio_device_feature *)local; + vf_token->argsz =VF_TOKEN ; + vf_token->flags =VFIO_DEVICE_FEATURE_SET | +VFIO_DEVICE_FEATURE_PCI_VF_TOKEN ; + +memcpy (local +sizeof (struct vfio_device_feature ), + &uuid_token,sizeof (uuid_token)); + + ret =ioctl (*vfio_dev_fd,VFIO_DEVICE_FEATURE , vf_token); +if (ret) { +RTE_LOG (ERR ,EAL ," Failed to set UUID on %s " +"error %i (%s)\n" , dev_addr, errno, +strerror (errno)); +return -1 ; + } + } +vfio_group_device_get (vfio_group_fd);return 0 ;
Signature · 1 line
2.8.4
Snipped
lib/librte_eal/linux/eal/eal_vfio.c
Quoted 5 lines
lib/librte_eal/linux/eal/eal_vfio.c +0
-0
DPDK supports freebsd where /dev/pci can be probed like Linux /sys/bus/pci. Will you be adding the functions in lib/librte_eal/freebsd/eal/ folder or implement as NOT supported?
What about window lib/librte_eal/windows/eal/?
snipped
lib/librte_eal/linux/eal/eal_vfio.c
Quoted 2 lines
lib/librte_eal/linux/eal/eal_vfio.c +2
-0+snprintf (linkname,sizeof (linkname), +"%s/%s/physfn" , sysfs_base, dev_addr);
Would you like to check the pointer sysfs_base or dev_addr?
Quoted 3 lines
int
rte_vfio_setup_device(const char*sysfs_base, const char*dev_addr,
int*vfio_dev_fd, structvfio_device_info*device_info) @@ -
Snipped
lib/librte_eal/linux/eal/eal_vfio.c rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr, · line 796
Quoted 1 line
lib/librte_eal/linux/eal/eal_vfio.c +1
-0+ ret =is_vf_token_required (sysfs_base, dev_addr);
The return value from the above function is 0 or -1.
lib/librte_eal/linux/eal/eal_vfio.c rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr, · line 801
Quoted 5 lines
lib/librte_eal/linux/eal/eal_vfio.c +5
-0+/* if negative, something failed */ +if (ret <0 ) +return -1 ; + +if (ret ==0 ) {
Will it be ok to skip this check?
lib/librte_eal/linux/eal/eal_vfio.c rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr, · line 807
Quoted 6 lines
lib/librte_eal/linux/eal/eal_vfio.c +6
-0+/* vf_token required to open device file descriptor */ +rte_uuid_unparse (uuid_token, + vf_token,sizeof (vf_token)); +snprintf (dev,sizeof (dev), +"%s vf_token=%s" , dev_addr, vf_token); + }
snipped
general comment can we use EAL LOG for info, warn, error?
Hi Vipin,
Please see inline.
Quoted 21 lines
-----Original Message-----
From: dev <[email protected]> On Behalf Of Varghese, Vipin
Sent: Wednesday, April 1, 2020 8:45 AM
To: Vamsi Krishna Attunuru <[email protected]>; [email protected]
Cc: Jerin Jacob Kollanukkaran <[email protected]>; [email protected]; [email protected]; [email protected]
Subject: Re: [dpdk-dev] [RFC v1 1/1] vfio: set vf token and gain vf device accessSnipped
lib/librte_eal/linux/eal/eal_vfio.c
Quoted 5 lines
lib/librte_eal/linux/eal/eal_vfio.c +0
-0DPDK supports freebsd where
/dev/pcican be probed like Linux/sys/bus/pci. Will you be adding the functions inlib/librte_eal/freebsd/eal/folder or implement asNOT supported?
Quoted 1 line
What about window
lib/librte_eal/windows/eal/?
If other kernel (freebsd & windows) drivers also support this vf token scheme, functions can be added in respective eal implementations.
Quoted 6 lines
snipped
lib/librte_eal/linux/eal/eal_vfio.c
Quoted 2 lines
lib/librte_eal/linux/eal/eal_vfio.c +2
-0+snprintf (linkname,sizeof (linkname), +"%s/%s/physfn" , sysfs_base, dev_addr);Would you like to check the pointer
sysfs_baseordev_addr?
Physfn pointer is being checked here.
Quoted 10 lines
Quoted 4 lines
int
rte_vfio_setup_device(const char*sysfs_base, const char*dev_addr,
int*vfio_dev_fd, structvfio_device_info*device_info) @@ -Snipped
lib/librte_eal/linux/eal/eal_vfio.c rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr, · line 796
Quoted 1 line
lib/librte_eal/linux/eal/eal_vfio.c +1
-0+ ret =is_vf_token_required (sysfs_base, dev_addr);The return value from the above function is 0 or -1.
Function also returns 1.
Quoted 8 lines
lib/librte_eal/linux/eal/eal_vfio.c rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr, · line 801
Quoted 6 lines
lib/librte_eal/linux/eal/eal_vfio.c +5
-0+/* if negative, something failed */ +if (ret <0 ) +return -1 ; + +if (ret ==0 ) {Will it be ok to skip this check?
No, it's a valid case.
Quoted 11 lines
lib/librte_eal/linux/eal/eal_vfio.c rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr, · line 807
Quoted 7 lines
lib/librte_eal/linux/eal/eal_vfio.c +6
-0+/* vf_token required to open device file descriptor */ +rte_uuid_unparse (uuid_token, + vf_token,sizeof (vf_token)); +snprintf (dev,sizeof (dev), +"%s vf_token=%s" , dev_addr, vf_token); + }snipped
general comment
can we use EAL LOG for info, warn, error?
Hi A Vamsi,
Quoted 122 lines
-----Original Message-----
From: dev <[email protected]> On Behalf Of [email protected]
Sent: Thursday, March 5, 2020 12:33
To: [email protected]
Cc: [email protected]; [email protected]; [email protected]; [email protected]; Vamsi Attunuru <[email protected]>
Subject: [dpdk-dev] [RFC v1 1/1] vfio: set vf token and gain vf device accessFrom: Vamsi Attunuru <[email protected]>
vfio-pci driver enables virtual function access from the DPDK applications when those vf device's physical function is also bound to vfio driver.
Patch adds the required configuration and checks to enable DPDK applications to access both pf and it's vf devices through vfio-pci driver.
See background on vf token scheme in linux vfio driver.
http://patches.dpdk.org/cover/65915/When a physical function is enabled with non-zero virtual functions, patch sets the UUID using
VFIO_DEVICE_FEATUREioctl from physical function's file descriptor. Same UUID is used to gain the access for the virtual functions on those physical function.Following changes required on top of this DPDK patch
* Kernel version check forVFIO_DEVICE_FEATUREioctl
* Use uuid gen API to generate UUID.Signed-off-by Vamsi Attunuru <[email protected]>lib/librte_eal/linux/eal/eal_vfio.c +67
-0@@ -12,6 +12,7 @@#include <rte_log.h> #include <rte_memory.h> #include <rte_eal_memconfig.h> +#include <rte_uuid.h> #include <rte_vfio.h> #include "eal_filesystem.h" @@ -50,6 +51,9 @@ struct vfio_config {struct user_mem_maps mem_maps; }; +rte_uuid_t uuid_token =RTE_UUID_INIT (0xf8615163 ,0xdf3e ,0x46c5 , +0x913f ,0xf2d2f965ed0eULL ); +/* per-process VFIO config */ static struct vfio_config vfio_cfgs[VFIO_MAX_CONTAINERS ];static struct vfio_config *default_vfio_cfg = &vfio_cfgs[0 ];@@ -657,6 +661,102 @@ rte_vfio_clear_group(int vfio_group_fd)return 0 ; } +static bool +rte_vfio_dev_is_physfn (const char *sysfs_base,const char *dev_addr) +{ +char linkname[PATH_MAX ]; +char filename[PATH_MAX ]; +int ret; + +memset (linkname,0 ,sizeof (linkname)); +memset (filename,0 ,sizeof (filename)); + +/* check if physfn directory exist for this device */ +snprintf (linkname,sizeof (linkname), +"%s/%s/physfn" , sysfs_base, dev_addr); + + ret =readlink (linkname, filename,sizeof (filename)); + +/* For PFs, physfn directory does not exist */ +if (ret <0 ) +return true; + +return false; +} + +static int +is_vf_token_required (const char *sysfs_base,const char *dev_addr) +{ +char *tok[16 ], *physfn, *physfn_drv; +char linkname[PATH_MAX ]; +char filename[PATH_MAX ]; +int ret; + +memset (linkname,0 ,sizeof (linkname)); +memset (filename,0 ,sizeof (filename)); + +snprintf (linkname,sizeof (linkname), +"%s/%s/physfn" , sysfs_base, dev_addr); + + ret =readlink (linkname, filename,sizeof (filename)); +if (ret <0 ) +return -1 ; + + ret =rte_strsplit (filename,sizeof (filename), + tok,RTE_DIM (tok),'/' ); +if (ret <=0 ) { +RTE_LOG (ERR ,EAL ," %s cannot get it's physfn\n" , dev_addr); +return -1 ; + } + + physfn = tok[ret -1 ]; + +snprintf (linkname,sizeof (linkname), +"/sys/bus/pci/devices/%s/driver" , physfn); + ret =readlink (linkname, filename,sizeof (filename)); +if (ret <0 ) +return -1 ; + + ret =rte_strsplit (filename,sizeof (filename), + tok,RTE_DIM (tok),'/' ); +if (ret <=0 ) { +RTE_LOG (ERR ,EAL ," %s cannot get it's physfn driver info\n" , + dev_addr); +return -1 ; + }
lib/librte_eal/linux/eal/eal_vfio.c rte_vfio_clear_group(int vfio_group_fd) · line 736
Quoted 10 lines
lib/librte_eal/linux/eal/eal_vfio.c +10
-0+ + physfn_drv = tok[ret -1 ]; + +if (strncmp (physfn_drv,"vfio-pci" ,sizeof ("vfio-pci" ))) +return 1 ; + +/* physfn is bound to vfio-pci */ +return 0 ; +} +
Based on Alex's vfio for qemu patch (https://lore.kernel.org/lkml/[email protected]/),
and understand the SR-IOV design by reading your RFC, I scratched a simple design.
1. ./usertools/dpdk-devbind.py -b vfio-pci 0000:87:00.0
2. echo 2 > /sys/bus/pci/devices/0000:87:00.0/sriov_numvfs
3. ./x86_64-native-linux-gcc/app/testpmd -l 22-25 -n 4 -w 87:00.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 --file-prefix=pf -- -i
If no token in VF:
./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 -w 87:02.0 --file-prefix=vf1 -- -i
Kernel error: vfio-pci 0000:87:02.0: VF token required to access device
./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 -w 87:02.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 --file-prefix=vf1 -- -i
static int vfio_pci_vf_token_arg_handler(__rte_unused const char *key,
const char *value, void *opaque)
{
if (rte_uuid_parse(value, opaque)) {
RTE_LOG(ERR, EAL,
"The VF token is not a valid uuid : %s\n", value);
return -1;
}
return 0;
}static intvfio_pci_vf_token_arg(struct rte_devargs *devargs, rte_uuid_t uu)
{
const char *key = "vf_token";
struct rte_kvargs *kvlist;
int ret = 0;
if (devargs == NULL)
return 0;
kvlist = rte_kvargs_parse(devargs->args, NULL);
if (kvlist == NULL)
return 0;
if (!rte_kvargs_count(kvlist, key))
goto exit;
if (rte_kvargs_process(kvlist, key,
vfio_pci_vf_token_arg_handler, uu) < 0)
goto exit;
ret = 1;exit:
rte_kvargs_free(kvlist); return ret; }
static int
pci_vfio_map_resource_primary(struct rte_pci_device *dev)
{
struct vfio_device_info device_info = { .argsz = sizeof(device_info) };
+ rte_uuid_t vf_token = RTE_UUID_INIT(0, 0, 0, 0, 0ULL);
char pci_addr[PATH_MAX] = {0};
int vfio_dev_fd;
struct rte_pci_addr *loc = &dev->addr;
@@ -668,8 +712,9 @@ pci_vfio_map_resource_primary(struct rte_pci_device *dev)
snprintf(pci_addr, sizeof(pci_addr), PCI_PRI_FMT,
loc->domain, loc->bus, loc->devid, loc->function);
+ vfio_pci_vf_token_arg(dev->device.devargs, vf_token);
ret = rte_vfio_setup_device(rte_pci_get_sysfs_path(), pci_addr,
- &vfio_dev_fd, &device_info);
+ &vfio_dev_fd, &device_info, vf_token);
if (ret)
return ret;
int
rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
- int *vfio_dev_fd, struct vfio_device_info *device_info)
+ int *vfio_dev_fd, struct vfio_device_info *device_info,
+ rte_uuid_t vf_token)
{
struct vfio_group_status group_status = {
.argsz = sizeof(group_status)
@@ -712,6 +713,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
int vfio_container_fd;
int vfio_group_fd;
int iommu_group_num;
+ char dev[PATH_MAX];
int i, ret;
/* get group number */
@@ -895,8 +897,19 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
t->type_id, t->name);
}
+ if (!rte_uuid_is_null(vf_token)) {
+ char vf_token_str[PATH_MAX];
+
+ rte_uuid_unparse(vf_token, vf_token_str, sizeof(vf_token_str));
+ snprintf(dev, sizeof(dev),
+ "%s vf_token=%s", dev_addr, vf_token_str);
+ } else {
+ snprintf(dev, sizeof(dev),
+ "%s", dev_addr);
+ }
+
/* get a file descriptor for the device */
- *vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev_addr);
+ *vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev);
Quoted 107 lines
lib/librte_eal/linux/eal/eal_vfio.c +73
-2+static bool +rte_vfio_dev_has_nonzero_numvfs (const char *sysfs_base,const char *dev_addr) +{ +char linkname[PATH_MAX ]; +unsigned long num_vfs; +int ret; + +if (!rte_vfio_dev_is_physfn (sysfs_base, dev_addr)) +return false; + +memset (linkname,0 ,sizeof (linkname)); + +snprintf (linkname,sizeof (linkname), +"%s/%s/sriov_numvfs" , sysfs_base, dev_addr); + + ret =eal_parse_sysfs_value (linkname, &num_vfs); + +if ((ret <0 ) || (num_vfs ==0 )) +return false; + +return true; +} +int rte_vfio_setup_device (const char *sysfs_base,const char *dev_addr,int *vfio_dev_fd,struct vfio_device_info *device_info)@@ -669,6 +769,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,int vfio_container_fd;int vfio_group_fd;int iommu_group_num; +char dev[PATH_MAX ];int i, ret;/* get group number */ @@ -683,6 +784,29 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,if (ret <0 )return -1 ; +snprintf (dev,sizeof (dev),"%s" , dev_addr); + +if (!rte_vfio_dev_is_physfn (sysfs_base, dev_addr)) { +char vf_token[PATH_MAX ]; +/* +* Check if vf_token is required or not, +* vf_token is required when the VF's physfn is +* binded with vfio-pci driver +*/ + ret =is_vf_token_required (sysfs_base, dev_addr); +/* if negative, something failed */ +if (ret <0 ) +return -1 ; + +if (ret ==0 ) { +/* vf_token required to open device file descriptor */ +rte_uuid_unparse (uuid_token, + vf_token,sizeof (vf_token)); +snprintf (dev,sizeof (dev), +"%s vf_token=%s" , dev_addr, vf_token); + } + } +/* get the actual group fd */ vfio_group_fd =rte_vfio_get_group_fd (iommu_group_num);if (vfio_group_fd <0 )@@ -853,7 +977,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr, }/* get a file descriptor for the device */ - *vfio_dev_fd =ioctl (vfio_group_fd,VFIO_GROUP_GET_DEVICE_FD , dev_addr); + *vfio_dev_fd =ioctl (vfio_group_fd,VFIO_GROUP_GET_DEVICE_FD , dev);if (*vfio_dev_fd <0 ) {/* if we cannot get a device fd, this implies a problem with * the VFIO group or the container not having IOMMU configured. @@ -877,6 +1001,31 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,rte_vfio_clear_group (vfio_group_fd);return -1 ; } + +if (rte_vfio_dev_has_nonzero_numvfs (sysfs_base, dev_addr)) { +#define VF_TOKEN (sizeof(struct vfio_device_feature) + sizeof(rte_uuid_t)) + +struct vfio_device_feature *vf_token; +uint8_t local[VF_TOKEN ]; + +memset (local,0 ,VF_TOKEN ); + vf_token = (struct vfio_device_feature *)local; + vf_token->argsz =VF_TOKEN ; + vf_token->flags =VFIO_DEVICE_FEATURE_SET | +VFIO_DEVICE_FEATURE_PCI_VF_TOKEN ; + +memcpy (local +sizeof (struct vfio_device_feature ), + &uuid_token,sizeof (uuid_token)); + + ret =ioctl (*vfio_dev_fd,VFIO_DEVICE_FEATURE , vf_token); +if (ret) { +RTE_LOG (ERR ,EAL ," Failed to set UUID on %s " +"error %i (%s)\n" , dev_addr, errno, +strerror (errno)); +return -1 ; + } + } +vfio_group_device_get (vfio_group_fd);return 0 ; --2.8.4
Hi Wang,
Quoted 153 lines
-----Original Message-----
From: dev <[email protected]> On Behalf Of Wang, Haiyue
Sent: Thursday, April 2, 2020 6:13 PM
To: Vamsi Krishna Attunuru <[email protected]>; [email protected]
Cc: Jerin Jacob Kollanukkaran <[email protected]>; [email protected]; [email protected]; [email protected]
Subject: Re: [dpdk-dev] [RFC v1 1/1] vfio: set vf token and gain vf device accessHi A Vamsi,
Quoted 16 lines
-----Original Message-----
From: dev <[email protected]> On Behalf Of [email protected]
Sent: Thursday, March 5, 2020 12:33
To: [email protected]
Cc: [email protected]; [email protected]; [email protected]; [email protected]; Vamsi Attunuru <[email protected]>
Subject: [dpdk-dev] [RFC v1 1/1] vfio: set vf token and gain vf device accessFrom: Vamsi Attunuru <[email protected]>
vfio-pci driver enables virtual function access from the DPDK applications when those vf device's physical function is also bound to vfio driver.
Quoted 7 lines
Patch adds the required configuration and checks to enable DPDK applications to access both pf and it's vf devices through vfio-pci driver.
See background on vf token scheme in linux vfio driver.
https://urldefense.proofpoint.com/v2/url?u=http-3A__patches.dpdk.org_cover_65915_&d=DwIFAg&c=nKjWec2b6R0mOyPaz7xtfQ&r=2rpxxNF2qeP02g VZIWTVrW
Quoted 1 line
-6zNZz5-
uKt9pRqpR_M3U&m=1z1lwkWphoVghrYUNLI304mER44Vei152GYrXBwn4kk& s=
Quoted 102 lines
tvJ4Li8zpLMbp_p3vKQL7QV09DttNcl70YAr6BVHZm8&e=
When a physical function is enabled with non-zero virtual functions, patch sets the UUID using
VFIO_DEVICE_FEATUREioctl from physical function's file descriptor. Same UUID is used to gain the access for the virtual functions on those physical function.Following changes required on top of this DPDK patch
* Kernel version check forVFIO_DEVICE_FEATUREioctl
* Use uuid gen API to generate UUID.Signed-off-by Vamsi Attunuru <[email protected]>lib/librte_eal/linux/eal/eal_vfio.c +4
-0@@ -12,6 +12,7 @@#include <rte_log.h> #include <rte_memory.h> #include <rte_eal_memconfig.h> +#include <rte_uuid.h> #include <rte_vfio.h> #include "eal_filesystem.h" @@ -50,6 +51,9 @@ struct vfio_config {struct user_mem_maps mem_maps; }; +rte_uuid_t uuid_token =RTE_UUID_INIT (0xf8615163 ,0xdf3e ,0x46c5 , +0x913f ,0xf2d2f965ed0eULL ); +/* per-process VFIO config */ static struct vfio_config vfio_cfgs[VFIO_MAX_CONTAINERS]; staticstruct
vfio_config*default_vfio_cfg= &vfio_cfgs[0]; @@ -657,6 +661,102 @@rte_vfio_clear_group(int vfio_group_fd)lib/librte_eal/linux/eal/eal_vfio.c +62
-0return 0 ; } +static bool +rte_vfio_dev_is_physfn (const char *sysfs_base,const char *dev_addr) +{ +char linkname[PATH_MAX ]; +char filename[PATH_MAX ]; +int ret; + +memset (linkname,0 ,sizeof (linkname)); +memset (filename,0 ,sizeof (filename)); + +/* check if physfn directory exist for this device */ +snprintf (linkname,sizeof (linkname), +"%s/%s/physfn" , sysfs_base, dev_addr); + + ret =readlink (linkname, filename,sizeof (filename)); + +/* For PFs, physfn directory does not exist */ +if (ret <0 ) +return true; + +return false; +} + +static int +is_vf_token_required (const char *sysfs_base,const char *dev_addr) { +char *tok[16 ], *physfn, *physfn_drv; +char linkname[PATH_MAX ]; +char filename[PATH_MAX ]; +int ret; + +memset (linkname,0 ,sizeof (linkname)); +memset (filename,0 ,sizeof (filename)); + +snprintf (linkname,sizeof (linkname), +"%s/%s/physfn" , sysfs_base, dev_addr); + + ret =readlink (linkname, filename,sizeof (filename)); +if (ret <0 ) +return -1 ; + + ret =rte_strsplit (filename,sizeof (filename), + tok,RTE_DIM (tok),'/' ); +if (ret <=0 ) { +RTE_LOG (ERR ,EAL ," %s cannot get it's physfn\n" , dev_addr); +return -1 ; + } + + physfn = tok[ret -1 ]; + +snprintf (linkname,sizeof (linkname), +"/sys/bus/pci/devices/%s/driver" , physfn); + ret =readlink (linkname, filename,sizeof (filename)); +if (ret <0 ) +return -1 ; + + ret =rte_strsplit (filename,sizeof (filename), + tok,RTE_DIM (tok),'/' ); +if (ret <=0 ) { +RTE_LOG (ERR ,EAL ," %s cannot get it's physfn driver info\n" , + dev_addr); +return -1 ; + }lib/librte_eal/linux/eal/eal_vfio.c rte_vfio_clear_group(int vfio_group_fd) · line 736
Quoted 10 lines
lib/librte_eal/linux/eal/eal_vfio.c +10
-0+ + physfn_drv = tok[ret -1 ]; + +if (strncmp (physfn_drv,"vfio-pci" ,sizeof ("vfio-pci" ))) +return 1 ; + +/* physfn is bound to vfio-pci */ +return 0 ; +} +
Quoted 8 lines
Based on Alex's vfio for qemu patch
(https://urldefense.proofpoint.com/v2/url?u=https-
3A__lore.kernel.org_lkml_20200204161737.34696b91- 40w520.home_&d=DwIFAg&c=nKjWec2b6R0mOyPaz7xtfQ&r=2rpxxNF2qeP0 2gVZIWTVrW-6zNZz5- uKt9pRqpR_M3U&m=1z1lwkWphoVghrYUNLI304mER44Vei152GYrXBwn4kk& s=0pgRda84NrL2omZa0mn-ufSEzKksSi440O2AYImZLaI&e= ), and understand the SR-IOV design by reading your RFC, I scratched a simple design.
Thanks, looks good to me, I guess now it serves the QEMU based use cases as well.
Quoted 237 lines
1.
./usertools/dpdk-devbind.py-b vfio-pci 0000:87:00.0 2. echo 2 > /sys/bus/pci/devices/0000:87:00.0/sriov_numvfs
3. ./x86_64-native-linux-gcc/app/testpmd -l 22-25 -n 4 -w 87:00.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 --file-prefix=pf -- -iIf no token in VF:
./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 -w 87:02.0 --file- prefix=vf1 -- -i
Kernel error: vfio-pci 0000:87:02.0: VF token required to access device./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 -w 87:02.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 --file-prefix=vf1 -
- -istatic int vfio_pci_vf_token_arg_handler(__rte_unused const char *key,
const char *value, void *opaque) { if (rte_uuid_parse(value, opaque)) { RTE_LOG(ERR, EAL, "The VF token is not a valid uuid : %s\n", value); return -1; } return 0; }static int
vfio_pci_vf_token_arg(struct rte_devargs *devargs, rte_uuid_t uu) { const char *key = "vf_token"; struct rte_kvargs *kvlist; int ret = 0; if (devargs == NULL) return 0; kvlist = rte_kvargs_parse(devargs->args, NULL); if (kvlist == NULL) return 0; if (!rte_kvargs_count(kvlist, key)) goto exit; if (rte_kvargs_process(kvlist, key, vfio_pci_vf_token_arg_handler, uu) < 0) goto exit; ret = 1;exit:
rte_kvargs_free(kvlist); return ret; }static int
pci_vfio_map_resource_primary(struct rte_pci_device *dev){(quoted diff) +1
-0struct vfio_device_info device_info = { .argsz = sizeof(device_info) }; + rte_uuid_t vf_token = RTE_UUID_INIT(0, 0, 0, 0, 0ULL); char pci_addr[PATH_MAX] = {0}; int vfio_dev_fd; struct rte_pci_addr *loc = &dev->addr; @@ -668,8 +712,9 @@
pci_vfio_map_resource_primary(struct rte_pci_device *dev)(quoted diff) +2
-1snprintf(pci_addr, sizeof(pci_addr), PCI_PRI_FMT, loc->domain, loc->bus, loc->devid, loc->function); + vfio_pci_vf_token_arg(dev->device.devargs, vf_token); ret = rte_vfio_setup_device(rte_pci_get_sysfs_path(), pci_addr, - &vfio_dev_fd, &device_info); + &vfio_dev_fd, &device_info,
vf_token);(quoted diff) +0
-0if (ret) return ret;int
rte_vfio_setup_device(const char*sysfs_base, const char*dev_addr,lib/librte_eal/linux/eal/eal_vfio.c +3
-1- int *vfio_dev_fd, struct vfio_device_info *device_info) + int *vfio_dev_fd, struct vfio_device_info *device_info, + rte_uuid_t vf_token) { struct vfio_group_status group_status = { .argsz = sizeof(group_status)@@ -712,6 +713,7 @@ rte_vfio_setup_device(const char *sysfs_base, constchar *dev_addr, int vfio_container_fd;int vfio_group_fd;int iommu_group_num; +char dev[PATH_MAX ];int i, ret;lib/librte_eal/linux/eal/eal_vfio.c +12
-1/* get group number */ @@ -895,8 +897,19 @@ rte_vfio_setup_device(const char *sysfs_base,const char *dev_addr, t->type_id, t->name); } +if (!rte_uuid_is_null (vf_token)) { +char vf_token_str[PATH_MAX ]; + +rte_uuid_unparse (vf_token, vf_token_str,sizeof (vf_token_str)); +snprintf (dev,sizeof (dev), +"%s vf_token=%s" , dev_addr, vf_token_str); + }else { +snprintf (dev,sizeof (dev), +"%s" , dev_addr); + } +/* get a file descriptor for the device */ - *vfio_dev_fd =ioctl (vfio_group_fd,VFIO_GROUP_GET_DEVICE_FD , dev_addr); + *vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD,dev);
Quoted 16 lines
lib/librte_eal/linux/eal/eal_vfio.c +16
-0+static bool +rte_vfio_dev_has_nonzero_numvfs (const char *sysfs_base,const char +*dev_addr) { +char linkname[PATH_MAX ]; +unsigned long num_vfs; +int ret; + +if (!rte_vfio_dev_is_physfn (sysfs_base, dev_addr)) +return false; + +memset (linkname,0 ,sizeof (linkname)); + +snprintf (linkname,sizeof (linkname), +"%s/%s/sriov_numvfs" , sysfs_base, dev_addr); + + ret =eal_parse_sysfs_value (linkname, &num_vfs);lib/librte_eal/linux/eal/eal_vfio.c rte_vfio_clear_group(int vfio_group_fd)
Quoted 10 lines
lib/librte_eal/linux/eal/eal_vfio.c +7
-0+ +if ((ret <0 ) || (num_vfs ==0 )) +return false; + +return true; +} +int rte_vfio_setup_device (const char *sysfs_base,const char *dev_addr, int *vfio_dev_fd, struct vfio_device_info *device_info) @@ -669,6
Quoted 1 line
+769,7 @@ rte_vfio_setup_device(const char
*sysfs_base, const char
*dev_addr,lib/librte_eal/linux/eal/eal_vfio.c rte_vfio_setup_device(const char *sysfs_base,
Quoted 8 lines
lib/librte_eal/linux/eal/eal_vfio.c +1
-0int vfio_container_fd;int vfio_group_fd;int iommu_group_num; +char dev[PATH_MAX ];int i, ret;/* get group number */ @@ -683,6 +784,29 @@ rte_vfio_setup_device(const char *sysfs_base,const char
*dev_addr,lib/librte_eal/linux/eal/eal_vfio.c rte_vfio_setup_device(const char *sysfs_base,
Quoted 30 lines
lib/librte_eal/linux/eal/eal_vfio.c +23
-0if (ret <0 )return -1 ; +snprintf (dev,sizeof (dev),"%s" , dev_addr); + +if (!rte_vfio_dev_is_physfn (sysfs_base, dev_addr)) { +char vf_token[PATH_MAX ]; +/* +* Check if vf_token is required or not, +* vf_token is required when the VF's physfn is +* binded with vfio-pci driver +*/ + ret =is_vf_token_required (sysfs_base, dev_addr); +/* if negative, something failed */ +if (ret <0 ) +return -1 ; + +if (ret ==0 ) { +/* vf_token required to open device file descriptor */ +rte_uuid_unparse (uuid_token, + vf_token,sizeof (vf_token)); +snprintf (dev,sizeof (dev), +"%s vf_token=%s" , dev_addr, vf_token); + } + } +/* get the actual group fd */ vfio_group_fd =rte_vfio_get_group_fd (iommu_group_num);if (vfio_group_fd <0 )@@ -853,7 +977,7 @@ rte_vfio_setup_device(const char *sysfs_base,const char
*dev_addr,lib/librte_eal/linux/eal/eal_vfio.c rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
Quoted 4 lines
lib/librte_eal/linux/eal/eal_vfio.c +0
-1}/* get a file descriptor for the device */ - *vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD,
dev_addr);lib/librte_eal/linux/eal/eal_vfio.c rte_vfio_setup_device(const char *sysfs_base, · line 980
Quoted 1 line
lib/librte_eal/linux/eal/eal_vfio.c +1
-0+ *vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD,dev);
lib/librte_eal/linux/eal/eal_vfio.c rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
Quoted 3 lines
lib/librte_eal/linux/eal/eal_vfio.c +0
-0if (*vfio_dev_fd <0 ) {/* if we cannot get a device fd, this implies a problem with * the VFIO group or the container not having IOMMU configured.
lib/librte_eal/linux/eal/eal_vfio.c rte_vfio_setup_device(const char *sysfs_base,
Quoted 1 line
lib/librte_eal/linux/eal/eal_vfio.c +0
-0@@ -877,6 +1001,31 @@ rte_vfio_setup_device(const char *sysfs_base,const char
*dev_addr,lib/librte_eal/linux/eal/eal_vfio.c rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr, · line 1005
Quoted 5 lines
lib/librte_eal/linux/eal/eal_vfio.c +2
-0rte_vfio_clear_group (vfio_group_fd);return -1 ; } + +if (rte_vfio_dev_has_nonzero_numvfs (sysfs_base, dev_addr)) {#define
lib/librte_eal/linux/eal/eal_vfio.c rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr, · line 1016
Quoted 10 lines
lib/librte_eal/linux/eal/eal_vfio.c +10
-0+VF_TOKEN (sizeof(struct vfio_device_feature) + sizeof(rte_uuid_t)) + +struct vfio_device_feature *vf_token; +uint8_t local[VF_TOKEN ]; + +memset (local,0 ,VF_TOKEN ); + vf_token = (struct vfio_device_feature *)local; + vf_token->argsz =VF_TOKEN ; + vf_token->flags = VFIO_DEVICE_FEATURE_SET | +VFIO_DEVICE_FEATURE_PCI_VF_TOKEN;Quoted 18 lines
lib/librte_eal/linux/eal/eal_vfio.c +13
-1+ +memcpy (local +sizeof (struct vfio_device_feature ), + &uuid_token,sizeof (uuid_token)); + + ret =ioctl (*vfio_dev_fd,VFIO_DEVICE_FEATURE , vf_token); +if (ret) { +RTE_LOG (ERR ,EAL ," Failed to set UUID on %s " +"error %i (%s)\n" , dev_addr, errno, +strerror (errno)); +return -1 ; + } + } +vfio_group_device_get (vfio_group_fd);return 0 ; --2.8.4
Hi Vamsi,
Fixed one dev_args issue, and send the whole patch to express the whole design:
https://patchwork.dpdk.org/patch/68114/
BR,
Haiyue
Quoted 410 lines
-----Original Message-----
From: Vamsi Krishna Attunuru <[email protected]>
Sent: Friday, April 10, 2020 15:28
To: Wang, Haiyue <[email protected]>; [email protected]
Cc: Jerin Jacob Kollanukkaran <[email protected]>; [email protected]; [email protected]; [email protected]
Subject: RE: [dpdk-dev] [RFC v1 1/1] vfio: set vf token and gain vf device accessHi Wang,
Quoted 153 lines
-----Original Message-----
From: dev <[email protected]> On Behalf Of Wang, Haiyue
Sent: Thursday, April 2, 2020 6:13 PM
To: Vamsi Krishna Attunuru <[email protected]>; [email protected]
Cc: Jerin Jacob Kollanukkaran <[email protected]>; [email protected]; [email protected]; [email protected]
Subject: Re: [dpdk-dev] [RFC v1 1/1] vfio: set vf token and gain vf device accessHi A Vamsi,
Quoted 16 lines
-----Original Message-----
From: dev <[email protected]> On Behalf Of [email protected]
Sent: Thursday, March 5, 2020 12:33
To: [email protected]
Cc: [email protected]; [email protected]; [email protected]; [email protected]; Vamsi Attunuru <[email protected]>
Subject: [dpdk-dev] [RFC v1 1/1] vfio: set vf token and gain vf device accessFrom: Vamsi Attunuru <[email protected]>
vfio-pci driver enables virtual function access from the DPDK applications when those vf device's physical function is also bound to vfio driver.
Quoted 7 lines
Patch adds the required configuration and checks to enable DPDK applications to access both pf and it's vf devices through vfio-pci driver.
See background on vf token scheme in linux vfio driver.
https://urldefense.proofpoint.com/v2/url?u=http-3A__patches.dpdk.org_cover_65915_&d=DwIFAg&c=nKjWec2b6R0mOyPaz7xtfQ&r=2rpxxNF2qeP02g VZIWTVrW
Quoted 1 line
-6zNZz5-
uKt9pRqpR_M3U&m=1z1lwkWphoVghrYUNLI304mER44Vei152GYrXBwn4kk& s=
Quoted 102 lines
tvJ4Li8zpLMbp_p3vKQL7QV09DttNcl70YAr6BVHZm8&e=
When a physical function is enabled with non-zero virtual functions, patch sets the UUID using
VFIO_DEVICE_FEATUREioctl from physical function's file descriptor. Same UUID is used to gain the access for the virtual functions on those physical function.Following changes required on top of this DPDK patch
* Kernel version check forVFIO_DEVICE_FEATUREioctl
* Use uuid gen API to generate UUID.Signed-off-by Vamsi Attunuru <[email protected]>lib/librte_eal/linux/eal/eal_vfio.c +4
-0@@ -12,6 +12,7 @@#include <rte_log.h> #include <rte_memory.h> #include <rte_eal_memconfig.h> +#include <rte_uuid.h> #include <rte_vfio.h> #include "eal_filesystem.h" @@ -50,6 +51,9 @@ struct vfio_config {struct user_mem_maps mem_maps; }; +rte_uuid_t uuid_token =RTE_UUID_INIT (0xf8615163 ,0xdf3e ,0x46c5 , +0x913f ,0xf2d2f965ed0eULL ); +/* per-process VFIO config */ static struct vfio_config vfio_cfgs[VFIO_MAX_CONTAINERS]; staticstruct
vfio_config*default_vfio_cfg= &vfio_cfgs[0]; @@ -657,6 +661,102 @@rte_vfio_clear_group(int vfio_group_fd)lib/librte_eal/linux/eal/eal_vfio.c +62
-0return 0 ; } +static bool +rte_vfio_dev_is_physfn (const char *sysfs_base,const char *dev_addr) +{ +char linkname[PATH_MAX ]; +char filename[PATH_MAX ]; +int ret; + +memset (linkname,0 ,sizeof (linkname)); +memset (filename,0 ,sizeof (filename)); + +/* check if physfn directory exist for this device */ +snprintf (linkname,sizeof (linkname), +"%s/%s/physfn" , sysfs_base, dev_addr); + + ret =readlink (linkname, filename,sizeof (filename)); + +/* For PFs, physfn directory does not exist */ +if (ret <0 ) +return true; + +return false; +} + +static int +is_vf_token_required (const char *sysfs_base,const char *dev_addr) { +char *tok[16 ], *physfn, *physfn_drv; +char linkname[PATH_MAX ]; +char filename[PATH_MAX ]; +int ret; + +memset (linkname,0 ,sizeof (linkname)); +memset (filename,0 ,sizeof (filename)); + +snprintf (linkname,sizeof (linkname), +"%s/%s/physfn" , sysfs_base, dev_addr); + + ret =readlink (linkname, filename,sizeof (filename)); +if (ret <0 ) +return -1 ; + + ret =rte_strsplit (filename,sizeof (filename), + tok,RTE_DIM (tok),'/' ); +if (ret <=0 ) { +RTE_LOG (ERR ,EAL ," %s cannot get it's physfn\n" , dev_addr); +return -1 ; + } + + physfn = tok[ret -1 ]; + +snprintf (linkname,sizeof (linkname), +"/sys/bus/pci/devices/%s/driver" , physfn); + ret =readlink (linkname, filename,sizeof (filename)); +if (ret <0 ) +return -1 ; + + ret =rte_strsplit (filename,sizeof (filename), + tok,RTE_DIM (tok),'/' ); +if (ret <=0 ) { +RTE_LOG (ERR ,EAL ," %s cannot get it's physfn driver info\n" , + dev_addr); +return -1 ; + }lib/librte_eal/linux/eal/eal_vfio.c rte_vfio_clear_group(int vfio_group_fd) · line 736
Quoted 10 lines
lib/librte_eal/linux/eal/eal_vfio.c +10
-0+ + physfn_drv = tok[ret -1 ]; + +if (strncmp (physfn_drv,"vfio-pci" ,sizeof ("vfio-pci" ))) +return 1 ; + +/* physfn is bound to vfio-pci */ +return 0 ; +} +Quoted 8 lines
Based on Alex's vfio for qemu patch
(https://urldefense.proofpoint.com/v2/url?u=https-
3A__lore.kernel.org_lkml_20200204161737.34696b91- 40w520.home_&d=DwIFAg&c=nKjWec2b6R0mOyPaz7xtfQ&r=2rpxxNF2qeP0 2gVZIWTVrW-6zNZz5- uKt9pRqpR_M3U&m=1z1lwkWphoVghrYUNLI304mER44Vei152GYrXBwn4kk& s=0pgRda84NrL2omZa0mn-ufSEzKksSi440O2AYImZLaI&e= ), and understand the SR-IOV design by reading your RFC, I scratched a simple design.Thanks, looks good to me, I guess now it serves the QEMU based use cases as well.
Quoted 237 lines
1.
./usertools/dpdk-devbind.py-b vfio-pci 0000:87:00.0 2. echo 2 > /sys/bus/pci/devices/0000:87:00.0/sriov_numvfs
3. ./x86_64-native-linux-gcc/app/testpmd -l 22-25 -n 4 -w 87:00.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 --file-prefix=pf -- -iIf no token in VF:
./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 -w 87:02.0 --file- prefix=vf1 -- -i
Kernel error: vfio-pci 0000:87:02.0: VF token required to access device./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 -w 87:02.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 --file-prefix=vf1 -
- -istatic int vfio_pci_vf_token_arg_handler(__rte_unused const char *key,
const char *value, void *opaque) { if (rte_uuid_parse(value, opaque)) { RTE_LOG(ERR, EAL, "The VF token is not a valid uuid : %s\n", value); return -1; } return 0; }static int
vfio_pci_vf_token_arg(struct rte_devargs *devargs, rte_uuid_t uu) { const char *key = "vf_token"; struct rte_kvargs *kvlist; int ret = 0; if (devargs == NULL) return 0; kvlist = rte_kvargs_parse(devargs->args, NULL); if (kvlist == NULL) return 0; if (!rte_kvargs_count(kvlist, key)) goto exit; if (rte_kvargs_process(kvlist, key, vfio_pci_vf_token_arg_handler, uu) < 0) goto exit; ret = 1;exit:
rte_kvargs_free(kvlist); return ret; }static int
pci_vfio_map_resource_primary(struct rte_pci_device *dev){(quoted diff) +1
-0struct vfio_device_info device_info = { .argsz = sizeof(device_info) }; + rte_uuid_t vf_token = RTE_UUID_INIT(0, 0, 0, 0, 0ULL); char pci_addr[PATH_MAX] = {0}; int vfio_dev_fd; struct rte_pci_addr *loc = &dev->addr; @@ -668,8 +712,9 @@
pci_vfio_map_resource_primary(struct rte_pci_device *dev)(quoted diff) +2
-1snprintf(pci_addr, sizeof(pci_addr), PCI_PRI_FMT, loc->domain, loc->bus, loc->devid, loc->function); + vfio_pci_vf_token_arg(dev->device.devargs, vf_token); ret = rte_vfio_setup_device(rte_pci_get_sysfs_path(), pci_addr, - &vfio_dev_fd, &device_info); + &vfio_dev_fd, &device_info,
vf_token);(quoted diff) +0
-0if (ret) return ret;int
rte_vfio_setup_device(const char*sysfs_base, const char*dev_addr,lib/librte_eal/linux/eal/eal_vfio.c +3
-1- int *vfio_dev_fd, struct vfio_device_info *device_info) + int *vfio_dev_fd, struct vfio_device_info *device_info, + rte_uuid_t vf_token) { struct vfio_group_status group_status = { .argsz = sizeof(group_status)@@ -712,6 +713,7 @@ rte_vfio_setup_device(const char *sysfs_base, constchar *dev_addr, int vfio_container_fd;int vfio_group_fd;int iommu_group_num; +char dev[PATH_MAX ];int i, ret;lib/librte_eal/linux/eal/eal_vfio.c +12
-1/* get group number */ @@ -895,8 +897,19 @@ rte_vfio_setup_device(const char *sysfs_base,const char *dev_addr, t->type_id, t->name); } +if (!rte_uuid_is_null (vf_token)) { +char vf_token_str[PATH_MAX ]; + +rte_uuid_unparse (vf_token, vf_token_str,sizeof (vf_token_str)); +snprintf (dev,sizeof (dev), +"%s vf_token=%s" , dev_addr, vf_token_str); + }else { +snprintf (dev,sizeof (dev), +"%s" , dev_addr); + } +/* get a file descriptor for the device */ - *vfio_dev_fd =ioctl (vfio_group_fd,VFIO_GROUP_GET_DEVICE_FD , dev_addr); + *vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD,dev);
Quoted 16 lines
lib/librte_eal/linux/eal/eal_vfio.c +16
-0+static bool +rte_vfio_dev_has_nonzero_numvfs (const char *sysfs_base,const char +*dev_addr) { +char linkname[PATH_MAX ]; +unsigned long num_vfs; +int ret; + +if (!rte_vfio_dev_is_physfn (sysfs_base, dev_addr)) +return false; + +memset (linkname,0 ,sizeof (linkname)); + +snprintf (linkname,sizeof (linkname), +"%s/%s/sriov_numvfs" , sysfs_base, dev_addr); + + ret =eal_parse_sysfs_value (linkname, &num_vfs);lib/librte_eal/linux/eal/eal_vfio.c rte_vfio_clear_group(int vfio_group_fd)
Quoted 10 lines
lib/librte_eal/linux/eal/eal_vfio.c +7
-0+ +if ((ret <0 ) || (num_vfs ==0 )) +return false; + +return true; +} +int rte_vfio_setup_device (const char *sysfs_base,const char *dev_addr, int *vfio_dev_fd, struct vfio_device_info *device_info) @@ -669,6
Quoted 1 line
+769,7 @@ rte_vfio_setup_device(const char
*sysfs_base, const char
*dev_addr,lib/librte_eal/linux/eal/eal_vfio.c rte_vfio_setup_device(const char *sysfs_base,
Quoted 8 lines
lib/librte_eal/linux/eal/eal_vfio.c +1
-0int vfio_container_fd;int vfio_group_fd;int iommu_group_num; +char dev[PATH_MAX ];int i, ret;/* get group number */ @@ -683,6 +784,29 @@ rte_vfio_setup_device(const char *sysfs_base,const char
*dev_addr,lib/librte_eal/linux/eal/eal_vfio.c rte_vfio_setup_device(const char *sysfs_base,
Quoted 30 lines
lib/librte_eal/linux/eal/eal_vfio.c +23
-0if (ret <0 )return -1 ; +snprintf (dev,sizeof (dev),"%s" , dev_addr); + +if (!rte_vfio_dev_is_physfn (sysfs_base, dev_addr)) { +char vf_token[PATH_MAX ]; +/* +* Check if vf_token is required or not, +* vf_token is required when the VF's physfn is +* binded with vfio-pci driver +*/ + ret =is_vf_token_required (sysfs_base, dev_addr); +/* if negative, something failed */ +if (ret <0 ) +return -1 ; + +if (ret ==0 ) { +/* vf_token required to open device file descriptor */ +rte_uuid_unparse (uuid_token, + vf_token,sizeof (vf_token)); +snprintf (dev,sizeof (dev), +"%s vf_token=%s" , dev_addr, vf_token); + } + } +/* get the actual group fd */ vfio_group_fd =rte_vfio_get_group_fd (iommu_group_num);if (vfio_group_fd <0 )@@ -853,7 +977,7 @@ rte_vfio_setup_device(const char *sysfs_base,const char
*dev_addr,lib/librte_eal/linux/eal/eal_vfio.c rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
Quoted 4 lines
lib/librte_eal/linux/eal/eal_vfio.c +0
-1}/* get a file descriptor for the device */ - *vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD,
dev_addr);lib/librte_eal/linux/eal/eal_vfio.c rte_vfio_setup_device(const char *sysfs_base, · line 980
Quoted 1 line
lib/librte_eal/linux/eal/eal_vfio.c +1
-0+ *vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD,dev);
lib/librte_eal/linux/eal/eal_vfio.c rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
Quoted 3 lines
lib/librte_eal/linux/eal/eal_vfio.c +0
-0if (*vfio_dev_fd <0 ) {/* if we cannot get a device fd, this implies a problem with * the VFIO group or the container not having IOMMU configured.
lib/librte_eal/linux/eal/eal_vfio.c rte_vfio_setup_device(const char *sysfs_base,
Quoted 1 line
lib/librte_eal/linux/eal/eal_vfio.c +0
-0@@ -877,6 +1001,31 @@ rte_vfio_setup_device(const char *sysfs_base,const char
*dev_addr,lib/librte_eal/linux/eal/eal_vfio.c rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr, · line 1005
Quoted 5 lines
lib/librte_eal/linux/eal/eal_vfio.c +2
-0rte_vfio_clear_group (vfio_group_fd);return -1 ; } + +if (rte_vfio_dev_has_nonzero_numvfs (sysfs_base, dev_addr)) {#define
lib/librte_eal/linux/eal/eal_vfio.c rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr, · line 1016
Quoted 10 lines
lib/librte_eal/linux/eal/eal_vfio.c +10
-0+VF_TOKEN (sizeof(struct vfio_device_feature) + sizeof(rte_uuid_t)) + +struct vfio_device_feature *vf_token; +uint8_t local[VF_TOKEN ]; + +memset (local,0 ,VF_TOKEN ); + vf_token = (struct vfio_device_feature *)local; + vf_token->argsz =VF_TOKEN ; + vf_token->flags = VFIO_DEVICE_FEATURE_SET | +VFIO_DEVICE_FEATURE_PCI_VF_TOKEN;Quoted 18 lines
lib/librte_eal/linux/eal/eal_vfio.c +13
-1+ +memcpy (local +sizeof (struct vfio_device_feature ), + &uuid_token,sizeof (uuid_token)); + + ret =ioctl (*vfio_dev_fd,VFIO_DEVICE_FEATURE , vf_token); +if (ret) { +RTE_LOG (ERR ,EAL ," Failed to set UUID on %s " +"error %i (%s)\n" , dev_addr, errno, +strerror (errno)); +return -1 ; + } + } +vfio_group_device_get (vfio_group_fd);return 0 ; --2.8.4
RFC v2 1/1 eal: add VFIO-PCI SR-IOV support
In reply to [email protected]The kernel module vfio-pci introduces the VF token to enable SR-IOV support.
The VF token can be set by a vfio-pci based PF driver and must be known by the vfio-pci based VF driver in order to gain access to the device.
An example VF token option would take this form:
1. ./usertools/dpdk-devbind.py -b vfio-pci 0000:87:00.0
2. echo 2 > /sys/bus/pci/devices/0000:87:00.0/sriov_numvfs
3. Start the PF:
./x86_64-native-linux-gcc/app/testpmd -l 22-25 -n 4 \
-w 87:00.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 \
--file-prefix=pf -- -i4. Start the VF:
./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 \
-w 87:02.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 \
--file-prefix=vf1 -- -iTest based on vfio-pci patch: https://patchwork.ozlabs.org/cover/1253222/
---
Based on RFC v1: https://patchwork.dpdk.org/patch/66281/
--- drivers/bus/pci/linux/pci_vfio.c | 56 +++++++++++++++++++++++++++++-- lib/librte_eal/freebsd/eal.c | 3 +- lib/librte_eal/include/rte_vfio.h | 8 ++++- lib/librte_eal/linux/eal_vfio.c | 20 +++++++++-- 4 files changed, 80 insertions(+), 7 deletions(-)
drivers/bus/pci/linux/pci_vfio.c +54 -2
@@ -11,6 +11,7 @@#include <sys/mman.h> #include <stdbool.h> +#include <rte_devargs.h> #include <rte_log.h> #include <rte_pci.h> #include <rte_bus_pci.h>
@@ -644,11 +645,59 @@ pci_vfio_msix_is_mappable(int vfio_dev_fd, int msix_region)return ret; } +static void +vfio_pci_vf_token_arg (struct rte_devargs *devargs,rte_uuid_t uu) +{ +#define VF_TOKEN_ARG "vf_token=" +char c, *p, *vf_token; + +if (devargs ==NULL ) +return ; + + p =strstr (devargs->args,VF_TOKEN_ARG ); +if (!p) +return ; + + vf_token = p +strlen (VF_TOKEN_ARG ); +if (strlen (vf_token) < (RTE_UUID_STRLEN -1 )) +return ; + + c = vf_token[RTE_UUID_STRLEN -1 ]; +if (c !='\0' && c !=',' ) +return ; + + vf_token[RTE_UUID_STRLEN -1 ] ='\0' ; +if (rte_uuid_parse (vf_token, uu)) { +RTE_LOG (ERR ,EAL , +"The VF token is not a valid uuid : %s\n" , vf_token); + vf_token[RTE_UUID_STRLEN -1 ] = c; +return ; + } + +RTE_LOG (DEBUG ,EAL , +"The VF token is found : %s\n" , vf_token); + + vf_token[RTE_UUID_STRLEN -1 ] = c; + +/* Purge this vfio-pci specific token from the device arguments */ +if (c !='\0' ) { +/* 1. Handle the case : 'vf_token=uuid,arg1=val1' */ +memmove (p, vf_token +RTE_UUID_STRLEN , +strlen (vf_token +RTE_UUID_STRLEN ) +1 ); + }else { +/* 2. Handle the case : 'arg1=val1,vf_token=uuid' */ +if (p != devargs->args) + p--; + + *p ='\0' ; + } +}static int pci_vfio_map_resource_primary (struct rte_pci_device *dev) {struct vfio_device_info device_info = { .argsz =sizeof (device_info) }; +rte_uuid_t vf_token =RTE_UUID_INIT (0 ,0 ,0 ,0 ,0ULL );char pci_addr[PATH_MAX ] = {0 };int vfio_dev_fd;struct rte_pci_addr *loc = &dev->addr;
@@ -668,8 +717,9 @@ pci_vfio_map_resource_primary(struct rte_pci_device *dev)snprintf (pci_addr,sizeof (pci_addr),PCI_PRI_FMT , loc->domain, loc->bus, loc->devid, loc->function); +vfio_pci_vf_token_arg (dev->device.devargs, vf_token); ret =rte_vfio_setup_device (rte_pci_get_sysfs_path (), pci_addr, - &vfio_dev_fd, &device_info); + &vfio_dev_fd, &device_info, vf_token);if (ret)return ret;
@@ -797,6 +847,7 @@ static int pci_vfio_map_resource_secondary(struct rte_pci_device *dev) {struct vfio_device_info device_info = { .argsz =sizeof (device_info) }; +rte_uuid_t vf_token =RTE_UUID_INIT (0 ,0 ,0 ,0 ,0ULL );char pci_addr[PATH_MAX ] = {0 };int vfio_dev_fd;struct rte_pci_addr *loc = &dev->addr;
@@ -830,8 +881,9 @@ pci_vfio_map_resource_secondary(struct rte_pci_device *dev)return -1 ; } +vfio_pci_vf_token_arg (dev->device.devargs, vf_token); ret =rte_vfio_setup_device (rte_pci_get_sysfs_path (), pci_addr, - &vfio_dev_fd, &device_info); + &vfio_dev_fd, &device_info, vf_token);if (ret)return ret;
lib/librte_eal/freebsd/eal.c +2 -1
@@ -995,7 +995,8 @@ rte_eal_vfio_intr_mode(void) int rte_vfio_setup_device(__rte_unused const char *sysfs_base, __rte_unused const char *dev_addr, __rte_unused int *vfio_dev_fd, - __rte_unused struct vfio_device_info *device_info) + __rte_unused struct vfio_device_info *device_info + __rte_unused rte_uuid_t vf_token) {return -1 ; }
lib/librte_eal/include/rte_vfio.h +7 -1
@@ -16,6 +16,8 @@ extern "C" {#include <stdint.h> +#include <rte_uuid.h> +/* * determine if VFIO is present on the system */
@@ -102,13 +104,17 @@ struct vfio_device_info;* @param device_info * Device information. * +* @param vf_token +* VF token. +* * @return * 0 on success. * <0 on failure. * >1 if the device cannot be managed this way. */ int rte_vfio_setup_device (const char *sysfs_base,const char *dev_addr, -int *vfio_dev_fd,struct vfio_device_info *device_info); +int *vfio_dev_fd,struct vfio_device_info *device_info, +rte_uuid_t vf_token);/** * Release a device mapped to a VFIO-managed I/O MMU group.
lib/librte_eal/linux/eal_vfio.c +17 -3
@@ -702,7 +702,8 @@ rte_vfio_clear_group(int vfio_group_fd)int rte_vfio_setup_device (const char *sysfs_base,const char *dev_addr, -int *vfio_dev_fd,struct vfio_device_info *device_info) +int *vfio_dev_fd,struct vfio_device_info *device_info, +rte_uuid_t vf_token) {struct vfio_group_status group_status = { .argsz =sizeof (group_status)
@@ -712,6 +713,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,int vfio_container_fd;int vfio_group_fd;int iommu_group_num; +char dev[PATH_MAX ];int i, ret;/* get group number */
@@ -895,8 +897,19 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr, t->type_id, t->name); } +if (!rte_uuid_is_null (vf_token)) { +char vf_token_str[RTE_UUID_STRLEN ]; + +rte_uuid_unparse (vf_token, vf_token_str,sizeof (vf_token_str)); +snprintf (dev,sizeof (dev), +"%s vf_token=%s" , dev_addr, vf_token_str); + }else { +snprintf (dev,sizeof (dev), +"%s" , dev_addr); + } +/* get a file descriptor for the device */ - *vfio_dev_fd =ioctl (vfio_group_fd,VFIO_GROUP_GET_DEVICE_FD , dev_addr); + *vfio_dev_fd =ioctl (vfio_group_fd,VFIO_GROUP_GET_DEVICE_FD , dev);if (*vfio_dev_fd <0 ) {/* if we cannot get a device fd, this implies a problem with * the VFIO group or the container not having IOMMU configured.
@@ -2081,7 +2094,8 @@ int rte_vfio_setup_device(__rte_unused const char *sysfs_base, __rte_unused const char *dev_addr, __rte_unused int *vfio_dev_fd, - __rte_unused struct vfio_device_info *device_info) + __rte_unused struct vfio_device_info *device_info + __rte_unused rte_uuid_t vf_token) {return -1 ; }
Signature · 1 line
2.26.0
Re: [dpdk-dev] [EXT] [RFC v2] eal: add VFIO-PCI SR-IOV support
Quoted 265 lines
-----Original Message-----
From: Haiyue Wang <[email protected]>
Sent: Friday, April 10, 2020 1:03 PM
To: [email protected]; [email protected]; Vamsi Krishna Attunuru <[email protected]>; Jerin Jacob Kollanukkaran <[email protected]>; [email protected]; [email protected]
Cc: Haiyue Wang <[email protected]>
Subject: [EXT] [RFC v2] eal: add VFIO-PCI SR-IOV supportExternal Email
----------------------------------------------------------------------
The kernel module vfio-pci introduces the VF token to enable SR-IOV support.The VF token can be set by a vfio-pci based PF driver and must be known by the vfio-pci based VF driver in order to gain access to the device.
An example VF token option would take this form:
1.
./usertools/dpdk-devbind.py-b vfio-pci 0000:87:00.02. echo 2 > /sys/bus/pci/devices/0000:87:00.0/sriov_numvfs
3. Start the PF:
./x86_64-native-linux-gcc/app/testpmd -l 22-25 -n 4 \ -w 87:00.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 \ --file-prefix=pf -- -i4. Start the VF:
./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 \ -w 87:02.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 \ --file-prefix=vf1 -- -iTest based on vfio-pci patch:
https://urldefense.proofpoint.com/v2/url?u=https-
3A__patchwork.ozlabs.org_cover_1253222_&d=DwIDAg&c=nKjWec2b6R0m OyPaz7xtfQ&r=WllrYaumVkxaWjgKto6E_rtDQshhIhik2jkvzFyRhW8&m=SFD0o TfFgnU88wT2X7qMWRDen3KFV8oSOlqz3WQm3nI&s=CU2jxKUsy4oiI26apob DF8BJebDrheBiu3eKlF3e76E&e=Signed-off-by Haiyue Wang <[email protected]>---
Based on RFC v1: https://urldefense.proofpoint.com/v2/url?u=https-
3A__patchwork.dpdk.org_patch_66281_&d=DwIDAg&c=nKjWec2b6R0mOyP az7xtfQ&r=WllrYaumVkxaWjgKto6E_rtDQshhIhik2jkvzFyRhW8&m=SFD0oTfFg nU88wT2X7qMWRDen3KFV8oSOlqz3WQm3nI&s=3dHnLcXpkRVOZs6wdZQylc Je0oNNBs77hXE6SOuLO7o&e=--- drivers/bus/pci/linux/pci_vfio.c | 56 +++++++++++++++++++++++++++++-- lib/librte_eal/freebsd/eal.c | 3 +- lib/librte_eal/include/rte_vfio.h | 8 ++++- lib/librte_eal/linux/eal_vfio.c | 20 +++++++++-- 4 files changed, 80 insertions(+), 7 deletions(-)drivers/bus/pci/linux/pci_vfio.c +48
-0@@ -11,6 +11,7 @@#include <sys/mman.h> #include <stdbool.h> +#include <rte_devargs.h> #include <rte_log.h> #include <rte_pci.h> #include <rte_bus_pci.h> @@ -644,11 +645,59 @@ pci_vfio_msix_is_mappable(int vfio_dev_fd, int msix_region)return ret; } +static void +vfio_pci_vf_token_arg (struct rte_devargs *devargs,rte_uuid_t uu) { +#define VF_TOKEN_ARG "vf_token=" +char c, *p, *vf_token; + +if (devargs ==NULL ) +return ; + + p =strstr (devargs->args,VF_TOKEN_ARG ); +if (!p) +return ; + + vf_token = p +strlen (VF_TOKEN_ARG ); +if (strlen (vf_token) < (RTE_UUID_STRLEN -1 )) +return ; + + c = vf_token[RTE_UUID_STRLEN -1 ]; +if (c !='\0' && c !=',' ) +return ; + + vf_token[RTE_UUID_STRLEN -1 ] ='\0' ; +if (rte_uuid_parse (vf_token, uu)) { +RTE_LOG (ERR ,EAL , +"The VF token is not a valid uuid : %s\n" , vf_token); + vf_token[RTE_UUID_STRLEN -1 ] = c; +return ; + } + +RTE_LOG (DEBUG ,EAL , +"The VF token is found : %s\n" , vf_token); + + vf_token[RTE_UUID_STRLEN -1 ] = c; + +/* Purge this vfio-pci specific token from the device arguments */ +if (c !='\0' ) { +/* 1. Handle the case : 'vf_token=uuid,arg1=val1' */ +memmove (p, vf_token +RTE_UUID_STRLEN , +strlen (vf_token +RTE_UUID_STRLEN ) +1 ); + }else { +/* 2. Handle the case : 'arg1=val1,vf_token=uuid' */ +if (p != devargs->args) + p--; + + *p ='\0' ; + } +}static int pci_vfio_map_resource_primary (struct rte_pci_device *dev) {struct vfio_device_info device_info = { .argsz =sizeof (device_info) }; +rte_uuid_t vf_token =RTE_UUID_INIT (0 ,0 ,0 ,0 ,0ULL );char pci_addr[PATH_MAX ] = {0 };int vfio_dev_fd; struct rte_pci_addr *loc = &dev->addr; @@ -668,8 +717,9 @@
pci_vfio_map_resource_primary(struct rte_pci_device *dev)drivers/bus/pci/linux/pci_vfio.c +2
-1snprintf (pci_addr,sizeof (pci_addr),PCI_PRI_FMT , loc->domain, loc->bus, loc->devid, loc->function); +vfio_pci_vf_token_arg (dev->device.devargs, vf_token); ret = rte_vfio_setup_device(rte_pci_get_sysfs_path(), pci_addr, - &vfio_dev_fd, &device_info); + &vfio_dev_fd, &device_info,
vf_token);drivers/bus/pci/linux/pci_vfio.c +1
-0if (ret)return ret;@@ -797,6 +847,7 @@ static int pci_vfio_map_resource_secondary(struct rte_pci_device *dev) {struct vfio_device_info device_info = { .argsz =sizeof (device_info) }; +rte_uuid_t vf_token =RTE_UUID_INIT (0 ,0 ,0 ,0 ,0ULL );char pci_addr[PATH_MAX ] = {0 };int vfio_dev_fd; struct rte_pci_addr *loc = &dev->addr; @@ -830,8 +881,9 @@ pci_vfio_map_resource_secondary(struct rte_pci_device *dev)drivers/bus/pci/linux/pci_vfio.c +2
-1return -1 ; } +vfio_pci_vf_token_arg (dev->device.devargs, vf_token); ret = rte_vfio_setup_device(rte_pci_get_sysfs_path(), pci_addr, - &vfio_dev_fd, &device_info); + &vfio_dev_fd, &device_info,
vf_token);drivers/bus/pci/linux/pci_vfio.c +0
-0if (ret)return ret;lib/librte_eal/freebsd/eal.c index renamed from lib/librte_eal/freebsd/eal.c +0
-0
6ae37e7e6..cbb4c400e100644lib/librte_eal/freebsd/eal.c +2
-1@@ -995,7 +995,8 @@ rte_eal_vfio_intr_mode(void) int rte_vfio_setup_device(__rte_unused const char *sysfs_base, __rte_unused const char *dev_addr, __rte_unused int *vfio_dev_fd, - __rte_unused struct vfio_device_info *device_info) + __rte_unused struct vfio_device_info *device_info + __rte_unused rte_uuid_t vf_token) {return -1 ; }lib/librte_eal/include/rte_vfio.h +7
-1@@ -16,6 +16,8 @@ extern "C" {#include <stdint.h> +#include <rte_uuid.h> +/* * determine if VFIO is present on the system */ @@ -102,13 +104,17 @@ struct vfio_device_info;* @param device_info * Device information. * +* @param vf_token +* VF token. +* * @return * 0 on success. * <0 on failure. * >1 if the device cannot be managed this way. */ int rte_vfio_setup_device (const char *sysfs_base,const char *dev_addr, -int *vfio_dev_fd,struct vfio_device_info *device_info); +int *vfio_dev_fd,struct vfio_device_info *device_info, +rte_uuid_t vf_token);/** * Release a device mapped to a VFIO-managed I/O MMU group. lib/librte_eal/linux/eal_vfio.c +3
-1@@ -702,7 +702,8 @@ rte_vfio_clear_group(int vfio_group_fd)int rte_vfio_setup_device (const char *sysfs_base,const char *dev_addr, -int *vfio_dev_fd,struct vfio_device_info *device_info) +int *vfio_dev_fd,struct vfio_device_info *device_info, +rte_uuid_t vf_token) {struct vfio_group_status group_status = { .argsz =sizeof (group_status)@@ -712,6 +713,7 @@ rte_vfio_setup_device(const char *sysfs_base, constchar *dev_addr, int vfio_container_fd;int vfio_group_fd;int iommu_group_num; +char dev[PATH_MAX ];int i, ret;lib/librte_eal/linux/eal_vfio.c +12
-1/* get group number */ @@ -895,8 +897,19 @@ rte_vfio_setup_device(const char *sysfs_base,const char *dev_addr, t->type_id, t->name); } +if (!rte_uuid_is_null (vf_token)) { +char vf_token_str[RTE_UUID_STRLEN ]; + +rte_uuid_unparse (vf_token, vf_token_str,sizeof (vf_token_str)); +snprintf (dev,sizeof (dev), +"%s vf_token=%s" , dev_addr, vf_token_str); + }else { +snprintf (dev,sizeof (dev), +"%s" , dev_addr); + } +/* get a file descriptor for the device */ - *vfio_dev_fd =ioctl (vfio_group_fd,VFIO_GROUP_GET_DEVICE_FD , dev_addr); + *vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD,dev);
lib/librte_eal/linux/eal_vfio.c +0
-0if (*vfio_dev_fd <0 ) {/* if we cannot get a device fd, this implies a problem with * the VFIO group or the container not having IOMMU configured.
lib/librte_eal/linux/eal_vfio.c +2
-2@@ -2081,7 +2094,8 @@ int rte_vfio_setup_device(__rte_unused const char *sysfs_base, __rte_unused const char *dev_addr, __rte_unused int *vfio_dev_fd, - __rte_unused struct vfio_device_info *device_info) + __rte_unused struct vfio_device_info *device_info + __rte_unused rte_uuid_t vf_token) {return -1 ; } --
Quoted 1 line
2.26.0
PATCH 1/1 eal: add VFIO-PCI SR-IOV support
In reply to [email protected]The kernel module vfio-pci introduces the VF token to enable SR-IOV support since 5.7.
The VF token can be set by a vfio-pci based PF driver and must be known by the vfio-pci based VF driver in order to gain access to the device.
An example VF token option would take this form:
1. Install vfio-pci with option 'enable_sriov=1'
2. ./usertools/dpdk-devbind.py -b vfio-pci 0000:87:00.0
3. echo 2 > /sys/bus/pci/devices/0000:87:00.0/sriov_numvfs
4. Start the PF:
./x86_64-native-linux-gcc/app/testpmd -l 22-25 -n 4 \
-w 87:00.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 \
--file-prefix=pf -- -i5. Start the VF:
./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 \
-w 87:02.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 \
--file-prefix=vf1 -- -i---
v1: Update the commit message.
RFC v2: https://patchwork.dpdk.org/patch/68114/
Based on Vamsi's RFC v1, and Alex's patch for Qemu
[https://lore.kernel.org/lkml/[email protected]/]:
Use the devarg to pass-down the VF token.
RFC v1: https://patchwork.dpdk.org/patch/66281/ by Vamsi.
--- drivers/bus/pci/linux/pci_vfio.c | 56 +++++++++++++++++++++++++++++-- lib/librte_eal/freebsd/eal.c | 3 +- lib/librte_eal/include/rte_vfio.h | 8 ++++- lib/librte_eal/linux/eal_vfio.c | 20 +++++++++-- 4 files changed, 80 insertions(+), 7 deletions(-)
drivers/bus/pci/linux/pci_vfio.c +54 -2
@@ -11,6 +11,7 @@#include <sys/mman.h> #include <stdbool.h> +#include <rte_devargs.h> #include <rte_log.h> #include <rte_pci.h> #include <rte_bus_pci.h>
@@ -644,11 +645,59 @@ pci_vfio_msix_is_mappable(int vfio_dev_fd, int msix_region)return ret; } +static void +vfio_pci_vf_token_arg (struct rte_devargs *devargs,rte_uuid_t uu) +{ +#define VF_TOKEN_ARG "vf_token=" +char c, *p, *vf_token; + +if (devargs ==NULL ) +return ; + + p =strstr (devargs->args,VF_TOKEN_ARG ); +if (!p) +return ; + + vf_token = p +strlen (VF_TOKEN_ARG ); +if (strlen (vf_token) < (RTE_UUID_STRLEN -1 )) +return ; + + c = vf_token[RTE_UUID_STRLEN -1 ]; +if (c !='\0' && c !=',' ) +return ; + + vf_token[RTE_UUID_STRLEN -1 ] ='\0' ; +if (rte_uuid_parse (vf_token, uu)) { +RTE_LOG (ERR ,EAL , +"The VF token is not a valid uuid : %s\n" , vf_token); + vf_token[RTE_UUID_STRLEN -1 ] = c; +return ; + } + +RTE_LOG (DEBUG ,EAL , +"The VF token is found : %s\n" , vf_token); + + vf_token[RTE_UUID_STRLEN -1 ] = c; + +/* Purge this vfio-pci specific token from the device arguments */ +if (c !='\0' ) { +/* 1. Handle the case : 'vf_token=uuid,arg1=val1' */ +memmove (p, vf_token +RTE_UUID_STRLEN , +strlen (vf_token +RTE_UUID_STRLEN ) +1 ); + }else { +/* 2. Handle the case : 'arg1=val1,vf_token=uuid' */ +if (p != devargs->args) + p--; + + *p ='\0' ; + } +}static int pci_vfio_map_resource_primary (struct rte_pci_device *dev) {struct vfio_device_info device_info = { .argsz =sizeof (device_info) }; +rte_uuid_t vf_token =RTE_UUID_INIT (0 ,0 ,0 ,0 ,0ULL );char pci_addr[PATH_MAX ] = {0 };int vfio_dev_fd;struct rte_pci_addr *loc = &dev->addr;
@@ -668,8 +717,9 @@ pci_vfio_map_resource_primary(struct rte_pci_device *dev)snprintf (pci_addr,sizeof (pci_addr),PCI_PRI_FMT , loc->domain, loc->bus, loc->devid, loc->function); +vfio_pci_vf_token_arg (dev->device.devargs, vf_token); ret =rte_vfio_setup_device (rte_pci_get_sysfs_path (), pci_addr, - &vfio_dev_fd, &device_info); + &vfio_dev_fd, &device_info, vf_token);if (ret)return ret;
@@ -797,6 +847,7 @@ static int pci_vfio_map_resource_secondary(struct rte_pci_device *dev) {struct vfio_device_info device_info = { .argsz =sizeof (device_info) }; +rte_uuid_t vf_token =RTE_UUID_INIT (0 ,0 ,0 ,0 ,0ULL );char pci_addr[PATH_MAX ] = {0 };int vfio_dev_fd;struct rte_pci_addr *loc = &dev->addr;
@@ -830,8 +881,9 @@ pci_vfio_map_resource_secondary(struct rte_pci_device *dev)return -1 ; } +vfio_pci_vf_token_arg (dev->device.devargs, vf_token); ret =rte_vfio_setup_device (rte_pci_get_sysfs_path (), pci_addr, - &vfio_dev_fd, &device_info); + &vfio_dev_fd, &device_info, vf_token);if (ret)return ret;
lib/librte_eal/freebsd/eal.c +2 -1
@@ -995,7 +995,8 @@ rte_eal_vfio_intr_mode(void) int rte_vfio_setup_device(__rte_unused const char *sysfs_base, __rte_unused const char *dev_addr, __rte_unused int *vfio_dev_fd, - __rte_unused struct vfio_device_info *device_info) + __rte_unused struct vfio_device_info *device_info + __rte_unused rte_uuid_t vf_token) {return -1 ; }
lib/librte_eal/include/rte_vfio.h +7 -1
@@ -16,6 +16,8 @@ extern "C" {#include <stdint.h> +#include <rte_uuid.h> +/* * determine if VFIO is present on the system */
@@ -102,13 +104,17 @@ struct vfio_device_info;* @param device_info * Device information. * +* @param vf_token +* VF token. +* * @return * 0 on success. * <0 on failure. * >1 if the device cannot be managed this way. */ int rte_vfio_setup_device (const char *sysfs_base,const char *dev_addr, -int *vfio_dev_fd,struct vfio_device_info *device_info); +int *vfio_dev_fd,struct vfio_device_info *device_info, +rte_uuid_t vf_token);/** * Release a device mapped to a VFIO-managed I/O MMU group.
lib/librte_eal/linux/eal_vfio.c +17 -3
@@ -702,7 +702,8 @@ rte_vfio_clear_group(int vfio_group_fd)int rte_vfio_setup_device (const char *sysfs_base,const char *dev_addr, -int *vfio_dev_fd,struct vfio_device_info *device_info) +int *vfio_dev_fd,struct vfio_device_info *device_info, +rte_uuid_t vf_token) {struct vfio_group_status group_status = { .argsz =sizeof (group_status)
@@ -712,6 +713,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,int vfio_container_fd;int vfio_group_fd;int iommu_group_num; +char dev[PATH_MAX ];int i, ret;/* get group number */
@@ -895,8 +897,19 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr, t->type_id, t->name); } +if (!rte_uuid_is_null (vf_token)) { +char vf_token_str[RTE_UUID_STRLEN ]; + +rte_uuid_unparse (vf_token, vf_token_str,sizeof (vf_token_str)); +snprintf (dev,sizeof (dev), +"%s vf_token=%s" , dev_addr, vf_token_str); + }else { +snprintf (dev,sizeof (dev), +"%s" , dev_addr); + } +/* get a file descriptor for the device */ - *vfio_dev_fd =ioctl (vfio_group_fd,VFIO_GROUP_GET_DEVICE_FD , dev_addr); + *vfio_dev_fd =ioctl (vfio_group_fd,VFIO_GROUP_GET_DEVICE_FD , dev);if (*vfio_dev_fd <0 ) {/* if we cannot get a device fd, this implies a problem with * the VFIO group or the container not having IOMMU configured.
@@ -2081,7 +2094,8 @@ int rte_vfio_setup_device(__rte_unused const char *sysfs_base, __rte_unused const char *dev_addr, __rte_unused int *vfio_dev_fd, - __rte_unused struct vfio_device_info *device_info) + __rte_unused struct vfio_device_info *device_info + __rte_unused rte_uuid_t vf_token) {return -1 ; }
Signature · 1 line
2.26.0
PATCH v2 1/1 eal: add VFIO-PCI SR-IOV support
In reply to [email protected]The kernel module vfio-pci introduces the VF token to enable SR-IOV support since 5.7.
The VF token can be set by a vfio-pci based PF driver and must be known by the vfio-pci based VF driver in order to gain access to the device.
An example VF token option would take this form:
1. Install vfio-pci with option 'enable_sriov=1'
2. ./usertools/dpdk-devbind.py -b vfio-pci 0000:87:00.0
3. echo 2 > /sys/bus/pci/devices/0000:87:00.0/sriov_numvfs
4. Start the PF:
./x86_64-native-linux-gcc/app/testpmd -l 22-25 -n 4 \
-w 87:00.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 \
--file-prefix=pf -- -i5. Start the VF:
./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 \
-w 87:02.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 \
--file-prefix=vf1 -- -i---
v2: Fix the FreeBSD build error.
v1: https://patchwork.dpdk.org/patch/68237/
Update the commit message.
RFC v2: https://patchwork.dpdk.org/patch/68114/
Based on Vamsi's RFC v1, and Alex's patch for Qemu
[https://lore.kernel.org/lkml/[email protected]/]:
Use the devarg to pass-down the VF token.
RFC v1: https://patchwork.dpdk.org/patch/66281/ by Vamsi.
--- drivers/bus/pci/linux/pci_vfio.c | 56 +++++++++++++++++++++++++++++-- lib/librte_eal/freebsd/eal.c | 3 +- lib/librte_eal/include/rte_vfio.h | 8 ++++- lib/librte_eal/linux/eal_vfio.c | 20 +++++++++-- 4 files changed, 80 insertions(+), 7 deletions(-)
drivers/bus/pci/linux/pci_vfio.c +54 -2
@@ -11,6 +11,7 @@#include <sys/mman.h> #include <stdbool.h> +#include <rte_devargs.h> #include <rte_log.h> #include <rte_pci.h> #include <rte_bus_pci.h>
@@ -644,11 +645,59 @@ pci_vfio_msix_is_mappable(int vfio_dev_fd, int msix_region)return ret; } +static void +vfio_pci_vf_token_arg (struct rte_devargs *devargs,rte_uuid_t uu) +{ +#define VF_TOKEN_ARG "vf_token=" +char c, *p, *vf_token; + +if (devargs ==NULL ) +return ; + + p =strstr (devargs->args,VF_TOKEN_ARG ); +if (!p) +return ; + + vf_token = p +strlen (VF_TOKEN_ARG ); +if (strlen (vf_token) < (RTE_UUID_STRLEN -1 )) +return ; + + c = vf_token[RTE_UUID_STRLEN -1 ]; +if (c !='\0' && c !=',' ) +return ; + + vf_token[RTE_UUID_STRLEN -1 ] ='\0' ; +if (rte_uuid_parse (vf_token, uu)) { +RTE_LOG (ERR ,EAL , +"The VF token is not a valid uuid : %s\n" , vf_token); + vf_token[RTE_UUID_STRLEN -1 ] = c; +return ; + } + +RTE_LOG (DEBUG ,EAL , +"The VF token is found : %s\n" , vf_token); + + vf_token[RTE_UUID_STRLEN -1 ] = c; + +/* Purge this vfio-pci specific token from the device arguments */ +if (c !='\0' ) { +/* 1. Handle the case : 'vf_token=uuid,arg1=val1' */ +memmove (p, vf_token +RTE_UUID_STRLEN , +strlen (vf_token +RTE_UUID_STRLEN ) +1 ); + }else { +/* 2. Handle the case : 'arg1=val1,vf_token=uuid' */ +if (p != devargs->args) + p--; + + *p ='\0' ; + } +}static int pci_vfio_map_resource_primary (struct rte_pci_device *dev) {struct vfio_device_info device_info = { .argsz =sizeof (device_info) }; +rte_uuid_t vf_token =RTE_UUID_INIT (0 ,0 ,0 ,0 ,0ULL );char pci_addr[PATH_MAX ] = {0 };int vfio_dev_fd;struct rte_pci_addr *loc = &dev->addr;
@@ -668,8 +717,9 @@ pci_vfio_map_resource_primary(struct rte_pci_device *dev)snprintf (pci_addr,sizeof (pci_addr),PCI_PRI_FMT , loc->domain, loc->bus, loc->devid, loc->function); +vfio_pci_vf_token_arg (dev->device.devargs, vf_token); ret =rte_vfio_setup_device (rte_pci_get_sysfs_path (), pci_addr, - &vfio_dev_fd, &device_info); + &vfio_dev_fd, &device_info, vf_token);if (ret)return ret;
@@ -797,6 +847,7 @@ static int pci_vfio_map_resource_secondary(struct rte_pci_device *dev) {struct vfio_device_info device_info = { .argsz =sizeof (device_info) }; +rte_uuid_t vf_token =RTE_UUID_INIT (0 ,0 ,0 ,0 ,0ULL );char pci_addr[PATH_MAX ] = {0 };int vfio_dev_fd;struct rte_pci_addr *loc = &dev->addr;
@@ -830,8 +881,9 @@ pci_vfio_map_resource_secondary(struct rte_pci_device *dev)return -1 ; } +vfio_pci_vf_token_arg (dev->device.devargs, vf_token); ret =rte_vfio_setup_device (rte_pci_get_sysfs_path (), pci_addr, - &vfio_dev_fd, &device_info); + &vfio_dev_fd, &device_info, vf_token);if (ret)return ret;
lib/librte_eal/freebsd/eal.c +2 -1
@@ -995,7 +995,8 @@ rte_eal_vfio_intr_mode(void) int rte_vfio_setup_device(__rte_unused const char *sysfs_base, __rte_unused const char *dev_addr, __rte_unused int *vfio_dev_fd, - __rte_unused struct vfio_device_info *device_info) + __rte_unused struct vfio_device_info *device_info, + __rte_unused rte_uuid_t vf_token) {return -1 ; }
lib/librte_eal/include/rte_vfio.h +7 -1
@@ -16,6 +16,8 @@ extern "C" {#include <stdint.h> +#include <rte_uuid.h> +/* * determine if VFIO is present on the system */
@@ -102,13 +104,17 @@ struct vfio_device_info;* @param device_info * Device information. * +* @param vf_token +* VF token. +* * @return * 0 on success. * <0 on failure. * >1 if the device cannot be managed this way. */ int rte_vfio_setup_device (const char *sysfs_base,const char *dev_addr, -int *vfio_dev_fd,struct vfio_device_info *device_info); +int *vfio_dev_fd,struct vfio_device_info *device_info, +rte_uuid_t vf_token);/** * Release a device mapped to a VFIO-managed I/O MMU group.
lib/librte_eal/linux/eal_vfio.c +17 -3
@@ -702,7 +702,8 @@ rte_vfio_clear_group(int vfio_group_fd)int rte_vfio_setup_device (const char *sysfs_base,const char *dev_addr, -int *vfio_dev_fd,struct vfio_device_info *device_info) +int *vfio_dev_fd,struct vfio_device_info *device_info, +rte_uuid_t vf_token) {struct vfio_group_status group_status = { .argsz =sizeof (group_status)
@@ -712,6 +713,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,int vfio_container_fd;int vfio_group_fd;int iommu_group_num; +char dev[PATH_MAX ];int i, ret;/* get group number */
@@ -895,8 +897,19 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr, t->type_id, t->name); } +if (!rte_uuid_is_null (vf_token)) { +char vf_token_str[RTE_UUID_STRLEN ]; + +rte_uuid_unparse (vf_token, vf_token_str,sizeof (vf_token_str)); +snprintf (dev,sizeof (dev), +"%s vf_token=%s" , dev_addr, vf_token_str); + }else { +snprintf (dev,sizeof (dev), +"%s" , dev_addr); + } +/* get a file descriptor for the device */ - *vfio_dev_fd =ioctl (vfio_group_fd,VFIO_GROUP_GET_DEVICE_FD , dev_addr); + *vfio_dev_fd =ioctl (vfio_group_fd,VFIO_GROUP_GET_DEVICE_FD , dev);if (*vfio_dev_fd <0 ) {/* if we cannot get a device fd, this implies a problem with * the VFIO group or the container not having IOMMU configured.
@@ -2081,7 +2094,8 @@ int rte_vfio_setup_device(__rte_unused const char *sysfs_base, __rte_unused const char *dev_addr, __rte_unused int *vfio_dev_fd, - __rte_unused struct vfio_device_info *device_info) + __rte_unused struct vfio_device_info *device_info, + __rte_unused rte_uuid_t vf_token) { return -1; }
Signature · 1 line
2.26.0
PATCH v3 1/1 eal: add VFIO-PCI SR-IOV support
In reply to [email protected]The kernel module vfio-pci introduces the VF token to enable SR-IOV support since 5.7.
The VF token can be set by a vfio-pci based PF driver and must be known by the vfio-pci based VF driver in order to gain access to the device.
An example VF token option would take this form:
1. Install vfio-pci with option 'enable_sriov=1'
2. ./usertools/dpdk-devbind.py -b vfio-pci 0000:87:00.0
3. echo 2 > /sys/bus/pci/devices/0000:87:00.0/sriov_numvfs
4. Start the PF:
./x86_64-native-linux-gcc/app/testpmd -l 22-25 -n 4 \
-w 87:00.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 \
--file-prefix=pf -- -i5. Start the VF:
./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 \
-w 87:02.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 \
--file-prefix=vf1 -- -i---
v3: Fix the Travis build failed:
(1). rte_uuid.h:97:55: error: unknown type name ‘size_t’ (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’
v2: https://patchwork.dpdk.org/patch/68240/
Fix the FreeBSD build error.
v1: https://patchwork.dpdk.org/patch/68237/
Update the commit message.
RFC v2: https://patchwork.dpdk.org/patch/68114/
Based on Vamsi's RFC v1, and Alex's patch for Qemu
[https://lore.kernel.org/lkml/[email protected]/]:
Use the devarg to pass-down the VF token.
RFC v1: https://patchwork.dpdk.org/patch/66281/ by Vamsi.
--- drivers/bus/pci/linux/pci_vfio.c | 56 +++++++++++++++++++++++++++++-- lib/librte_eal/freebsd/eal.c | 3 +- lib/librte_eal/include/rte_uuid.h | 2 ++ lib/librte_eal/include/rte_vfio.h | 8 ++++- lib/librte_eal/linux/eal_vfio.c | 20 +++++++++-- 5 files changed, 82 insertions(+), 7 deletions(-)
drivers/bus/pci/linux/pci_vfio.c +54 -2
@@ -11,6 +11,7 @@#include <sys/mman.h> #include <stdbool.h> +#include <rte_devargs.h> #include <rte_log.h> #include <rte_pci.h> #include <rte_bus_pci.h>
@@ -644,11 +645,59 @@ pci_vfio_msix_is_mappable(int vfio_dev_fd, int msix_region)return ret; } +static void +vfio_pci_vf_token_arg (struct rte_devargs *devargs,rte_uuid_t uu) +{ +#define VF_TOKEN_ARG "vf_token=" +char c, *p, *vf_token; + +if (devargs ==NULL ) +return ; + + p =strstr (devargs->args,VF_TOKEN_ARG ); +if (!p) +return ; + + vf_token = p +strlen (VF_TOKEN_ARG ); +if (strlen (vf_token) < (RTE_UUID_STRLEN -1 )) +return ; + + c = vf_token[RTE_UUID_STRLEN -1 ]; +if (c !='\0' && c !=',' ) +return ; + + vf_token[RTE_UUID_STRLEN -1 ] ='\0' ; +if (rte_uuid_parse (vf_token, uu)) { +RTE_LOG (ERR ,EAL , +"The VF token is not a valid uuid : %s\n" , vf_token); + vf_token[RTE_UUID_STRLEN -1 ] = c; +return ; + } + +RTE_LOG (DEBUG ,EAL , +"The VF token is found : %s\n" , vf_token); + + vf_token[RTE_UUID_STRLEN -1 ] = c; + +/* Purge this vfio-pci specific token from the device arguments */ +if (c !='\0' ) { +/* 1. Handle the case : 'vf_token=uuid,arg1=val1' */ +memmove (p, vf_token +RTE_UUID_STRLEN , +strlen (vf_token +RTE_UUID_STRLEN ) +1 ); + }else { +/* 2. Handle the case : 'arg1=val1,vf_token=uuid' */ +if (p != devargs->args) + p--; + + *p ='\0' ; + } +}static int pci_vfio_map_resource_primary (struct rte_pci_device *dev) {struct vfio_device_info device_info = { .argsz =sizeof (device_info) }; +rte_uuid_t vf_token =RTE_UUID_INIT (0 ,0 ,0 ,0 ,0ULL );char pci_addr[PATH_MAX ] = {0 };int vfio_dev_fd;struct rte_pci_addr *loc = &dev->addr;
@@ -668,8 +717,9 @@ pci_vfio_map_resource_primary(struct rte_pci_device *dev)snprintf (pci_addr,sizeof (pci_addr),PCI_PRI_FMT , loc->domain, loc->bus, loc->devid, loc->function); +vfio_pci_vf_token_arg (dev->device.devargs, vf_token); ret =rte_vfio_setup_device (rte_pci_get_sysfs_path (), pci_addr, - &vfio_dev_fd, &device_info); + &vfio_dev_fd, &device_info, vf_token);if (ret)return ret;
@@ -797,6 +847,7 @@ static int pci_vfio_map_resource_secondary(struct rte_pci_device *dev) {struct vfio_device_info device_info = { .argsz =sizeof (device_info) }; +rte_uuid_t vf_token =RTE_UUID_INIT (0 ,0 ,0 ,0 ,0ULL );char pci_addr[PATH_MAX ] = {0 };int vfio_dev_fd;struct rte_pci_addr *loc = &dev->addr;
@@ -830,8 +881,9 @@ pci_vfio_map_resource_secondary(struct rte_pci_device *dev)return -1 ; } +vfio_pci_vf_token_arg (dev->device.devargs, vf_token); ret =rte_vfio_setup_device (rte_pci_get_sysfs_path (), pci_addr, - &vfio_dev_fd, &device_info); + &vfio_dev_fd, &device_info, vf_token);if (ret)return ret;
lib/librte_eal/freebsd/eal.c +2 -1
@@ -995,7 +995,8 @@ rte_eal_vfio_intr_mode(void) int rte_vfio_setup_device(__rte_unused const char *sysfs_base, __rte_unused const char *dev_addr, __rte_unused int *vfio_dev_fd, - __rte_unused struct vfio_device_info *device_info) + __rte_unused struct vfio_device_info *device_info, + __rte_unused rte_uuid_t vf_token) {return -1 ; }
lib/librte_eal/include/rte_uuid.h +2 -0
@@ -15,6 +15,8 @@ extern "C" {#endif #include <stdbool.h> +#include <stddef.h> +#include <string.h> /** * Struct describing a Universal Unique Identifier
lib/librte_eal/include/rte_vfio.h +7 -1
@@ -16,6 +16,8 @@ extern "C" {#include <stdint.h> +#include <rte_uuid.h> +/* * determine if VFIO is present on the system */
@@ -102,13 +104,17 @@ struct vfio_device_info;* @param device_info * Device information. * +* @param vf_token +* VF token. +* * @return * 0 on success. * <0 on failure. * >1 if the device cannot be managed this way. */ int rte_vfio_setup_device (const char *sysfs_base,const char *dev_addr, -int *vfio_dev_fd,struct vfio_device_info *device_info); +int *vfio_dev_fd,struct vfio_device_info *device_info, +rte_uuid_t vf_token);/** * Release a device mapped to a VFIO-managed I/O MMU group.
lib/librte_eal/linux/eal_vfio.c +17 -3
@@ -702,7 +702,8 @@ rte_vfio_clear_group(int vfio_group_fd)int rte_vfio_setup_device (const char *sysfs_base,const char *dev_addr, -int *vfio_dev_fd,struct vfio_device_info *device_info) +int *vfio_dev_fd,struct vfio_device_info *device_info, +rte_uuid_t vf_token) {struct vfio_group_status group_status = { .argsz =sizeof (group_status)
@@ -712,6 +713,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,int vfio_container_fd;int vfio_group_fd;int iommu_group_num; +char dev[PATH_MAX ];int i, ret;/* get group number */
@@ -895,8 +897,19 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr, t->type_id, t->name); } +if (!rte_uuid_is_null (vf_token)) { +char vf_token_str[RTE_UUID_STRLEN ]; + +rte_uuid_unparse (vf_token, vf_token_str,sizeof (vf_token_str)); +snprintf (dev,sizeof (dev), +"%s vf_token=%s" , dev_addr, vf_token_str); + }else { +snprintf (dev,sizeof (dev), +"%s" , dev_addr); + } +/* get a file descriptor for the device */ - *vfio_dev_fd =ioctl (vfio_group_fd,VFIO_GROUP_GET_DEVICE_FD , dev_addr); + *vfio_dev_fd =ioctl (vfio_group_fd,VFIO_GROUP_GET_DEVICE_FD , dev);if (*vfio_dev_fd <0 ) {/* if we cannot get a device fd, this implies a problem with * the VFIO group or the container not having IOMMU configured.
@@ -2081,7 +2094,8 @@ int rte_vfio_setup_device(__rte_unused const char *sysfs_base, __rte_unused const char *dev_addr, __rte_unused int *vfio_dev_fd, - __rte_unused struct vfio_device_info *device_info) + __rte_unused struct vfio_device_info *device_info + __rte_unused rte_uuid_t vf_token) {return -1 ; }
Signature · 1 line
2.26.0
PATCH v4 1/1 eal: add VFIO-PCI SR-IOV support
In reply to [email protected]The kernel module vfio-pci introduces the VF token to enable SR-IOV support since 5.7.
The VF token can be set by a vfio-pci based PF driver and must be known by the vfio-pci based VF driver in order to gain access to the device.
An example VF token option would take this form:
1. Install vfio-pci with option 'enable_sriov=1'
2. ./usertools/dpdk-devbind.py -b vfio-pci 0000:87:00.0
3. echo 2 > /sys/bus/pci/devices/0000:87:00.0/sriov_numvfs
4. Start the PF:
./x86_64-native-linux-gcc/app/testpmd -l 22-25 -n 4 \
-w 87:00.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 \
--file-prefix=pf -- -i5. Start the VF:
./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 \
-w 87:02.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 \
--file-prefix=vf1 -- -i---
v4: 1. Ignore rte_vfio_setup_device ABI check since it is
for Linux driver use.
v3: https://patchwork.dpdk.org/patch/68254/
Fix the Travis build failed:
(1). rte_uuid.h:97:55: error: unknown type name ‘size_t’ (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’
v2: https://patchwork.dpdk.org/patch/68240/
Fix the FreeBSD build error.
v1: https://patchwork.dpdk.org/patch/68237/
Update the commit message.
RFC v2: https://patchwork.dpdk.org/patch/68114/
Based on Vamsi's RFC v1, and Alex's patch for Qemu
[https://lore.kernel.org/lkml/[email protected]/]:
Use the devarg to pass-down the VF token.
RFC v1: https://patchwork.dpdk.org/patch/66281/ by Vamsi.
--- devtools/libabigail.abignore | 3 ++ drivers/bus/pci/linux/pci_vfio.c | 56 +++++++++++++++++++++++++++++-- lib/librte_eal/freebsd/eal.c | 3 +- lib/librte_eal/include/rte_uuid.h | 2 ++ lib/librte_eal/include/rte_vfio.h | 8 ++++- lib/librte_eal/linux/eal_vfio.c | 20 +++++++++-- 6 files changed, 85 insertions(+), 7 deletions(-)
devtools/libabigail.abignore +3 -0
@@ -11,3 +11,6 @@ type_kind = enum name = rte_crypto_asym_xform_type changed_enumerators = RTE_CRYPTO_ASYM_XFORM_TYPE_LIST_END +; Ignore this function which is only relevant to linux for driver +[suppress_type] + name = rte_vfio_setup_device
drivers/bus/pci/linux/pci_vfio.c +54 -2
93 lines not on this page · raw message
lib/librte_eal/freebsd/eal.c +2 -1
@@ -995,7 +995,8 @@ rte_eal_vfio_intr_mode(void) int rte_vfio_setup_device(__rte_unused const char *sysfs_base, __rte_unused const char *dev_addr, __rte_unused int *vfio_dev_fd, - __rte_unused struct vfio_device_info *device_info) + __rte_unused struct vfio_device_info *device_info, + __rte_unused rte_uuid_t vf_token) {return -1 ; }
lib/librte_eal/include/rte_uuid.h +2 -0
@@ -15,6 +15,8 @@ extern "C" {#endif #include <stdbool.h> +#include <stddef.h> +#include <string.h> /** * Struct describing a Universal Unique Identifier
lib/librte_eal/include/rte_vfio.h +7 -1
@@ -16,6 +16,8 @@ extern "C" {#include <stdint.h> +#include <rte_uuid.h> +/* * determine if VFIO is present on the system */
@@ -102,13 +104,17 @@ struct vfio_device_info;* @param device_info * Device information. * +* @param vf_token +* VF token. +* * @return * 0 on success. * <0 on failure. * >1 if the device cannot be managed this way. */ int rte_vfio_setup_device (const char *sysfs_base,const char *dev_addr, -int *vfio_dev_fd,struct vfio_device_info *device_info); +int *vfio_dev_fd,struct vfio_device_info *device_info, +rte_uuid_t vf_token);/** * Release a device mapped to a VFIO-managed I/O MMU group.
lib/librte_eal/linux/eal_vfio.c +17 -3
@@ -702,7 +702,8 @@ rte_vfio_clear_group(int vfio_group_fd)int rte_vfio_setup_device (const char *sysfs_base,const char *dev_addr, -int *vfio_dev_fd,struct vfio_device_info *device_info) +int *vfio_dev_fd,struct vfio_device_info *device_info, +rte_uuid_t vf_token) {struct vfio_group_status group_status = { .argsz =sizeof (group_status)
@@ -712,6 +713,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,int vfio_container_fd;int vfio_group_fd;int iommu_group_num; +char dev[PATH_MAX ];int i, ret;/* get group number */
@@ -895,8 +897,19 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr, t->type_id, t->name); } +if (!rte_uuid_is_null (vf_token)) { +char vf_token_str[RTE_UUID_STRLEN ]; + +rte_uuid_unparse (vf_token, vf_token_str,sizeof (vf_token_str)); +snprintf (dev,sizeof (dev), +"%s vf_token=%s" , dev_addr, vf_token_str); + }else { +snprintf (dev,sizeof (dev), +"%s" , dev_addr); + } +/* get a file descriptor for the device */ - *vfio_dev_fd =ioctl (vfio_group_fd,VFIO_GROUP_GET_DEVICE_FD , dev_addr); + *vfio_dev_fd =ioctl (vfio_group_fd,VFIO_GROUP_GET_DEVICE_FD , dev);if (*vfio_dev_fd <0 ) {/* if we cannot get a device fd, this implies a problem with * the VFIO group or the container not having IOMMU configured.
@@ -2081,7 +2094,8 @@ int rte_vfio_setup_device(__rte_unused const char *sysfs_base, __rte_unused const char *dev_addr, __rte_unused int *vfio_dev_fd, - __rte_unused struct vfio_device_info *device_info) + __rte_unused struct vfio_device_info *device_info, + __rte_unused rte_uuid_t vf_token) { return -1; }
Signature · 1 line
2.26.0
Re: [dpdk-dev] [PATCH v4] eal: add VFIO-PCI SR-IOV support
Hi,
About the title, I think it does not convey what is new here. VFIO is not new, SR-IOV is already supported.
The title should mention the new VFIO feature in few simple words. Is it only about using VFIO for PF?
13/04/2020 10:29, Haiyue Wang:
Quoted 2 lines
v4: 1. Ignore
rte_vfio_setup_deviceABI check since it is
for Linux driver use.
[...]
devtools/libabigail.abignore line 16
Quoted 3 lines
devtools/libabigail.abignore +3
-0+; Ignore this function which is only relevant to linux for driver +[suppress_type] + name = rte_vfio_setup_device
Adding such exception for all internal "driver interface" functions is not scaling. Please use __rte_internal.
I am waiting for the patchset about rte_internal to be reviewed or completed. As it is not progressing, the decision is to block any patch having ABI issue because of internal false positive.
Please help, thanks.
Re: [dpdk-dev] [PATCH v4] eal: add VFIO-PCI SR-IOV support
Hi Thomas,
Quoted 8 lines
-----Original Message-----
From: Thomas Monjalon <[email protected]>
Sent: Monday, April 13, 2020 20:18
To: Wang, Haiyue <[email protected]>
Cc: [email protected]; [email protected]; [email protected]; [email protected]; [email protected]
Subject: Re: [PATCH v4] eal: add VFIO-PCI SR-IOV support
Quoted 7 lines
Hi,
About the title, I think it does not convey what is new here. VFIO is not new, SR-IOV is already supported.
The title should mention the new VFIO feature in few simple words. Is it only about using VFIO for PF?
For Both. Align with Alex's : vfio-pci: QEMU support for vfio-pci VF tokens
How about: 'eal: support for VFIO-PCI VF tokens' ?
Quoted 5 lines
13/04/2020 10:29, Haiyue Wang:
Quoted 2 lines
v4: 1. Ignore
rte_vfio_setup_deviceABI check since it is
for Linux driver use.[...]
Quoted 11 lines
devtools/libabigail.abignore line 16
Quoted 3 lines
devtools/libabigail.abignore +3
-0+; Ignore this function which is only relevant to linux for driver +[suppress_type] + name = rte_vfio_setup_deviceAdding such exception for all internal "driver interface" functions is not scaling. Please use
__rte_internal.
I am waiting for the patchset aboutrte_internalto be reviewed or completed. As it is not progressing, the decision is to block any patch having ABI issue because of internal false positive.
Please help, thanks.
I will drop this patch ABI workaround, and try to catch the '__rte_internal ' design.
Re: [dpdk-dev] [PATCH v4] eal: add VFIO-PCI SR-IOV support
In reply to Haiyue WangQuoted 10 lines · Haiyue Wang
The kernel module vfio-pci introduces the VF token to enable SR-IOV support since 5.7.
The VF token can be set by a vfio-pci based PF driver and must be known by the vfio-pci based VF driver in order to gain access to the device.
An example VF token option would take this form:
1. Install vfio-pci with option 'enable_sriov=1'
Quoted 8 lines
2.
./usertools/dpdk-devbind.py-b vfio-pci 0000:87:00.03. echo 2 > /sys/bus/pci/devices/0000:87:00.0/sriov_numvfs
4. Start the PF:
./x86_64-native-linux-gcc/app/testpmd -l 22-25 -n 4 \ -w 87:00.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 \ --file-prefix=pf -- -i
Should I get a token from my head? Any?
Quoted 76 lines
5. Start the VF:
./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 \ -w 87:02.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 \ --file-prefix=vf1 -- -iSigned-off-by Haiyue Wang <[email protected]>Acked-by Vamsi Attunuru <[email protected]>---
v4: 1. Ignorerte_vfio_setup_deviceABI check since it is
for Linux driver use.v3: https://patchwork.dpdk.org/patch/68254/
Fix the Travis build failed:
(1). rte_uuid.h:97:55: error: unknown type name ‘size_t’ (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’v2: https://patchwork.dpdk.org/patch/68240/
Fix the FreeBSD build error.v1: https://patchwork.dpdk.org/patch/68237/
Update the commit message.RFC v2: https://patchwork.dpdk.org/patch/68114/
Based on Vamsi's RFC v1, and Alex's patch for Qemu
[https://lore.kernel.org/lkml/[email protected]/]:
Use the devarg to pass-down the VF token.RFC v1: https://patchwork.dpdk.org/patch/66281/ by Vamsi.
--- devtools/libabigail.abignore | 3 ++ drivers/bus/pci/linux/pci_vfio.c | 56 +++++++++++++++++++++++++++++-- lib/librte_eal/freebsd/eal.c | 3 +- lib/librte_eal/include/rte_uuid.h | 2 ++ lib/librte_eal/include/rte_vfio.h | 8 ++++- lib/librte_eal/linux/eal_vfio.c | 20 +++++++++-- 6 files changed, 85 insertions(+), 7 deletions(-)devtools/libabigail.abignore +3
-0@@ -11,3 +11,6 @@ type_kind = enum name = rte_crypto_asym_xform_type changed_enumerators = RTE_CRYPTO_ASYM_XFORM_TYPE_LIST_END +; Ignore this function which is only relevant to linux for driver +[suppress_type] + name = rte_vfio_setup_devicedrivers/bus/pci/linux/pci_vfio.c +13
-0@@ -11,6 +11,7 @@#include <sys/mman.h> #include <stdbool.h> +#include <rte_devargs.h> #include <rte_log.h> #include <rte_pci.h> #include <rte_bus_pci.h> @@ -644,11 +645,59 @@ pci_vfio_msix_is_mappable(int vfio_dev_fd, int msix_region)return ret; } +static void +vfio_pci_vf_token_arg (struct rte_devargs *devargs,rte_uuid_t uu) +{ +#define VF_TOKEN_ARG "vf_token=" +char c, *p, *vf_token; + +if (devargs ==NULL ) +return ; + + p =strstr (devargs->args,VF_TOKEN_ARG ); +if (!p) +return ;
drivers/bus/pci/linux/pci_vfio.c pci_vfio_msix_is_mappable(int vfio_dev_fd, int msix_region) · line 669
Quoted 10 lines
drivers/bus/pci/linux/pci_vfio.c +10
-0+ + vf_token = p +strlen (VF_TOKEN_ARG ); +if (strlen (vf_token) < (RTE_UUID_STRLEN -1 )) +return ; + + c = vf_token[RTE_UUID_STRLEN -1 ]; +if (c !='\0' && c !=',' ) +return ; + + vf_token[RTE_UUID_STRLEN -1 ] ='\0' ;
Is it possible to parse and handle devargs using rte_kvargs.h?
drivers/bus/pci/linux/pci_vfio.c pci_vfio_msix_is_mappable(int vfio_dev_fd, int msix_region) · line 674
Quoted 5 lines
drivers/bus/pci/linux/pci_vfio.c +5
-0+if (rte_uuid_parse (vf_token, uu)) { +RTE_LOG (ERR ,EAL , +"The VF token is not a valid uuid : %s\n" , vf_token); + vf_token[RTE_UUID_STRLEN -1 ] = c; +return ;
I think that the function must return error which is handled by the caller when something bad happens (e.g. invalid UUID).
Quoted 9 lines
drivers/bus/pci/linux/pci_vfio.c +9
-0+ } + +RTE_LOG (DEBUG ,EAL , +"The VF token is found : %s\n" , vf_token); + + vf_token[RTE_UUID_STRLEN -1 ] = c; + +/* Purge this vfio-pci specific token from the device arguments */ +if (c !='\0' ) {
drivers/bus/pci/linux/pci_vfio.c pci_vfio_msix_is_mappable(int vfio_dev_fd, int msix_region) · line 693
Quoted 10 lines
drivers/bus/pci/linux/pci_vfio.c +10
-0+/* 1. Handle the case : 'vf_token=uuid,arg1=val1' */ +memmove (p, vf_token +RTE_UUID_STRLEN , +strlen (vf_token +RTE_UUID_STRLEN ) +1 ); + }else { +/* 2. Handle the case : 'arg1=val1,vf_token=uuid' */ +if (p != devargs->args) + p--; + + *p ='\0' ; + }
Is it really required to purge? Why? If yes, it should be explained in the comment above.
drivers/bus/pci/linux/pci_vfio.c pci_vfio_msix_is_mappable(int vfio_dev_fd, int msix_region) · line 700
Quoted 7 lines
drivers/bus/pci/linux/pci_vfio.c +2
-0+}static int pci_vfio_map_resource_primary (struct rte_pci_device *dev) {struct vfio_device_info device_info = { .argsz =sizeof (device_info) }; +rte_uuid_t vf_token =RTE_UUID_INIT (0 ,0 ,0 ,0 ,0ULL );
May be it would be better if vfio_pci_vf_token_arg() initializes it anyway instead of duplication init in two places?
Quoted 74 lines
drivers/bus/pci/linux/pci_vfio.c +5
-2char pci_addr[PATH_MAX ] = {0 };int vfio_dev_fd;struct rte_pci_addr *loc = &dev->addr;@@ -668,8 +717,9 @@ pci_vfio_map_resource_primary(struct rte_pci_device *dev)snprintf (pci_addr,sizeof (pci_addr),PCI_PRI_FMT , loc->domain, loc->bus, loc->devid, loc->function); +vfio_pci_vf_token_arg (dev->device.devargs, vf_token); ret =rte_vfio_setup_device (rte_pci_get_sysfs_path (), pci_addr, - &vfio_dev_fd, &device_info); + &vfio_dev_fd, &device_info, vf_token);if (ret)return ret;@@ -797,6 +847,7 @@ static int pci_vfio_map_resource_secondary(struct rte_pci_device *dev) {struct vfio_device_info device_info = { .argsz =sizeof (device_info) }; +rte_uuid_t vf_token =RTE_UUID_INIT (0 ,0 ,0 ,0 ,0ULL );char pci_addr[PATH_MAX ] = {0 };int vfio_dev_fd;struct rte_pci_addr *loc = &dev->addr;@@ -830,8 +881,9 @@ pci_vfio_map_resource_secondary(struct rte_pci_device *dev)return -1 ; } +vfio_pci_vf_token_arg (dev->device.devargs, vf_token); ret =rte_vfio_setup_device (rte_pci_get_sysfs_path (), pci_addr, - &vfio_dev_fd, &device_info); + &vfio_dev_fd, &device_info, vf_token);if (ret)return ret;lib/librte_eal/freebsd/eal.c +2
-1@@ -995,7 +995,8 @@ rte_eal_vfio_intr_mode(void) int rte_vfio_setup_device(__rte_unused const char *sysfs_base, __rte_unused const char *dev_addr, __rte_unused int *vfio_dev_fd, - __rte_unused struct vfio_device_info *device_info) + __rte_unused struct vfio_device_info *device_info, + __rte_unused rte_uuid_t vf_token) {return -1 ; }lib/librte_eal/include/rte_uuid.h +2
-0@@ -15,6 +15,8 @@ extern "C" {#endif #include <stdbool.h> +#include <stddef.h> +#include <string.h> /** * Struct describing a Universal Unique Identifier lib/librte_eal/include/rte_vfio.h +2
-0@@ -16,6 +16,8 @@ extern "C" {#include <stdint.h> +#include <rte_uuid.h> +/* * determine if VFIO is present on the system */
lib/librte_eal/include/rte_vfio.h struct vfio_device_info; · line 108
Quoted 5 lines
lib/librte_eal/include/rte_vfio.h +2
-0@@ -102,13 +104,17 @@ struct vfio_device_info;* @param device_info * Device information. * +* @param vf_token +* VF token.
Such comments are useles and just eat space adding nothing useful. Please, make it useful and explain what is behind the parameter, when it is necessary, why? Should it be specified for PF case, VF case, both?
lib/librte_eal/include/rte_vfio.h struct vfio_device_info; · line 117
Quoted 10 lines
lib/librte_eal/include/rte_vfio.h +3
-1+* * @return * 0 on success. * <0 on failure. * >1 if the device cannot be managed this way. */ int rte_vfio_setup_device (const char *sysfs_base,const char *dev_addr, -int *vfio_dev_fd,struct vfio_device_info *device_info); +int *vfio_dev_fd,struct vfio_device_info *device_info, +rte_uuid_t vf_token);
"rte_uuid_t vf_token" looks confusing. Shouldn't it be "rte_uuid_t *vf_token"?
Quoted 18 lines
lib/librte_eal/include/rte_vfio.h +0
-0/** * Release a device mapped to a VFIO-managed I/O MMU group. lib/librte_eal/linux/eal_vfio.c +2
-1@@ -702,7 +702,8 @@ rte_vfio_clear_group(int vfio_group_fd)int rte_vfio_setup_device (const char *sysfs_base,const char *dev_addr, -int *vfio_dev_fd,struct vfio_device_info *device_info) +int *vfio_dev_fd,struct vfio_device_info *device_info, +rte_uuid_t vf_token) {struct vfio_group_status group_status = { .argsz =sizeof (group_status)
lib/librte_eal/linux/eal_vfio.c rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr, · line 716
Quoted 4 lines
lib/librte_eal/linux/eal_vfio.c +1
-0@@ -712,6 +713,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,int vfio_container_fd;int vfio_group_fd;int iommu_group_num; +char dev[PATH_MAX ];
Why PATH_MAX?
Quoted 35 lines
lib/librte_eal/linux/eal_vfio.c +14
-2int i, ret;/* get group number */ @@ -895,8 +897,19 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr, t->type_id, t->name); } +if (!rte_uuid_is_null (vf_token)) { +char vf_token_str[RTE_UUID_STRLEN ]; + +rte_uuid_unparse (vf_token, vf_token_str,sizeof (vf_token_str)); +snprintf (dev,sizeof (dev), +"%s vf_token=%s" , dev_addr, vf_token_str); + }else { +snprintf (dev,sizeof (dev), +"%s" , dev_addr); + } +/* get a file descriptor for the device */ - *vfio_dev_fd =ioctl (vfio_group_fd,VFIO_GROUP_GET_DEVICE_FD , dev_addr); + *vfio_dev_fd =ioctl (vfio_group_fd,VFIO_GROUP_GET_DEVICE_FD , dev);if (*vfio_dev_fd <0 ) {/* if we cannot get a device fd, this implies a problem with * the VFIO group or the container not having IOMMU configured. @@ -2081,7 +2094,8 @@ int rte_vfio_setup_device(__rte_unused const char *sysfs_base, __rte_unused const char *dev_addr, __rte_unused int *vfio_dev_fd, - __rte_unused struct vfio_device_info *device_info) + __rte_unused struct vfio_device_info *device_info, + __rte_unused rte_uuid_t vf_token) { return -1; }
Re: [dpdk-dev] [PATCH v4] eal: add VFIO-PCI SR-IOV support
Hi Andrew,
Quoted 109 lines
-----Original Message-----
From: Andrew Rybchenko <[email protected]>
Sent: Monday, April 13, 2020 23:38
To: Wang, Haiyue <[email protected]>; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]; [email protected]
Subject: Re: [dpdk-dev] [PATCH v4] eal: add VFIO-PCI SR-IOV supportQuoted 10 lines · Haiyue Wang
The kernel module vfio-pci introduces the VF token to enable SR-IOV support since 5.7.
The VF token can be set by a vfio-pci based PF driver and must be known by the vfio-pci based VF driver in order to gain access to the device.
An example VF token option would take this form:
1. Install vfio-pci with option 'enable_sriov=1'
Quoted 8 lines
2.
./usertools/dpdk-devbind.py-b vfio-pci 0000:87:00.03. echo 2 > /sys/bus/pci/devices/0000:87:00.0/sriov_numvfs
4. Start the PF:
./x86_64-native-linux-gcc/app/testpmd -l 22-25 -n 4 \ -w 87:00.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 \ --file-prefix=pf -- -iShould I get a token from my head? Any?
Quoted 80 lines
5. Start the VF:
./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 \ -w 87:02.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 \ --file-prefix=vf1 -- -iSigned-off-by Haiyue Wang <[email protected]>Acked-by Vamsi Attunuru <[email protected]>---
v4: 1. Ignorerte_vfio_setup_deviceABI check since it is
for Linux driver use.v3: https://patchwork.dpdk.org/patch/68254/
Fix the Travis build failed:
(1). rte_uuid.h:97:55: error: unknown type name ‘size_t’ (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’v2: https://patchwork.dpdk.org/patch/68240/
Fix the FreeBSD build error.v1: https://patchwork.dpdk.org/patch/68237/
Update the commit message.RFC v2: https://patchwork.dpdk.org/patch/68114/
Based on Vamsi's RFC v1, and Alex's patch for Qemu
[https://lore.kernel.org/lkml/[email protected]/]:
Use the devarg to pass-down the VF token.RFC v1: https://patchwork.dpdk.org/patch/66281/ by Vamsi.
--- devtools/libabigail.abignore | 3 ++ drivers/bus/pci/linux/pci_vfio.c | 56 +++++++++++++++++++++++++++++-- lib/librte_eal/freebsd/eal.c | 3 +- lib/librte_eal/include/rte_uuid.h | 2 ++ lib/librte_eal/include/rte_vfio.h | 8 ++++- lib/librte_eal/linux/eal_vfio.c | 20 +++++++++-- 6 files changed, 85 insertions(+), 7 deletions(-)devtools/libabigail.abignore +3
-0@@ -11,3 +11,6 @@ type_kind = enum name = rte_crypto_asym_xform_type changed_enumerators = RTE_CRYPTO_ASYM_XFORM_TYPE_LIST_END +; Ignore this function which is only relevant to linux for driver +[suppress_type] + name = rte_vfio_setup_devicedrivers/bus/pci/linux/pci_vfio.c +17
-0@@ -11,6 +11,7 @@#include <sys/mman.h> #include <stdbool.h> +#include <rte_devargs.h> #include <rte_log.h> #include <rte_pci.h> #include <rte_bus_pci.h> @@ -644,11 +645,59 @@ pci_vfio_msix_is_mappable(int vfio_dev_fd, int msix_region)return ret; } +static void +vfio_pci_vf_token_arg (struct rte_devargs *devargs,rte_uuid_t uu) +{ +#define VF_TOKEN_ARG "vf_token=" +char c, *p, *vf_token; + +if (devargs ==NULL ) +return ; + + p =strstr (devargs->args,VF_TOKEN_ARG ); +if (!p) +return ; + + vf_token = p +strlen (VF_TOKEN_ARG ); +if (strlen (vf_token) < (RTE_UUID_STRLEN -1 )) +return ;
Quoted 9 lines
drivers/bus/pci/linux/pci_vfio.c pci_vfio_msix_is_mappable(int vfio_dev_fd, int msix_region) · line 669
Quoted 6 lines
drivers/bus/pci/linux/pci_vfio.c +6
-0+ + c = vf_token[RTE_UUID_STRLEN -1 ]; +if (c !='\0' && c !=',' ) +return ; + + vf_token[RTE_UUID_STRLEN -1 ] ='\0' ;Is it possible to parse and handle devargs using
rte_kvargs.h?
Since it needs to remove the 'vf_token', as 'vf_token' is not a valid PMD related args, so need to parse and delete it.
rte_kvargs_parse(const char *args, const char * const valid_keys[])
Quoted 10 lines
drivers/bus/pci/linux/pci_vfio.c pci_vfio_msix_is_mappable(int vfio_dev_fd, int msix_region) · line 674
Quoted 5 lines
drivers/bus/pci/linux/pci_vfio.c +5
-0+if (rte_uuid_parse (vf_token, uu)) { +RTE_LOG (ERR ,EAL , +"The VF token is not a valid uuid : %s\n" , vf_token); + vf_token[RTE_UUID_STRLEN -1 ] = c; +return ;I think that the function must return error which is handled by the caller when something bad happens (e.g. invalid UUID).
Yes, make sense, will add the error handling.
Quoted 14 lines
Quoted 14 lines
drivers/bus/pci/linux/pci_vfio.c +14
-0+ } + +RTE_LOG (DEBUG ,EAL , +"The VF token is found : %s\n" , vf_token); + + vf_token[RTE_UUID_STRLEN -1 ] = c; + +/* Purge this vfio-pci specific token from the device arguments */ +if (c !='\0' ) { +/* 1. Handle the case : 'vf_token=uuid,arg1=val1' */ +memmove (p, vf_token +RTE_UUID_STRLEN , +strlen (vf_token +RTE_UUID_STRLEN ) +1 ); + }else { +/* 2. Handle the case : 'arg1=val1,vf_token=uuid' */
Quoted 8 lines
drivers/bus/pci/linux/pci_vfio.c pci_vfio_msix_is_mappable(int vfio_dev_fd, int msix_region) · line 693
Quoted 5 lines
drivers/bus/pci/linux/pci_vfio.c +5
-0+if (p != devargs->args) + p--; + + *p ='\0' ; + }Is it really required to purge? Why? If yes, it should be explained in the comment above.
Please see above reply.
Quoted 12 lines
drivers/bus/pci/linux/pci_vfio.c pci_vfio_msix_is_mappable(int vfio_dev_fd, int msix_region) · line 700
Quoted 8 lines
drivers/bus/pci/linux/pci_vfio.c +2
-0+}static int pci_vfio_map_resource_primary (struct rte_pci_device *dev) {struct vfio_device_info device_info = { .argsz =sizeof (device_info) }; +rte_uuid_t vf_token =RTE_UUID_INIT (0 ,0 ,0 ,0 ,0ULL );May be it would be better if
vfio_pci_vf_token_arg()initializes it anyway instead of duplication init in two places?
+1, will update it.
Quoted 69 lines
Quoted 69 lines
drivers/bus/pci/linux/pci_vfio.c +5
-2char pci_addr[PATH_MAX ] = {0 };int vfio_dev_fd;struct rte_pci_addr *loc = &dev->addr;@@ -668,8 +717,9 @@ pci_vfio_map_resource_primary(struct rte_pci_device *dev)snprintf (pci_addr,sizeof (pci_addr),PCI_PRI_FMT , loc->domain, loc->bus, loc->devid, loc->function); +vfio_pci_vf_token_arg (dev->device.devargs, vf_token); ret =rte_vfio_setup_device (rte_pci_get_sysfs_path (), pci_addr, - &vfio_dev_fd, &device_info); + &vfio_dev_fd, &device_info, vf_token);if (ret)return ret;@@ -797,6 +847,7 @@ static int pci_vfio_map_resource_secondary(struct rte_pci_device *dev) {struct vfio_device_info device_info = { .argsz =sizeof (device_info) }; +rte_uuid_t vf_token =RTE_UUID_INIT (0 ,0 ,0 ,0 ,0ULL );char pci_addr[PATH_MAX ] = {0 };int vfio_dev_fd;struct rte_pci_addr *loc = &dev->addr;@@ -830,8 +881,9 @@ pci_vfio_map_resource_secondary(struct rte_pci_device *dev)return -1 ; } +vfio_pci_vf_token_arg (dev->device.devargs, vf_token); ret =rte_vfio_setup_device (rte_pci_get_sysfs_path (), pci_addr, - &vfio_dev_fd, &device_info); + &vfio_dev_fd, &device_info, vf_token);if (ret)return ret;lib/librte_eal/freebsd/eal.c +2
-1@@ -995,7 +995,8 @@ rte_eal_vfio_intr_mode(void) int rte_vfio_setup_device(__rte_unused const char *sysfs_base, __rte_unused const char *dev_addr, __rte_unused int *vfio_dev_fd, - __rte_unused struct vfio_device_info *device_info) + __rte_unused struct vfio_device_info *device_info, + __rte_unused rte_uuid_t vf_token) {return -1 ; }lib/librte_eal/include/rte_uuid.h +2
-0@@ -15,6 +15,8 @@ extern "C" {#endif #include <stdbool.h> +#include <stddef.h> +#include <string.h> /** * Struct describing a Universal Unique Identifier lib/librte_eal/include/rte_vfio.h +0
-0@@ -16,6 +16,8 @@ extern "C" {#include <stdint.h>
Quoted 17 lines
lib/librte_eal/include/rte_vfio.h struct vfio_device_info; · line 108
Quoted 11 lines
lib/librte_eal/include/rte_vfio.h +4
-0+#include <rte_uuid.h> +/* * determine if VFIO is present on the system */ @@ -102,13 +104,17 @@ struct vfio_device_info;* @param device_info * Device information. * +* @param vf_token +* VF token. Such comments are useles and just eat space adding nothing useful. Please, make it useful and explain what is behind the parameter, when it is necessary, why? Should it be specified for PF case, VF case, both?
Will add more comments, yes for both PF and VF, as Alex's linux patch explained.
Quoted 5 lines
Quoted 5 lines
lib/librte_eal/include/rte_vfio.h +1
-0+* * @return * 0 on success. * <0 on failure. * >1 if the device cannot be managed this way.
Quoted 8 lines
lib/librte_eal/include/rte_vfio.h struct vfio_device_info; · line 117
Quoted 5 lines
lib/librte_eal/include/rte_vfio.h +2
-1*/ int rte_vfio_setup_device (const char *sysfs_base,const char *dev_addr, -int *vfio_dev_fd,struct vfio_device_info *device_info); +int *vfio_dev_fd,struct vfio_device_info *device_info, +rte_uuid_t vf_token);"
rte_uuid_tvf_token" looks confusing. Shouldn't it be "rte_uuid_t*vf_token"?
This is UUID API design and type definition:
bool rte_uuid_is_null(const rte_uuid_t uu);
DPDK: typedef unsigned char rte_uuid_t[16];
vs
Linux: typedef struct {
__u8 b[UUID_SIZE];
} uuid_t;Quoted 10 lines
Quoted 10 lines
lib/librte_eal/include/rte_vfio.h +0
-0/** * Release a device mapped to a VFIO-managed I/O MMU group. lib/librte_eal/linux/eal_vfio.c +0
-0@@ -702,7 +702,8 @@ rte_vfio_clear_group(int vfio_group_fd)
Quoted 15 lines
lib/librte_eal/linux/eal_vfio.c rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr, · line 716
Quoted 13 lines
lib/librte_eal/linux/eal_vfio.c +3
-1int rte_vfio_setup_device (const char *sysfs_base,const char *dev_addr, -int *vfio_dev_fd,struct vfio_device_info *device_info) +int *vfio_dev_fd,struct vfio_device_info *device_info, +rte_uuid_t vf_token) {struct vfio_group_status group_status = { .argsz =sizeof (group_status)@@ -712,6 +713,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,int vfio_container_fd;int vfio_group_fd;int iommu_group_num; +char dev[PATH_MAX ];Why
PATH_MAX?
Based on Vamsi's RFC v1, and found that it looked a little reasonable, ' char pci_addr[PATH_MAX] = {0}; '
static intpci_vfio_map_resource_primary(struct rte_pci_device *dev)
{
struct vfio_device_info device_info = { .argsz = sizeof(device_info) };
char pci_addr[PATH_MAX] = {0}; <----Quoted 36 lines
Quoted 36 lines
lib/librte_eal/linux/eal_vfio.c +14
-2int i, ret;/* get group number */ @@ -895,8 +897,19 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr, t->type_id, t->name); } +if (!rte_uuid_is_null (vf_token)) { +char vf_token_str[RTE_UUID_STRLEN ]; + +rte_uuid_unparse (vf_token, vf_token_str,sizeof (vf_token_str)); +snprintf (dev,sizeof (dev), +"%s vf_token=%s" , dev_addr, vf_token_str); + }else { +snprintf (dev,sizeof (dev), +"%s" , dev_addr); + } +/* get a file descriptor for the device */ - *vfio_dev_fd =ioctl (vfio_group_fd,VFIO_GROUP_GET_DEVICE_FD , dev_addr); + *vfio_dev_fd =ioctl (vfio_group_fd,VFIO_GROUP_GET_DEVICE_FD , dev);if (*vfio_dev_fd <0 ) {/* if we cannot get a device fd, this implies a problem with * the VFIO group or the container not having IOMMU configured. @@ -2081,7 +2094,8 @@ int rte_vfio_setup_device(__rte_unused const char *sysfs_base, __rte_unused const char *dev_addr, __rte_unused int *vfio_dev_fd, - __rte_unused struct vfio_device_info *device_info) + __rte_unused struct vfio_device_info *device_info, + __rte_unused rte_uuid_t vf_token) { return -1; }
v5 cover support for VFIO-PCI VF token interface
In reply to [email protected]v5: 1. Add the VF token parse error handling.
2. Split into two patches for different logic module.
3. Add more comments into the code for explaining the design.
4. Drop the ABI change workaround, this patch set focuses on code review.
v4: 1. Ignore rte_vfio_setup_device ABI check since it is
for Linux driver use.
https://patchwork.dpdk.org/patch/68255/
v3: Fix the Travis build failed:
(1). rte_uuid.h:97:55: error: unknown type name ‘size_t’ (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’
https://patchwork.dpdk.org/patch/68254/
v2: Fix the FreeBSD build error.
https://patchwork.dpdk.org/patch/68240/
v1: Update the commit message.
https://patchwork.dpdk.org/patch/68237/
RFC v2: https://patchwork.dpdk.org/patch/68114/
Based on Vamsi's RFC v1, and Alex's patch for Qemu
[https://lore.kernel.org/lkml/[email protected]/]:
Use the devarg to pass-down the VF token.
RFC v1: https://patchwork.dpdk.org/patch/66281/ by Vamsi.
Haiyue Wang (2):
eal: add uuid dependent header files explicitly
eal: support for VFIO-PCI VF token
drivers/bus/pci/linux/pci_vfio.c | 74 ++++++++++++++++++++++++++++++- lib/librte_eal/freebsd/eal.c | 3 +- lib/librte_eal/include/rte_uuid.h | 2 + lib/librte_eal/include/rte_vfio.h | 21 ++++++++- lib/librte_eal/linux/eal_vfio.c | 20 +++++++-- 5 files changed, 113 insertions(+), 7 deletions(-)
Signature · 1 line
2.26.0
PATCH v5 1/2 eal: add uuid dependent header files explicitly
Add the dependent header files explicitly, so that the user just needs to include the 'rte_uuid.h' header file directly to avoid compile error:
(1). rte_uuid.h:97:55: error: unknown type name ‘size_t’ (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’
--- lib/librte_eal/include/rte_uuid.h | 2 ++ 1 file changed, 2 insertions(+)
lib/librte_eal/include/rte_uuid.h +2 -0
@@ -15,6 +15,8 @@ extern "C" {#endif #include <stdbool.h> +#include <stddef.h> +#include <string.h> /** * Struct describing a Universal Unique Identifier
Signature · 1 line
2.26.0
PATCH v5 2/2 eal: support for VFIO-PCI VF token
In reply to Haiyue WangThe kernel module vfio-pci introduces the VF token to enable SR-IOV support since 5.7.
The VF token can be set by a vfio-pci based PF driver and must be known by the vfio-pci based VF driver in order to gain access to the device.
An example VF token option would take this form:
1. Install vfio-pci with option 'enable_sriov=1'
2. ./usertools/dpdk-devbind.py -b vfio-pci 0000:87:00.0
3. echo 2 > /sys/bus/pci/devices/0000:87:00.0/sriov_numvfs
4. Start the PF:
./x86_64-native-linux-gcc/app/testpmd -l 22-25 -n 4 \
-w 87:00.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 \
--file-prefix=pf -- -i5. Start the VF:
./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 \
-w 87:02.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 \
--file-prefix=vf1 -- -i--- drivers/bus/pci/linux/pci_vfio.c | 74 ++++++++++++++++++++++++++++++- lib/librte_eal/freebsd/eal.c | 3 +- lib/librte_eal/include/rte_vfio.h | 21 ++++++++- lib/librte_eal/linux/eal_vfio.c | 20 +++++++-- 4 files changed, 111 insertions(+), 7 deletions(-)
drivers/bus/pci/linux/pci_vfio.c +72 -2
@@ -11,6 +11,7 @@#include <sys/mman.h> #include <stdbool.h> +#include <rte_devargs.h> #include <rte_log.h> #include <rte_pci.h> #include <rte_bus_pci.h>
@@ -644,12 +645,72 @@ pci_vfio_msix_is_mappable(int vfio_dev_fd, int msix_region)return ret; } +static int +vfio_pci_vf_token_arg (struct rte_devargs *devargs,rte_uuid_t uuid) +{ +#define VF_TOKEN_ARG "vf_token=" +char c, *p, *vf_token; + +memset (uuid,0 ,sizeof (rte_uuid_t)); + +if (devargs ==NULL ) +return 0 ; + + p =strstr (devargs->args,VF_TOKEN_ARG ); +if (!p) +return 0 ; + + vf_token = p +strlen (VF_TOKEN_ARG ); +if (strlen (vf_token) < (RTE_UUID_STRLEN -1 )) { +RTE_LOG (ERR ,EAL ,"The VF token length is too short\n" ); +return -1 ; + } + + c = vf_token[RTE_UUID_STRLEN -1 ]; +if (c !='\0' && c !=',' ) { +RTE_LOG (ERR ,EAL , +"The VF token ends with a invalid character : %c\n" , c); +return -1 ; + } + + vf_token[RTE_UUID_STRLEN -1 ] ='\0' ; +if (rte_uuid_parse (vf_token, uuid)) { +RTE_LOG (ERR ,EAL , +"The VF token is invalid : %s\n" , vf_token); + vf_token[RTE_UUID_STRLEN -1 ] = c; +return -1 ; + } + +RTE_LOG (DEBUG ,EAL , +"The VF token is found : %s\n" , vf_token); + + vf_token[RTE_UUID_STRLEN -1 ] = c; + +/* This VF token will be treated as a invalid device argument if the +* PMD calls the rte_devargs parse API with its own valid argument list, +* so it needs to purge this vfio-pci specific argument. +*/ +if (c !='\0' ) { +/* 1. Handle the case : 'vf_token=uuid,arg1=val1' */ +memmove (p, vf_token +RTE_UUID_STRLEN , +strlen (vf_token +RTE_UUID_STRLEN ) +1 ); + }else { +/* 2. Handle the case : 'arg1=val1,vf_token=uuid' */ +if (p != devargs->args) + p--; + + *p ='\0' ; + } + +return 0 ; +}static int pci_vfio_map_resource_primary (struct rte_pci_device *dev) {struct vfio_device_info device_info = { .argsz =sizeof (device_info) };char pci_addr[PATH_MAX ] = {0 }; +rte_uuid_t vf_token;int vfio_dev_fd;struct rte_pci_addr *loc = &dev->addr;int i, ret;
@@ -668,8 +729,12 @@ pci_vfio_map_resource_primary(struct rte_pci_device *dev)snprintf (pci_addr,sizeof (pci_addr),PCI_PRI_FMT , loc->domain, loc->bus, loc->devid, loc->function); + ret =vfio_pci_vf_token_arg (dev->device.devargs, vf_token); +if (ret) +return ret; + ret =rte_vfio_setup_device (rte_pci_get_sysfs_path (), pci_addr, - &vfio_dev_fd, &device_info); + &vfio_dev_fd, &device_info, vf_token);if (ret)return ret;
@@ -798,6 +863,7 @@ pci_vfio_map_resource_secondary(struct rte_pci_device *dev) {struct vfio_device_info device_info = { .argsz =sizeof (device_info) };char pci_addr[PATH_MAX ] = {0 }; +rte_uuid_t vf_token;int vfio_dev_fd;struct rte_pci_addr *loc = &dev->addr;int i, ret;
@@ -830,8 +896,12 @@ pci_vfio_map_resource_secondary(struct rte_pci_device *dev)return -1 ; } + ret =vfio_pci_vf_token_arg (dev->device.devargs, vf_token); +if (ret) +return ret; + ret =rte_vfio_setup_device (rte_pci_get_sysfs_path (), pci_addr, - &vfio_dev_fd, &device_info); + &vfio_dev_fd, &device_info, vf_token);if (ret)return ret;
lib/librte_eal/freebsd/eal.c +2 -1
@@ -995,7 +995,8 @@ rte_eal_vfio_intr_mode(void) int rte_vfio_setup_device(__rte_unused const char *sysfs_base, __rte_unused const char *dev_addr, __rte_unused int *vfio_dev_fd, - __rte_unused struct vfio_device_info *device_info) + __rte_unused struct vfio_device_info *device_info, + __rte_unused rte_uuid_t vf_token) {return -1 ; }
lib/librte_eal/include/rte_vfio.h +20 -1
@@ -16,6 +16,8 @@ extern "C" {#include <stdint.h> +#include <rte_uuid.h> +/* * determine if VFIO is present on the system */
@@ -102,13 +104,30 @@ struct vfio_device_info;* @param device_info * Device information. * +* @param vf_token +* Before linux 5.7, the PF bound to vfio-pci doesn't support SR-IOV to +* create VFs for security reason. Now the VF token is introduced to work +* as some degree of trust or collaboration between PF and VFs. +* +* A). as VF device, if the PF is a vfio device and it is bound to the +* vfio-pci driver, the user needs to provide a VF token to access the +* device, in the form of appending a vf_token to the device name, for +* example: +* "0000:04:10.0 vf_token=bd8d9d2b-5a5f-4f5a-a211-f591514ba1f3" +* +* B). as PF device, When presented with a PF which has VFs in use, the +* user must also provide the current VF token to prove collaboration with +* existing VF users. If VFs are not in use, the VF token provided for the +* PF device will act to set the VF token. +* * @return * 0 on success. * <0 on failure. * >1 if the device cannot be managed this way. */ int rte_vfio_setup_device (const char *sysfs_base,const char *dev_addr, -int *vfio_dev_fd,struct vfio_device_info *device_info); +int *vfio_dev_fd,struct vfio_device_info *device_info, +rte_uuid_t vf_token);/** * Release a device mapped to a VFIO-managed I/O MMU group.
lib/librte_eal/linux/eal_vfio.c +17 -3
@@ -702,7 +702,8 @@ rte_vfio_clear_group(int vfio_group_fd)int rte_vfio_setup_device (const char *sysfs_base,const char *dev_addr, -int *vfio_dev_fd,struct vfio_device_info *device_info) +int *vfio_dev_fd,struct vfio_device_info *device_info, +rte_uuid_t vf_token) {struct vfio_group_status group_status = { .argsz =sizeof (group_status)
@@ -712,6 +713,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,int vfio_container_fd;int vfio_group_fd;int iommu_group_num; +char dev[PATH_MAX ];int i, ret;/* get group number */
@@ -895,8 +897,19 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr, t->type_id, t->name); } +if (!rte_uuid_is_null (vf_token)) { +char vf_token_str[RTE_UUID_STRLEN ]; + +rte_uuid_unparse (vf_token, vf_token_str,sizeof (vf_token_str)); +snprintf (dev,sizeof (dev), +"%s vf_token=%s" , dev_addr, vf_token_str); + }else { +snprintf (dev,sizeof (dev), +"%s" , dev_addr); + } +/* get a file descriptor for the device */ - *vfio_dev_fd =ioctl (vfio_group_fd,VFIO_GROUP_GET_DEVICE_FD , dev_addr); + *vfio_dev_fd =ioctl (vfio_group_fd,VFIO_GROUP_GET_DEVICE_FD , dev);if (*vfio_dev_fd <0 ) {/* if we cannot get a device fd, this implies a problem with * the VFIO group or the container not having IOMMU configured.
@@ -2081,7 +2094,8 @@ int rte_vfio_setup_device(__rte_unused const char *sysfs_base, __rte_unused const char *dev_addr, __rte_unused int *vfio_dev_fd, - __rte_unused struct vfio_device_info *device_info) + __rte_unused struct vfio_device_info *device_info, + __rte_unused rte_uuid_t vf_token) { return -1; }
Signature · 1 line
2.26.0
v6 cover support for VFIO-PCI VF token interface
In reply to [email protected]v6: Drop the Fixes tag in uuid, since the file has been
moved to another place, not suitable to apply on stable. And this is not a bug, just some kind of enhancement.
v5: 1. Add the VF token parse error handling.
2. Split into two patches for different logic module.
3. Add more comments into the code for explaining the design.
4. Drop the ABI change workaround, this patch set focuses on code review.
https://patchwork.dpdk.org/cover/68364/
v4: 1. Ignore rte_vfio_setup_device ABI check since it is
for Linux driver use.
https://patchwork.dpdk.org/patch/68255/
v3: Fix the Travis build failed:
(1). rte_uuid.h:97:55: error: unknown type name ‘size_t’ (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’
https://patchwork.dpdk.org/patch/68254/
v2: Fix the FreeBSD build error.
https://patchwork.dpdk.org/patch/68240/
v1: Update the commit message.
https://patchwork.dpdk.org/patch/68237/
RFC v2: https://patchwork.dpdk.org/patch/68114/
Based on Vamsi's RFC v1, and Alex's patch for Qemu
[https://lore.kernel.org/lkml/[email protected]/]:
Use the devarg to pass-down the VF token.
RFC v1: https://patchwork.dpdk.org/patch/66281/ by Vamsi.
Haiyue Wang (2):
eal: add uuid dependent header files explicitly
eal: support for VFIO-PCI VF token
drivers/bus/pci/linux/pci_vfio.c | 74 ++++++++++++++++++++++++++++++- lib/librte_eal/freebsd/eal.c | 3 +- lib/librte_eal/include/rte_uuid.h | 2 + lib/librte_eal/include/rte_vfio.h | 21 ++++++++- lib/librte_eal/linux/eal_vfio.c | 20 +++++++-- 5 files changed, 113 insertions(+), 7 deletions(-)
Signature · 1 line
2.26.0
PATCH v6 1/2 eal: add uuid dependent header files explicitly
Add the dependent header files explicitly, so that the user just needs to include the 'rte_uuid.h' header file directly to avoid compile error:
(1). rte_uuid.h:97:55: error: unknown type name ‘size_t’ (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’
--- lib/librte_eal/include/rte_uuid.h | 2 ++ 1 file changed, 2 insertions(+)
lib/librte_eal/include/rte_uuid.h +2 -0
@@ -15,6 +15,8 @@ extern "C" {#endif #include <stdbool.h> +#include <stddef.h> +#include <string.h> /** * Struct describing a Universal Unique Identifier
Signature · 1 line
2.26.0
PATCH v6 2/2 eal: support for VFIO-PCI VF token
In reply to Haiyue WangThe kernel module vfio-pci introduces the VF token to enable SR-IOV support since 5.7.
The VF token can be set by a vfio-pci based PF driver and must be known by the vfio-pci based VF driver in order to gain access to the device.
An example VF token option would take this form:
1. Install vfio-pci with option 'enable_sriov=1'
2. ./usertools/dpdk-devbind.py -b vfio-pci 0000:87:00.0
3. echo 2 > /sys/bus/pci/devices/0000:87:00.0/sriov_numvfs
4. Start the PF:
./x86_64-native-linux-gcc/app/testpmd -l 22-25 -n 4 \
-w 87:00.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 \
--file-prefix=pf -- -i5. Start the VF:
./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 \
-w 87:02.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 \
--file-prefix=vf1 -- -i--- drivers/bus/pci/linux/pci_vfio.c | 74 ++++++++++++++++++++++++++++++- lib/librte_eal/freebsd/eal.c | 3 +- lib/librte_eal/include/rte_vfio.h | 21 ++++++++- lib/librte_eal/linux/eal_vfio.c | 20 +++++++-- 4 files changed, 111 insertions(+), 7 deletions(-)
drivers/bus/pci/linux/pci_vfio.c +72 -2
@@ -11,6 +11,7 @@#include <sys/mman.h> #include <stdbool.h> +#include <rte_devargs.h> #include <rte_log.h> #include <rte_pci.h> #include <rte_bus_pci.h>
@@ -644,12 +645,72 @@ pci_vfio_msix_is_mappable(int vfio_dev_fd, int msix_region)return ret; } +static int +vfio_pci_vf_token_arg (struct rte_devargs *devargs,rte_uuid_t uuid) +{ +#define VF_TOKEN_ARG "vf_token=" +char c, *p, *vf_token; + +memset (uuid,0 ,sizeof (rte_uuid_t)); + +if (devargs ==NULL ) +return 0 ; + + p =strstr (devargs->args,VF_TOKEN_ARG ); +if (!p) +return 0 ; + + vf_token = p +strlen (VF_TOKEN_ARG ); +if (strlen (vf_token) < (RTE_UUID_STRLEN -1 )) { +RTE_LOG (ERR ,EAL ,"The VF token length is too short\n" ); +return -1 ; + } + + c = vf_token[RTE_UUID_STRLEN -1 ]; +if (c !='\0' && c !=',' ) { +RTE_LOG (ERR ,EAL , +"The VF token ends with a invalid character : %c\n" , c); +return -1 ; + } + + vf_token[RTE_UUID_STRLEN -1 ] ='\0' ; +if (rte_uuid_parse (vf_token, uuid)) { +RTE_LOG (ERR ,EAL , +"The VF token is invalid : %s\n" , vf_token); + vf_token[RTE_UUID_STRLEN -1 ] = c; +return -1 ; + } + +RTE_LOG (DEBUG ,EAL , +"The VF token is found : %s\n" , vf_token); + + vf_token[RTE_UUID_STRLEN -1 ] = c; + +/* This VF token will be treated as a invalid device argument if the +* PMD calls the rte_devargs parse API with its own valid argument list, +* so it needs to purge this vfio-pci specific argument. +*/ +if (c !='\0' ) { +/* 1. Handle the case : 'vf_token=uuid,arg1=val1' */ +memmove (p, vf_token +RTE_UUID_STRLEN , +strlen (vf_token +RTE_UUID_STRLEN ) +1 ); + }else { +/* 2. Handle the case : 'arg1=val1,vf_token=uuid' */ +if (p != devargs->args) + p--; + + *p ='\0' ; + } + +return 0 ; +}static int pci_vfio_map_resource_primary (struct rte_pci_device *dev) {struct vfio_device_info device_info = { .argsz =sizeof (device_info) };char pci_addr[PATH_MAX ] = {0 }; +rte_uuid_t vf_token;int vfio_dev_fd;struct rte_pci_addr *loc = &dev->addr;int i, ret;
@@ -668,8 +729,12 @@ pci_vfio_map_resource_primary(struct rte_pci_device *dev)snprintf (pci_addr,sizeof (pci_addr),PCI_PRI_FMT , loc->domain, loc->bus, loc->devid, loc->function); + ret =vfio_pci_vf_token_arg (dev->device.devargs, vf_token); +if (ret) +return ret; + ret =rte_vfio_setup_device (rte_pci_get_sysfs_path (), pci_addr, - &vfio_dev_fd, &device_info); + &vfio_dev_fd, &device_info, vf_token);if (ret)return ret;
@@ -798,6 +863,7 @@ pci_vfio_map_resource_secondary(struct rte_pci_device *dev) {struct vfio_device_info device_info = { .argsz =sizeof (device_info) };char pci_addr[PATH_MAX ] = {0 }; +rte_uuid_t vf_token;int vfio_dev_fd;struct rte_pci_addr *loc = &dev->addr;int i, ret;
@@ -830,8 +896,12 @@ pci_vfio_map_resource_secondary(struct rte_pci_device *dev)return -1 ; } + ret =vfio_pci_vf_token_arg (dev->device.devargs, vf_token); +if (ret) +return ret; + ret =rte_vfio_setup_device (rte_pci_get_sysfs_path (), pci_addr, - &vfio_dev_fd, &device_info); + &vfio_dev_fd, &device_info, vf_token);if (ret)return ret;
lib/librte_eal/freebsd/eal.c +2 -1
@@ -995,7 +995,8 @@ rte_eal_vfio_intr_mode(void) int rte_vfio_setup_device(__rte_unused const char *sysfs_base, __rte_unused const char *dev_addr, __rte_unused int *vfio_dev_fd, - __rte_unused struct vfio_device_info *device_info) + __rte_unused struct vfio_device_info *device_info, + __rte_unused rte_uuid_t vf_token) {return -1 ; }
lib/librte_eal/include/rte_vfio.h +20 -1
@@ -16,6 +16,8 @@ extern "C" {#include <stdint.h> +#include <rte_uuid.h> +/* * determine if VFIO is present on the system */
@@ -102,13 +104,30 @@ struct vfio_device_info;* @param device_info * Device information. * +* @param vf_token +* Before linux 5.7, the PF bound to vfio-pci doesn't support SR-IOV to +* create VFs for security reason. Now the VF token is introduced to work +* as some degree of trust or collaboration between PF and VFs. +* +* A). as VF device, if the PF is a vfio device and it is bound to the +* vfio-pci driver, the user needs to provide a VF token to access the +* device, in the form of appending a vf_token to the device name, for +* example: +* "0000:04:10.0 vf_token=bd8d9d2b-5a5f-4f5a-a211-f591514ba1f3" +* +* B). as PF device, When presented with a PF which has VFs in use, the +* user must also provide the current VF token to prove collaboration with +* existing VF users. If VFs are not in use, the VF token provided for the +* PF device will act to set the VF token. +* * @return * 0 on success. * <0 on failure. * >1 if the device cannot be managed this way. */ int rte_vfio_setup_device (const char *sysfs_base,const char *dev_addr, -int *vfio_dev_fd,struct vfio_device_info *device_info); +int *vfio_dev_fd,struct vfio_device_info *device_info, +rte_uuid_t vf_token);/** * Release a device mapped to a VFIO-managed I/O MMU group.
lib/librte_eal/linux/eal_vfio.c +17 -3
@@ -702,7 +702,8 @@ rte_vfio_clear_group(int vfio_group_fd)int rte_vfio_setup_device (const char *sysfs_base,const char *dev_addr, -int *vfio_dev_fd,struct vfio_device_info *device_info) +int *vfio_dev_fd,struct vfio_device_info *device_info, +rte_uuid_t vf_token) {struct vfio_group_status group_status = { .argsz =sizeof (group_status)
@@ -712,6 +713,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,int vfio_container_fd;int vfio_group_fd;int iommu_group_num; +char dev[PATH_MAX ];int i, ret;/* get group number */
@@ -895,8 +897,19 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr, t->type_id, t->name); } +if (!rte_uuid_is_null (vf_token)) { +char vf_token_str[RTE_UUID_STRLEN ]; + +rte_uuid_unparse (vf_token, vf_token_str,sizeof (vf_token_str)); +snprintf (dev,sizeof (dev), +"%s vf_token=%s" , dev_addr, vf_token_str); + }else { +snprintf (dev,sizeof (dev), +"%s" , dev_addr); + } +/* get a file descriptor for the device */ - *vfio_dev_fd =ioctl (vfio_group_fd,VFIO_GROUP_GET_DEVICE_FD , dev_addr); + *vfio_dev_fd =ioctl (vfio_group_fd,VFIO_GROUP_GET_DEVICE_FD , dev);if (*vfio_dev_fd <0 ) {/* if we cannot get a device fd, this implies a problem with * the VFIO group or the container not having IOMMU configured.
@@ -2081,7 +2094,8 @@ int rte_vfio_setup_device(__rte_unused const char *sysfs_base, __rte_unused const char *dev_addr, __rte_unused int *vfio_dev_fd, - __rte_unused struct vfio_device_info *device_info) + __rte_unused struct vfio_device_info *device_info, + __rte_unused rte_uuid_t vf_token) { return -1; }
Signature · 1 line
2.26.0
Re: [dpdk-dev] [EXT] [PATCH v6 0/2] support for VFIO-PCI VF token interface
In reply to Haiyue WangQuoted 90 lines
-----Original Message-----
From: Haiyue Wang <[email protected]>
Sent: Tuesday, April 14, 2020 8:51 AM
To: [email protected]; [email protected]; Vamsi Krishna Attunuru <[email protected]>; Jerin Jacob Kollanukkaran <[email protected]>; [email protected]; [email protected]
Cc: Haiyue Wang <[email protected]>
Subject: [EXT] [PATCH v6 0/2] support for VFIO-PCI VF token interfaceExternal Email
----------------------------------------------------------------------
v6: Drop the Fixes tag in uuid, since the file has been
moved to another place, not suitable to apply on stable. And this is not a bug, just some kind of enhancement.v5: 1. Add the VF token parse error handling.
2. Split into two patches for different logic module.
3. Add more comments into the code for explaining the design.
4. Drop the ABI change workaround, this patch set focuses on code review.
https://urldefense.proofpoint.com/v2/url?u=https-
3A__patchwork.dpdk.org_cover_68364_&d=DwIDaQ&c=nKjWec2b6R0mOyP az7xtfQ&r=WllrYaumVkxaWjgKto6E_rtDQshhIhik2jkvzFyRhW8&m=SxW07R9 V0iyd_fAAZm4bU3jimkCD0khdQeabCt9pESg&s=mAkB5KtaFDGFdmqfNZV4dy w1A8Xy61tm22TNM8kGSrQ&e=v4: 1. Ignore
rte_vfio_setup_deviceABI check since it is
for Linux driver use.
https://urldefense.proofpoint.com/v2/url?u=https-
3A__patchwork.dpdk.org_patch_68255_&d=DwIDaQ&c=nKjWec2b6R0mOyP az7xtfQ&r=WllrYaumVkxaWjgKto6E_rtDQshhIhik2jkvzFyRhW8&m=SxW07R9 V0iyd_fAAZm4bU3jimkCD0khdQeabCt9pESg&s=Qv5ArXmk_WeQn1bTnM6JS XpNp3NEhuenHP1VsjUPzGE&e=v3: Fix the Travis build failed:
(1). rte_uuid.h:97:55: error: unknown type name ‘size_t’ (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’
https://urldefense.proofpoint.com/v2/url?u=https-
3A__patchwork.dpdk.org_patch_68254_&d=DwIDaQ&c=nKjWec2b6R0mOyP az7xtfQ&r=WllrYaumVkxaWjgKto6E_rtDQshhIhik2jkvzFyRhW8&m=SxW07R9 V0iyd_fAAZm4bU3jimkCD0khdQeabCt9pESg&s=8U5HwQ-j- dkzQPk3E99wJauItTQLH0f48qay37pnTZ8&e=v2: Fix the FreeBSD build error.
https://urldefense.proofpoint.com/v2/url?u=https-
3A__patchwork.dpdk.org_patch_68240_&d=DwIDaQ&c=nKjWec2b6R0mOyP az7xtfQ&r=WllrYaumVkxaWjgKto6E_rtDQshhIhik2jkvzFyRhW8&m=SxW07R9 V0iyd_fAAZm4bU3jimkCD0khdQeabCt9pESg&s=7gq0iGeXay4YQ4ydHRdCZ9_ oWDRQFtGOxGu3chq0z_s&e=v1: Update the commit message.
https://urldefense.proofpoint.com/v2/url?u=https-
3A__patchwork.dpdk.org_patch_68237_&d=DwIDaQ&c=nKjWec2b6R0mOyP az7xtfQ&r=WllrYaumVkxaWjgKto6E_rtDQshhIhik2jkvzFyRhW8&m=SxW07R9 V0iyd_fAAZm4bU3jimkCD0khdQeabCt9pESg&s=ibSeCUMMleRvQRH9WvJDYV TfBAt-Fi4lgfdTBOZ5xwc&e=RFC v2: https://urldefense.proofpoint.com/v2/url?u=https-
3A__patchwork.dpdk.org_patch_68114_&d=DwIDaQ&c=nKjWec2b6R0mOyP az7xtfQ&r=WllrYaumVkxaWjgKto6E_rtDQshhIhik2jkvzFyRhW8&m=SxW07R9 V0iyd_fAAZm4bU3jimkCD0khdQeabCt9pESg&s=JmSNjqCLE3Xh0_VymptqrViX VlRgpFN_PZMvM6C2X2s&e=
Based on Vamsi's RFC v1, and Alex's patch for Qemu
[https://urldefense.proofpoint.com/v2/url?u=https-
3A__lore.kernel.org_lkml_20200204161737.34696b91- 40w520.home_&d=DwIDaQ&c=nKjWec2b6R0mOyPaz7xtfQ&r=WllrYaumVkx aWjgKto6E_rtDQshhIhik2jkvzFyRhW8&m=SxW07R9V0iyd_fAAZm4bU3jimkCD 0khdQeabCt9pESg&s=7tRxVtn0y5xpxqC38hgAB7Bjz7kHR8mbD- pIFgP1E7o&e= ]:
Use the devarg to pass-down the VF token.RFC v1: https://urldefense.proofpoint.com/v2/url?u=https-
3A__patchwork.dpdk.org_patch_66281_&d=DwIDaQ&c=nKjWec2b6R0mOyP az7xtfQ&r=WllrYaumVkxaWjgKto6E_rtDQshhIhik2jkvzFyRhW8&m=SxW07R9 V0iyd_fAAZm4bU3jimkCD0khdQeabCt9pESg&s=riYbXycUMPQxA30y75nRzs08 f2wM-uAGcQEnK9ENR7w&e= by Vamsi.Haiyue Wang (2):
eal: add uuid dependent header files explicitly
eal: support for VFIO-PCI VF tokendrivers/bus/pci/linux/pci_vfio.c | 74 ++++++++++++++++++++++++++++++- lib/librte_eal/freebsd/eal.c | 3 +- lib/librte_eal/include/rte_uuid.h | 2 + lib/librte_eal/include/rte_vfio.h | 21 ++++++++- lib/librte_eal/linux/eal_vfio.c | 20 +++++++-- 5 files changed, 113 insertions(+), 7 deletions(-)
Quoted 2 lines
--
2.26.0
Tested following combinations:
* Both PF & VFs are with DPDK (Single process & multi process)
* PF is with DPDK & VFs are with Kernel driver
* PF is with kernel driver & VFs are with DPDK
Re: [dpdk-dev] [PATCH v6 0/2] support for VFIO-PCI VF token interface
In reply to Haiyue WangThanks a lot for working on this.
I can see the support on the kernel side has been pulled in what will be 5.7.
https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=2fb732b33b0f4d73f391066c0a970b942c600dab
I'd like to get the dpdk support in 20.05, but for this, can I have a summary of the kernel discussions?
Is there anything missing/still wip in the kernel?
On the series itself, I did not look in detail, but there is no documentation on how to use the feature.
We are missing a release note update.
And a small comment on the last changes:
Quoted 3 lines · Haiyue Wang
v6: Drop the Fixes tag in uuid, since the file has been
moved to another place, not suitable to apply on stable.
This is *not* a reason to drop the Fixes: tag. Hopefully, we still fix bugs once files have been moved.
Quoted 1 line
And this is not a bug, just some kind of enhancement.
But on this point, I suppose this is ok not to backport.
Signature · 1 line
David Marchand
Re: [dpdk-dev] [PATCH v6 0/2] support for VFIO-PCI VF token interface
Hi David,
Quoted 15 lines
-----Original Message-----
From: David Marchand <[email protected]>
Sent: Saturday, April 18, 2020 17:22
To: Wang, Haiyue <[email protected]>; Vamsi Attunuru <[email protected]>; [email protected]
Cc: dev <[email protected]>; Thomas Monjalon <[email protected]>; Jerin Jacob Kollanukkaran <[email protected]>
Subject: Re: [PATCH v6 0/2] support for VFIO-PCI VF token interfaceThanks a lot for working on this.
I can see the support on the kernel side has been pulled in what will be 5.7.
https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=2fb732b33b0f4d73f391066c0a970b942c600dab
Quoted 8 lines
I'd like to get the dpdk support in 20.05, but for this, can I have a summary of the kernel discussions?
Is there anything missing/still wip in the kernel?On the series itself, I did not look in detail, but there is no documentation on how to use the feature.
We are missing a release note update.
If needed in 20.05, I can add it in release note. And about how to, I just added it in commit message, and some comments in the API ' rte_vfio_setup_device '.
Not sure which document part is reasonable for adding this, any good suggestion ?
Quoted 10 lines
And a small comment on the last changes:
Quoted 3 lines · Haiyue Wang
v6: Drop the Fixes tag in uuid, since the file has been
moved to another place, not suitable to apply on stable.This is *not* a reason to drop the Fixes: tag. Hopefully, we still fix bugs once files have been moved.
OK, will add the tag later.
Quoted 7 lines
Quoted 1 line
And this is not a bug, just some kind of enhancement.
But on this point, I suppose this is ok not to backport.
--
David Marchand
Re: [dpdk-dev] [PATCH v6 0/2] support for VFIO-PCI VF token interface
18/04/2020 11:38, Wang, Haiyue:
Quoted 11 lines
From: David Marchand <[email protected]>
Quoted 3 lines
On the series itself, I did not look in detail, but there is no documentation on how to use the feature.
We are missing a release note update.If needed in 20.05, I can add it in release note. And about how to, I just added it in commit message, and some comments in the API '
rte_vfio_setup_device'.Not sure which document part is reasonable for adding this, any good suggestion ?
Here: http://doc.dpdk.org/guides/linux_gsg/linux_drivers.html#vfio
Re: [dpdk-dev] [PATCH v6 0/2] support for VFIO-PCI VF token interface
Quoted 14 lines
-----Original Message-----
From: Thomas Monjalon <[email protected]>
Sent: Saturday, April 18, 2020 17:50
To: David Marchand <[email protected]>; Vamsi Attunuru <[email protected]>; [email protected]; Wang, Haiyue <[email protected]>
Cc: dev <[email protected]>; Jerin Jacob Kollanukkaran <[email protected]>
Subject: Re: [PATCH v6 0/2] support for VFIO-PCI VF token interface18/04/2020 11:38, Wang, Haiyue:
Quoted 5 lines
From: David Marchand <[email protected]>
Quoted 3 lines
On the series itself, I did not look in detail, but there is no documentation on how to use the feature.
We are missing a release note update.
Quoted 10 lines
Quoted 6 lines
If needed in 20.05, I can add it in release note. And about how to, I just added it in commit message, and some comments in the API '
rte_vfio_setup_device'.Not sure which document part is reasonable for adding this, any good suggestion ?
Here: http://doc.dpdk.org/guides/linux_gsg/linux_drivers.html#vfio
OK.
v7 cover support for VFIO-PCI VF token interface
In reply to [email protected]v7: Add the Fixes tag in uuid, the release note and help
document.
v6: Drop the Fixes tag in uuid, since the file has been
moved to another place, not suitable to apply on stable. And this is not a bug, just some kind of enhancement.
https://patchwork.dpdk.org/cover/68367/
v5: 1. Add the VF token parse error handling.
2. Split into two patches for different logic module.
3. Add more comments into the code for explaining the design.
4. Drop the ABI change workaround, this patch set focuses on code review.
https://patchwork.dpdk.org/cover/68364/
v4: 1. Ignore rte_vfio_setup_device ABI check since it is
for Linux driver use.
https://patchwork.dpdk.org/patch/68255/
v3: Fix the Travis build failed:
(1). rte_uuid.h:97:55: error: unknown type name ‘size_t’ (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’
https://patchwork.dpdk.org/patch/68254/
v2: Fix the FreeBSD build error.
https://patchwork.dpdk.org/patch/68240/
v1: Update the commit message.
https://patchwork.dpdk.org/patch/68237/
RFC v2: https://patchwork.dpdk.org/patch/68114/
Based on Vamsi's RFC v1, and Alex's patch for Qemu
[https://lore.kernel.org/lkml/[email protected]/]:
Use the devarg to pass-down the VF token.
RFC v1: https://patchwork.dpdk.org/patch/66281/ by Vamsi.
Haiyue Wang (2):
eal: add uuid dependent header files explicitly
eal: support for VFIO-PCI VF token
doc/guides/linux_gsg/linux_drivers.rst | 23 +++++++- doc/guides/rel_notes/release_20_05.rst | 5 ++ drivers/bus/pci/linux/pci_vfio.c | 74 +++++++++++++++++++++++++- lib/librte_eal/freebsd/eal.c | 3 +- lib/librte_eal/include/rte_uuid.h | 2 + lib/librte_eal/include/rte_vfio.h | 21 +++++++- lib/librte_eal/linux/eal_vfio.c | 20 +++++-- 7 files changed, 140 insertions(+), 8 deletions(-)
Signature · 1 line
2.26.1
PATCH v7 1/2 eal: add uuid dependent header files explicitly
Add the dependent header files explicitly, so that the user just needs to include the 'rte_uuid.h' header file directly to avoid compile error:
(1). rte_uuid.h:97:55: error: unknown type name ‘size_t’ (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’
--- lib/librte_eal/include/rte_uuid.h | 2 ++ 1 file changed, 2 insertions(+)
lib/librte_eal/include/rte_uuid.h +2 -0
@@ -15,6 +15,8 @@ extern "C" {#endif #include <stdbool.h> +#include <stddef.h> +#include <string.h> /** * Struct describing a Universal Unique Identifier
Signature · 1 line
2.26.1
PATCH v7 2/2 eal: support for VFIO-PCI VF token
In reply to Haiyue WangThe kernel module vfio-pci introduces the VF token to enable SR-IOV support since 5.7.
The VF token can be set by a vfio-pci based PF driver and must be known by the vfio-pci based VF driver in order to gain access to the device.
--- doc/guides/linux_gsg/linux_drivers.rst | 23 +++++++- doc/guides/rel_notes/release_20_05.rst | 5 ++ drivers/bus/pci/linux/pci_vfio.c | 74 +++++++++++++++++++++++++- lib/librte_eal/freebsd/eal.c | 3 +- lib/librte_eal/include/rte_vfio.h | 21 +++++++- lib/librte_eal/linux/eal_vfio.c | 20 +++++-- 6 files changed, 138 insertions(+), 8 deletions(-)
doc/guides/linux_gsg/linux_drivers.rst +22 -1
@@ -72,11 +72,32 @@ Note that in order to use VFIO, your kernel must support it. VFIO kernel modules have been included in the Linux kernel since version 3.6.0 and are usually present by default, however please consult your distributions documentation to make sure that is the case. +The Linux kernel since version 5.7 supports the creation of virtual functions, and by default this feature is off. +After enabled, the PF needs the VF token in UUID format to setup the trust or collaboration between SR-IOV PF and VFs. +The VFs created are bound to vfio-pci module automatically, and the VF also needs VF token to gain access to the device. +DPDK will use the keyword ``vf_token`` as device argument to pass the VF token value to PF and its related VFs. + +.. code-block:: console + + 1. sudo modprobe vfio-pci enable_sriov=1 + + 2. ./usertools/dpdk-devbind.py -b vfio-pci 0000:87:00.0 + + 3. echo 2 > /sys/bus/pci/devices/0000:87:00.0/sriov_numvfs + + 4. Start the PF: + ./x86_64-native-linux-gcc/app/testpmd -l 22-25 -n 4 -w 87:00.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 \ + --file-prefix=pf -- -i + + 5. Start the VF: + ./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 -w 87:02.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 \ + --file-prefix=vf0 -- -i + Also, to use VFIO, both kernel and BIOS must support and be configured to use IO virtualization (such as Intel® VT-d). .. note:: - ``vfio-pci`` module doesn't support the creation of virtual functions. + ``vfio-pci`` module doesn't support the creation of virtual functions before Linux version 5.7. For proper operation of VFIO when running DPDK applications as a non-privileged user, correct permissions should also be set up. This can be done by using the DPDK setup script (called dpdk-setup.sh and located in the usertools directory).
doc/guides/rel_notes/release_20_05.rst +5 -0
@@ -81,6 +81,11 @@ New Features by making use of the event device capabilities. The event mode currently supports only inline IPsec protocol offload. +* **Added the creation of virtual functions support for vfio-pci module.** + + Since Linux version 5.7, vfio-pci supports a shared VF token (UUID) to represent + the trust between SR-IOV PF and the created VFs. Update the method to gain access + to the device by appending the VF token. Removed Items -------------
drivers/bus/pci/linux/pci_vfio.c +72 -2
112 lines not on this page · raw message
lib/librte_eal/freebsd/eal.c +2 -1
@@ -995,7 +995,8 @@ rte_eal_vfio_intr_mode(void) int rte_vfio_setup_device(__rte_unused const char *sysfs_base, __rte_unused const char *dev_addr, __rte_unused int *vfio_dev_fd, - __rte_unused struct vfio_device_info *device_info) + __rte_unused struct vfio_device_info *device_info, + __rte_unused rte_uuid_t vf_token) {return -1 ; }
lib/librte_eal/include/rte_vfio.h +20 -1
@@ -16,6 +16,8 @@ extern "C" {#include <stdint.h> +#include <rte_uuid.h> +/* * determine if VFIO is present on the system */
@@ -102,13 +104,30 @@ struct vfio_device_info;* @param device_info * Device information. * +* @param vf_token +* Before linux 5.7, the PF bound to vfio-pci doesn't support SR-IOV to +* create VFs for security reason. Now the VF token is introduced to work +* as some degree of trust or collaboration between PF and VFs. +* +* A). as VF device, if the PF is a vfio device and it is bound to the +* vfio-pci driver, the user needs to provide a VF token to access the +* device, in the form of appending a vf_token to the device name, for +* example: +* "-w 04:10.0,vf_token=bd8d9d2b-5a5f-4f5a-a211-f591514ba1f3" +* +* B). as PF device, When presented with a PF which has VFs in use, the +* user must also provide the current VF token to prove collaboration with +* existing VF users. If VFs are not in use, the VF token provided for the +* PF device will act to set the VF token. +* * @return * 0 on success. * <0 on failure. * >1 if the device cannot be managed this way. */ int rte_vfio_setup_device (const char *sysfs_base,const char *dev_addr, -int *vfio_dev_fd,struct vfio_device_info *device_info); +int *vfio_dev_fd,struct vfio_device_info *device_info, +rte_uuid_t vf_token);/** * Release a device mapped to a VFIO-managed I/O MMU group.
lib/librte_eal/linux/eal_vfio.c +17 -3
@@ -702,7 +702,8 @@ rte_vfio_clear_group(int vfio_group_fd)int rte_vfio_setup_device (const char *sysfs_base,const char *dev_addr, -int *vfio_dev_fd,struct vfio_device_info *device_info) +int *vfio_dev_fd,struct vfio_device_info *device_info, +rte_uuid_t vf_token) {struct vfio_group_status group_status = { .argsz =sizeof (group_status)
@@ -712,6 +713,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,int vfio_container_fd;int vfio_group_fd;int iommu_group_num; +char dev[PATH_MAX ];int i, ret;/* get group number */
@@ -895,8 +897,19 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr, t->type_id, t->name); } +if (!rte_uuid_is_null (vf_token)) { +char vf_token_str[RTE_UUID_STRLEN ]; + +rte_uuid_unparse (vf_token, vf_token_str,sizeof (vf_token_str)); +snprintf (dev,sizeof (dev), +"%s vf_token=%s" , dev_addr, vf_token_str); + }else { +snprintf (dev,sizeof (dev), +"%s" , dev_addr); + } +/* get a file descriptor for the device */ - *vfio_dev_fd =ioctl (vfio_group_fd,VFIO_GROUP_GET_DEVICE_FD , dev_addr); + *vfio_dev_fd =ioctl (vfio_group_fd,VFIO_GROUP_GET_DEVICE_FD , dev);if (*vfio_dev_fd <0 ) {/* if we cannot get a device fd, this implies a problem with * the VFIO group or the container not having IOMMU configured.
@@ -2081,7 +2094,8 @@ int rte_vfio_setup_device(__rte_unused const char *sysfs_base, __rte_unused const char *dev_addr, __rte_unused int *vfio_dev_fd, - __rte_unused struct vfio_device_info *device_info) + __rte_unused struct vfio_device_info *device_info, + __rte_unused rte_uuid_t vf_token) { return -1; }
Signature · 1 line
2.26.1
Re: [dpdk-dev] [PATCH v7 2/2] eal: support for VFIO-PCI VF token
18/04/2020 13:16, Haiyue Wang:
doc/guides/linux_gsg/linux_drivers.rst Note that in order to use VFIO, your kernel must support it. · line 75
Quoted 3 lines
doc/guides/linux_gsg/linux_drivers.rst +1
-0+The Linux kernel since version 5.7 supports the creation of virtual functions, and by default this feature is off.
Creation of virtual function has always been supported. You need to be more specific about the new use case please.
doc/guides/linux_gsg/linux_drivers.rst Note that in order to use VFIO, your kernel must support it. · line 78
Quoted 3 lines
doc/guides/linux_gsg/linux_drivers.rst +3
-0+After enabled, the PF needs the VF token in UUID format to setup the trust or collaboration between SR-IOV PF and VFs. +The VFs created are bound to vfio-pci module automatically, and the VF also needs VF token to gain access to the device. +DPDK will use the keyword ``vf_token`` as device argument to pass the VF token value to PF and its related VFs.
You need to explain where the token comes from. Can it be any random UUID?
Re: [dpdk-dev] [PATCH v7 2/2] eal: support for VFIO-PCI VF token
Quoted 17 lines
-----Original Message-----
From: Thomas Monjalon <[email protected]>
Sent: Saturday, April 18, 2020 21:27
To: [email protected]; [email protected]; Wang, Haiyue <[email protected]>
Cc: [email protected]; Burakov, Anatoly <[email protected]>; [email protected]; [email protected]
Subject: Re: [PATCH v7 2/2] eal: support for VFIO-PCI VF token18/04/2020 13:16, Haiyue Wang:
doc/guides/linux_gsg/linux_drivers.rst
Quoted 3 lines
doc/guides/linux_gsg/linux_drivers.rst +1
-0+The Linux kernel since version 5.7 supports the creation of virtual functions, and by default thisfeature is off.
Creation of virtual function has always been supported. You need to be more specific about the new use case please.
Quoted 9 lines
Quoted 1 line
+After enabled, the PF needs the VF token in UUID format to setup the trust or collaboration between
SR-IOV PF and VFs.
Quoted 1 line
+The VFs created are bound to vfio-pci module automatically, and the VF also needs VF token to gain
access to the device.
Quoted 1 line
+DPDK will use the keyword ``vf_token`` as device argument to pass the VF token value to PF and its
related VFs.
You need to explain where the token comes from. Can it be any random UUID?
Try to add more in v8, please review.
v8 cover support for VFIO-PCI VF token interface
In reply to [email protected]v8: Update the document.
v7: Add the Fixes tag in uuid, the release note and help
document.
https://patchwork.dpdk.org/cover/68845/v6: Drop the Fixes tag in uuid, since the file has been
moved to another place, not suitable to apply on stable. And this is not a bug, just some kind of enhancement.
https://patchwork.dpdk.org/cover/68367/
v5: 1. Add the VF token parse error handling.
2. Split into two patches for different logic module.
3. Add more comments into the code for explaining the design.
4. Drop the ABI change workaround, this patch set focuses on code review.
https://patchwork.dpdk.org/cover/68364/
v4: 1. Ignore rte_vfio_setup_device ABI check since it is
for Linux driver use.
https://patchwork.dpdk.org/patch/68255/
v3: Fix the Travis build failed:
(1). rte_uuid.h:97:55: error: unknown type name ‘size_t’ (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’
https://patchwork.dpdk.org/patch/68254/
v2: Fix the FreeBSD build error.
https://patchwork.dpdk.org/patch/68240/
v1: Update the commit message.
https://patchwork.dpdk.org/patch/68237/
RFC v2: https://patchwork.dpdk.org/patch/68114/
Based on Vamsi's RFC v1, and Alex's patch for Qemu
[https://lore.kernel.org/lkml/[email protected]/]:
Use the devarg to pass-down the VF token.
RFC v1: https://patchwork.dpdk.org/patch/66281/ by Vamsi.
Haiyue Wang (2):
eal: add uuid dependent header files explicitly
eal: support for VFIO-PCI VF token
doc/guides/linux_gsg/linux_drivers.rst | 38 ++++++++++++- doc/guides/rel_notes/release_20_05.rst | 5 ++ drivers/bus/pci/linux/pci_vfio.c | 74 +++++++++++++++++++++++++- lib/librte_eal/freebsd/eal.c | 3 +- lib/librte_eal/include/rte_uuid.h | 2 + lib/librte_eal/include/rte_vfio.h | 21 +++++++- lib/librte_eal/linux/eal_vfio.c | 20 +++++-- 7 files changed, 155 insertions(+), 8 deletions(-)
Signature · 1 line
2.26.1
PATCH v8 1/2 eal: add uuid dependent header files explicitly
Add the dependent header files explicitly, so that the user just needs to include the 'rte_uuid.h' header file directly to avoid compile error:
(1). rte_uuid.h:97:55: error: unknown type name ‘size_t’ (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’
--- lib/librte_eal/include/rte_uuid.h | 2 ++ 1 file changed, 2 insertions(+)
lib/librte_eal/include/rte_uuid.h +2 -0
@@ -15,6 +15,8 @@ extern "C" {#endif #include <stdbool.h> +#include <stddef.h> +#include <string.h> /** * Struct describing a Universal Unique Identifier
Signature · 1 line
2.26.1
PATCH v8 2/2 eal: support for VFIO-PCI VF token
In reply to Haiyue WangThe kernel module vfio-pci introduces the VF token to enable SR-IOV support since 5.7.
The VF token can be set by a vfio-pci based PF driver and must be known by the vfio-pci based VF driver in order to gain access to the device.
--- doc/guides/linux_gsg/linux_drivers.rst | 38 ++++++++++++- doc/guides/rel_notes/release_20_05.rst | 5 ++ drivers/bus/pci/linux/pci_vfio.c | 74 +++++++++++++++++++++++++- lib/librte_eal/freebsd/eal.c | 3 +- lib/librte_eal/include/rte_vfio.h | 21 +++++++- lib/librte_eal/linux/eal_vfio.c | 20 +++++-- 6 files changed, 153 insertions(+), 8 deletions(-)
doc/guides/linux_gsg/linux_drivers.rst +37 -1
@@ -72,11 +72,47 @@ Note that in order to use VFIO, your kernel must support it. VFIO kernel modules have been included in the Linux kernel since version 3.6.0 and are usually present by default, however please consult your distributions documentation to make sure that is the case. +The ``vfio-pci`` module since Linux version 5.7 supports the creation of virtual functions, this feature is disabled +by default. When enabled, the PF needs a shared VF token (UUID) to setup the trust between SR-IOV PF and VFs. The VF +token is any kind of valid UUID value selected by the user. When the PF device is bound to ``vfio-pci`` module, it should +not have any VFs created, this is consistent as before for security reason. + +Some use cases about how to use the VF token: + + - The user just uses PF only for DPDK, then no VF token is required to start the PF device. + + - The user wants to creat SR-IOV VFs on the PF device which is bound to ``vfio-pci`` module, then the user needs to select + a valid UUID type VF token to start the PF device; after the VFs are created, this VF token is also required to access each + VF device. + + - If the DPDK application that runs on PF device exits, and the user wants to start it with another different VF token + value, it will be OK if no application (DPDP or KVM) runs on VF, otherwise, it will fail to start with a kernel message + "[19145.688094] vfio-pci 0000:87:00.0: Incorrect VF token provided for device" shown. When all of the VFs are free, the + user can select a new VF token to start the PF device. + +The VFs created are bound to ``vfio-pci`` module automatically. DPDK will use the keyword ``vf_token`` as the device argument +to pass the VF token value to PF and its related VFs, the PMD should not use it, and this argument will be pruned from the +device argument list, so the PMD can parse its own valid device arguments successfully without seeing it. + +.. code-block:: console + + 1. sudo modprobe vfio-pci enable_sriov=1 + + 2. ./usertools/dpdk-devbind.py -b vfio-pci 0000:87:00.0 + + 3. echo 2 > /sys/bus/pci/devices/0000:87:00.0/sriov_numvfs + + 4. Start the PF: + ./x86_64-native-linux-gcc/app/testpmd -l 22-25 -n 4 -w 87:00.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 --file-prefix=pf -- -i + + 5. Start the VF: + ./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 -w 87:02.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 --file-prefix=vf0 -- -i + Also, to use VFIO, both kernel and BIOS must support and be configured to use IO virtualization (such as Intel® VT-d). .. note:: - ``vfio-pci`` module doesn't support the creation of virtual functions. + ``vfio-pci`` module doesn't support the creation of virtual functions before Linux version 5.7. For proper operation of VFIO when running DPDK applications as a non-privileged user, correct permissions should also be set up. This can be done by using the DPDK setup script (called dpdk-setup.sh and located in the usertools directory).
doc/guides/rel_notes/release_20_05.rst +5 -0
@@ -81,6 +81,11 @@ New Features by making use of the event device capabilities. The event mode currently supports only inline IPsec protocol offload. +* **Added the support for vfio-pci new VF token interface.** + + Since Linux version 5.7, vfio-pci supports a shared VF token (UUID) to represent + the trust between SR-IOV PF and the created VFs. Update the method to gain access + to the device by appending the VF token. Removed Items -------------
drivers/bus/pci/linux/pci_vfio.c +72 -2
112 lines not on this page · raw message
lib/librte_eal/freebsd/eal.c +2 -1
@@ -995,7 +995,8 @@ rte_eal_vfio_intr_mode(void) int rte_vfio_setup_device(__rte_unused const char *sysfs_base, __rte_unused const char *dev_addr, __rte_unused int *vfio_dev_fd, - __rte_unused struct vfio_device_info *device_info) + __rte_unused struct vfio_device_info *device_info, + __rte_unused rte_uuid_t vf_token) {return -1 ; }
lib/librte_eal/include/rte_vfio.h +20 -1
@@ -16,6 +16,8 @@ extern "C" {#include <stdint.h> +#include <rte_uuid.h> +/* * determine if VFIO is present on the system */
@@ -102,13 +104,30 @@ struct vfio_device_info;* @param device_info * Device information. * +* @param vf_token +* Before linux 5.7, the PF bound to vfio-pci doesn't support SR-IOV to +* create VFs for security reason. Now the VF token is introduced to work +* as some degree of trust or collaboration between PF and VFs. +* +* A). as VF device, if the PF is a vfio device and it is bound to the +* vfio-pci driver, the user needs to provide a VF token to access the +* device, in the form of appending a vf_token to the device name, for +* example: +* "-w 04:10.0,vf_token=bd8d9d2b-5a5f-4f5a-a211-f591514ba1f3" +* +* B). as PF device, When presented with a PF which has VFs in use, the +* user must also provide the current VF token to prove collaboration with +* existing VF users. If VFs are not in use, the VF token provided for the +* PF device will act to set the VF token. +* * @return * 0 on success. * <0 on failure. * >1 if the device cannot be managed this way. */ int rte_vfio_setup_device (const char *sysfs_base,const char *dev_addr, -int *vfio_dev_fd,struct vfio_device_info *device_info); +int *vfio_dev_fd,struct vfio_device_info *device_info, +rte_uuid_t vf_token);/** * Release a device mapped to a VFIO-managed I/O MMU group.
lib/librte_eal/linux/eal_vfio.c +17 -3
@@ -702,7 +702,8 @@ rte_vfio_clear_group(int vfio_group_fd)int rte_vfio_setup_device (const char *sysfs_base,const char *dev_addr, -int *vfio_dev_fd,struct vfio_device_info *device_info) +int *vfio_dev_fd,struct vfio_device_info *device_info, +rte_uuid_t vf_token) {struct vfio_group_status group_status = { .argsz =sizeof (group_status)
@@ -712,6 +713,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,int vfio_container_fd;int vfio_group_fd;int iommu_group_num; +char dev[PATH_MAX ];int i, ret;/* get group number */
@@ -895,8 +897,19 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr, t->type_id, t->name); } +if (!rte_uuid_is_null (vf_token)) { +char vf_token_str[RTE_UUID_STRLEN ]; + +rte_uuid_unparse (vf_token, vf_token_str,sizeof (vf_token_str)); +snprintf (dev,sizeof (dev), +"%s vf_token=%s" , dev_addr, vf_token_str); + }else { +snprintf (dev,sizeof (dev), +"%s" , dev_addr); + } +/* get a file descriptor for the device */ - *vfio_dev_fd =ioctl (vfio_group_fd,VFIO_GROUP_GET_DEVICE_FD , dev_addr); + *vfio_dev_fd =ioctl (vfio_group_fd,VFIO_GROUP_GET_DEVICE_FD , dev);if (*vfio_dev_fd <0 ) {/* if we cannot get a device fd, this implies a problem with * the VFIO group or the container not having IOMMU configured.
@@ -2081,7 +2094,8 @@ int rte_vfio_setup_device(__rte_unused const char *sysfs_base, __rte_unused const char *dev_addr, __rte_unused int *vfio_dev_fd, - __rte_unused struct vfio_device_info *device_info) + __rte_unused struct vfio_device_info *device_info, + __rte_unused rte_uuid_t vf_token) { return -1; }
Signature · 1 line
2.26.1
Re: [dpdk-dev] [PATCH v8 2/2] eal: support for VFIO-PCI VF token
Hi,
A general comment, please try to limit line length (80 is a good limit), and prefer breaking lines after a punctuation sign.
Try shortening sentences as well.
I would love seeing some documentation reviews from english native speakers.
18/04/2020 19:30, Haiyue Wang:
doc/guides/linux_gsg/linux_drivers.rst Note that in order to use VFIO, your kernel must support it. · line 82
Quoted 10 lines
doc/guides/linux_gsg/linux_drivers.rst +8
-0+The ``vfio-pci`` module since Linux version 5.7 supports the creation of virtual functions, this feature is disabled +by default. When enabled, the PF needs a shared VF token (UUID) to setup the trust between SR-IOV PF and VFs. The VF +token is any kind of valid UUID value selected by the user. When the PF device is bound to ``vfio-pci`` module, it should +not have any VFs created, this is consistent as before for security reason. + +Some use cases about how to use the VF token: + + - The user just uses PF only for DPDK, then no VF token is required to start the PF device.
This use case is not using the token, so it should be mentioned out of this list.
Quoted 2 lines
+
+ - The user wants to creat SR-IOV VFs on the PF device which is bound to ``vfio-pci`` module, then the user needs to select
creat -> create
doc/guides/linux_gsg/linux_drivers.rst Note that in order to use VFIO, your kernel must support it. · line 89
Quoted 5 lines
doc/guides/linux_gsg/linux_drivers.rst +5
-0+ a valid UUID type VF token to start the PF device; after the VFs are created, this VF token is also required to access each + VF device. + + - If the DPDK application that runs on PF device exits, and the user wants to start it with another different VF token + value, it will be OK if no application (DPDP or KVM) runs on VF, otherwise, it will fail to start with a kernel message
DPDP -> DPDK.
Quoted 1 line
+ "[19145.688094] vfio-pci 0000:87:00.0: Incorrect VF token provided for device" shown. When all of the VFs are free, the
The timestamp is useless in the log.
Please insert the log after :: as a fixed-size font line.
Quoted 6 lines
doc/guides/linux_gsg/linux_drivers.rst +6
-0+ user can select a new VF token to start the PF device. + +The VFs created are bound to ``vfio-pci`` module automatically. DPDK will use the keyword ``vf_token`` as the device argument +to pass the VF token value to PF and its related VFs, the PMD should not use it, and this argument will be pruned from the +device argument list, so the PMD can parse its own valid device arguments successfully without seeing it. +
doc/guides/linux_gsg/linux_drivers.rst Note that in order to use VFIO, your kernel must support it. · line 106
Quoted 10 lines
doc/guides/linux_gsg/linux_drivers.rst +10
-0+.. code-block:: console + + 1. sudo modprobe vfio-pci enable_sriov=1 + + 2. ./usertools/dpdk-devbind.py -b vfio-pci 0000:87:00.0 + + 3. echo 2 > /sys/bus/pci/devices/0000:87:00.0/sriov_numvfs + + 4. Start the PF: + ./x86_64-native-linux-gcc/app/testpmd -l 22-25 -n 4 -w 87:00.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 --file-prefix=pf -- -i
How did you generate this token?
doc/guides/linux_gsg/linux_drivers.rst Note that in order to use VFIO, your kernel must support it. · line 109
Quoted 3 lines
doc/guides/linux_gsg/linux_drivers.rst +3
-0+ + 5. Start the VF: + ./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 -w 87:02.0,vf_token=2ab74924-c335-45f4-9b16-8569e5b08258 --file-prefix=vf0 -- -i
Please do not make bullets and comments part of code-block.
Re: [dpdk-dev] [PATCH v8 0/2] support for VFIO-PCI VF token interface
In reply to Haiyue WangHello,
Quoted 2 lines · Haiyue Wang
v8: Update the document.
Thanks for the last version.
I had a look at the CI, I can see we are still missing bits to handle the ABI failure on rte_vfio_setup_device.
Signature · 1 line
David Marchand
Re: [dpdk-dev] [PATCH v8 0/2] support for VFIO-PCI VF token interface
Quoted 11 lines
-----Original Message-----
From: David Marchand <[email protected]>
Sent: Tuesday, April 21, 2020 00:53
To: Wang, Haiyue <[email protected]>
Cc: dev <[email protected]>; Burakov, Anatoly <[email protected]>; Thomas Monjalon <[email protected]>; Vamsi Attunuru <[email protected]>; Jerin Jacob Kollanukkaran <[email protected]>; Alex Williamson <[email protected]>
Subject: Re: [PATCH v8 0/2] support for VFIO-PCI VF token interfaceHello,
Quoted 7 lines
Quoted 2 lines · Haiyue Wang
v8: Update the document.
Thanks for the last version.
I had a look at the CI, I can see we are still missing bits to handle the ABI failure onrte_vfio_setup_device.
Yes, not handle it now.
If 'rte_vfio_setup_device' can be internal, not official DPDK API, then __rte_internal is the best way to handle ABI issue.
Quoted 4 lines
--
David Marchand
Re: [dpdk-dev] [PATCH v8 0/2] support for VFIO-PCI VF token interface
20/04/2020 19:02, Wang, Haiyue:
Quoted 8 lines
From: David Marchand <[email protected]>
Quoted 7 lines
Hello,
Quoted 2 lines · Haiyue Wang
v8: Update the document.
Quoted 8 lines
Quoted 3 lines
Thanks for the last version.
I had a look at the CI, I can see we are still missing bits to handle the ABI failure onrte_vfio_setup_device.Yes, not handle it now.
If '
rte_vfio_setup_device' can be internal, not official DPDK API, then__rte_internalis the best way to handle ABI issue.
Please could you help finishing integration of __rte_internal?
This patch is blocked for now.
Re: [dpdk-dev] [PATCH v8 0/2] support for VFIO-PCI VF token interface
+Neil,
Quoted 22 lines
-----Original Message-----
From: Thomas Monjalon <[email protected]>
Sent: Tuesday, April 21, 2020 01:13
To: Wang, Haiyue <[email protected]>
Cc: David Marchand <[email protected]>; dev <[email protected]>; Burakov, Anatoly <[email protected]>; Vamsi Attunuru <[email protected]>; Jerin Jacob Kollanukkaran <[email protected]>; Alex Williamson <[email protected]>
Subject: Re: [PATCH v8 0/2] support for VFIO-PCI VF token interface20/04/2020 19:02, Wang, Haiyue:
Quoted 12 lines
From: David Marchand <[email protected]>
Quoted 10 lines
Hello,
Quoted 2 lines · Haiyue Wang
v8: Update the document.
Thanks for the last version.
I had a look at the CI, I can see we are still missing bits to handle the ABI failure onrte_vfio_setup_device.
Quoted 7 lines
Quoted 4 lines
Yes, not handle it now.
If '
rte_vfio_setup_device' can be internal, not official DPDK API, then__rte_internalis the best way to handle ABI issue.Please could you help finishing integration of
__rte_internal?
I thought it should be Neil ? "Yes, I'll get back to this today" ;-)
http://inbox.dpdk.org/dev/CAJFAV8ydLkV0afEHqbh6KeA3Box0Yxb3N0MNGtMD4S9bmSgT0A@mail.gmail.com/
Quoted 2 lines
This patch is blocked for now.
Re: [dpdk-dev] [PATCH v8 0/2] support for VFIO-PCI VF token interface
20/04/2020 19:37, Wang, Haiyue:
Quoted 8 lines
From: Thomas Monjalon <[email protected]>
Quoted 7 lines
20/04/2020 19:02, Wang, Haiyue:
Quoted 6 lines
From: David Marchand <[email protected]>
Quoted 2 lines
I had a look at the CI, I can see we are still missing bits to handle the ABI failure on
rte_vfio_setup_device.Yes, not handle it now.
Quoted 7 lines
Quoted 4 lines
Quoted 2 lines
If '
rte_vfio_setup_device' can be internal, not official DPDK API, then__rte_internalis the best way to handle ABI issue.Please could you help finishing integration of
__rte_internal?I thought it should be Neil ? "Yes, I'll get back to this today" ;-)
http://inbox.dpdk.org/dev/CAJFAV8ydLkV0afEHqbh6KeA3Box0Yxb3N0MNGtMD4S9bmSgT0A@mail.gmail.com/
It did not happen after several months.
If you want to unblock your patches, I think it is safer to finish yourself.
Re: [dpdk-dev] [PATCH v8 0/2] support for VFIO-PCI VF token interface
Hi Thomas & David,
Quoted 21 lines
-----Original Message-----
From: Thomas Monjalon <[email protected]>
Sent: Tuesday, April 21, 2020 01:43
To: Wang, Haiyue <[email protected]>
Cc: Neil Horman <[email protected]>; David Marchand <[email protected]>; dev <[email protected]>; Burakov, Anatoly <[email protected]>; Vamsi Attunuru <[email protected]>; Jerin Jacob Kollanukkaran <[email protected]>; Alex Williamson <[email protected]>
Subject: Re: [PATCH v8 0/2] support for VFIO-PCI VF token interface20/04/2020 19:37, Wang, Haiyue:
Quoted 11 lines
From: Thomas Monjalon <[email protected]>
Quoted 10 lines
20/04/2020 19:02, Wang, Haiyue:
Quoted 8 lines
From: David Marchand <[email protected]>
Quoted 2 lines
I had a look at the CI, I can see we are still missing bits to handle the ABI failure on
rte_vfio_setup_device.Yes, not handle it now.
If '
rte_vfio_setup_device' can be internal, not official DPDK API, then__rte_internalis the best way to handle ABI issue.
Quoted 8 lines
Quoted 4 lines
Quoted 1 line
Please could you help finishing integration of
__rte_internal?I thought it should be Neil ? "Yes, I'll get back to this today" ;-)
http://inbox.dpdk.org/dev/CAJFAV8ydLkV0afEHqbh6KeA3Box0Yxb3N0MNGtMD4S9bmSgT0A@mail.gmail.com/It did not happen after several months.
If you want to unblock your patches, I think it is safer to finish yourself.
Unfortunately, this __rte_internal will still make the ci fail to run when move the function to INTERNAL session:
devtools/libabigail.abignore +13 -2
@@ -3,6 +3,11 @@ [suppress_variable] symbol_version = EXPERIMENTAL +[suppress_function] + symbol_version = INTERNAL +[suppress_variable] + symbol_version = INTERNAL + --- a/lib/librte_eal/rte_eal_version.map +++ b/lib/librte_eal/rte_eal_version.map
@@ -213,7 +213,6 @@ DPDK_20.0 { rte_vfio_is_enabled; rte_vfio_noiommu_is_enabled; rte_vfio_release_device; - rte_vfio_setup_device; rte_vlog; rte_zmalloc; rte_zmalloc_socket;
@@ -339,3 +338,10 @@ EXPERIMENTAL { # added in 20.05 rte_log_can_log; }; + +INTERNAL { + global: + + # added in 20.05 + rte_vfio_setup_device; +};
Functions changes summary: 1 Removed, 0 Changed, 0 Added function Variables changes summary: 0 Removed, 0 Changed, 0 Added variable
1 Removed function:
[D] 'function int rte_vfio_setup_device(const char*, const char*, int*, vfio_device_info*)' {rte_vfio_setup_device@@DPDK_20.0}
Error: ABI issue reported for 'abidiff --suppr ./devtools/libabigail.abignore --no-added-syms --headers-dir1 old_abi/include --headers-dir2 new_abi/include old_abi/dump/librte_eal.dump new_abi/dump/librte_eal.dump'
Re: [dpdk-dev] [PATCH v8 0/2] support for VFIO-PCI VF token interface
21/04/2020 03:38, Wang, Haiyue:
Quoted 24 lines
From: Thomas Monjalon <[email protected]>
Quoted 12 lines
20/04/2020 19:37, Wang, Haiyue:
Quoted 11 lines
From: Thomas Monjalon <[email protected]>
Quoted 10 lines
20/04/2020 19:02, Wang, Haiyue:
Quoted 8 lines
From: David Marchand <[email protected]>
Quoted 2 lines
I had a look at the CI, I can see we are still missing bits to handle the ABI failure on
rte_vfio_setup_device.Yes, not handle it now.
If '
rte_vfio_setup_device' can be internal, not official DPDK API, then__rte_internalis the best way to handle ABI issue.Quoted 8 lines
Quoted 4 lines
Quoted 1 line
Please could you help finishing integration of
__rte_internal?I thought it should be Neil ? "Yes, I'll get back to this today" ;-)
http://inbox.dpdk.org/dev/CAJFAV8ydLkV0afEHqbh6KeA3Box0Yxb3N0MNGtMD4S9bmSgT0A@mail.gmail.com/It did not happen after several months.
If you want to unblock your patches, I think it is safer to finish yourself.Unfortunately, this
__rte_internalwill still make the ci fail to run when move the function to INTERNAL session:
[...]
devtools/libabigail.abignore EXPERIMENTAL { · line 347
Quoted 6 lines
devtools/libabigail.abignore +6
-0+INTERNAL { + global: + + # added in 20.05 + rte_vfio_setup_device; +};
Why do you need to move the symbol explicitly in .map?
The tool should ignore symbols moving to internal, as an exception until 20.11.
Re: [dpdk-dev] [PATCH v8 0/2] support for VFIO-PCI VF token interface
Quoted 36 lines
-----Original Message-----
From: Thomas Monjalon <[email protected]>
Sent: Tuesday, April 21, 2020 10:13
To: David Marchand <[email protected]>; Wang, Haiyue <[email protected]>
Cc: Neil Horman <[email protected]>; dev <[email protected]>; Burakov, Anatoly <[email protected]>; Vamsi Attunuru <[email protected]>; Jerin Jacob Kollanukkaran <[email protected]>; Yigit, Ferruh <[email protected]>; Richardson, Bruce <[email protected]>; Kinsella, Ray <[email protected]>
Subject: Re: [PATCH v8 0/2] support for VFIO-PCI VF token interface21/04/2020 03:38, Wang, Haiyue:
Quoted 24 lines
From: Thomas Monjalon <[email protected]>
Quoted 12 lines
20/04/2020 19:37, Wang, Haiyue:
Quoted 11 lines
From: Thomas Monjalon <[email protected]>
Quoted 10 lines
20/04/2020 19:02, Wang, Haiyue:
Quoted 8 lines
From: David Marchand <[email protected]>
Quoted 2 lines
I had a look at the CI, I can see we are still missing bits to handle the ABI failure on
rte_vfio_setup_device.Yes, not handle it now.
If '
rte_vfio_setup_device' can be internal, not official DPDK API, then__rte_internalis the best way to handle ABI issue.Quoted 8 lines
Quoted 4 lines
Quoted 1 line
Please could you help finishing integration of
__rte_internal?I thought it should be Neil ? "Yes, I'll get back to this today" ;-)
http://inbox.dpdk.org/dev/CAJFAV8ydLkV0afEHqbh6KeA3Box0Yxb3N0MNGtMD4S9bmSgT0A@mail.gmail.com/It did not happen after several months.
If you want to unblock your patches, I think it is safer to finish yourself.Unfortunately, this
__rte_internalwill still make the ci fail to run when move the function to INTERNAL session:[...]
Quoted 10 lines
devtools/libabigail.abignore EXPERIMENTAL { · line 347
Quoted 6 lines
devtools/libabigail.abignore +6
-0+INTERNAL { + global: + + # added in 20.05 + rte_vfio_setup_device; +};Why do you need to move the symbol explicitly in .map?
The tool should ignore symbols moving to internal, as an exception until 20.11.
If not move the symbol explicitly in .map, another kind of error happened.
./devtools/check-abi.sh old_abi new_abi
Functions changes summary: 0 Removed, 1 Changed, 0 Added function Variables changes summary: 0 Removed, 0 Changed, 0 Added variable
1 function with some indirect sub-type change:
[C] 'function int rte_vfio_setup_device(const char*, const char*, int*, vfio_device_info*)' at eal_vfio.c:704:1 has some indirect sub-type changes:
parameter 5 of type 'unsigned char*' was added
Error: ABI issue reported for 'abidiff --suppr ./devtools/libabigail.abignore --no-added-syms --headers-dir1 old_abi/include --headers-dir2 new_abi/include old_abi/dump/librte_eal.dump new_abi/dump/librte_eal.dump'
Re: [dpdk-dev] [PATCH v8 0/2] support for VFIO-PCI VF token interface
21/04/2020 04:52, Wang, Haiyue:
Quoted 43 lines
From: Thomas Monjalon <[email protected]>
Quoted 26 lines
21/04/2020 03:38, Wang, Haiyue:
Quoted 24 lines
From: Thomas Monjalon <[email protected]>
Quoted 12 lines
20/04/2020 19:37, Wang, Haiyue:
Quoted 11 lines
From: Thomas Monjalon <[email protected]>
Quoted 10 lines
20/04/2020 19:02, Wang, Haiyue:
Quoted 8 lines
From: David Marchand <[email protected]>
Quoted 2 lines
I had a look at the CI, I can see we are still missing bits to handle the ABI failure on
rte_vfio_setup_device.Yes, not handle it now.
If '
rte_vfio_setup_device' can be internal, not official DPDK API, then__rte_internalis the best way to handle ABI issue.Quoted 8 lines
Quoted 4 lines
Quoted 1 line
Please could you help finishing integration of
__rte_internal?I thought it should be Neil ? "Yes, I'll get back to this today" ;-)
http://inbox.dpdk.org/dev/CAJFAV8ydLkV0afEHqbh6KeA3Box0Yxb3N0MNGtMD4S9bmSgT0A@mail.gmail.com/It did not happen after several months.
If you want to unblock your patches, I think it is safer to finish yourself.Unfortunately, this
__rte_internalwill still make the ci fail to run when move the function to INTERNAL session:[...]
Quoted 9 lines
devtools/libabigail.abignore EXPERIMENTAL { · line 347
Quoted 6 lines
devtools/libabigail.abignore +6
-0+INTERNAL { + global: + + # added in 20.05 + rte_vfio_setup_device; +};Why do you need to move the symbol explicitly in .map?
The tool should ignore symbols moving to internal, as an exception until 20.11.If not move the symbol explicitly in .map, another kind of error happened.
./devtools/check-abi.shold_abinew_abi
Functions changes summary: 0 Removed, 1 Changed, 0 Added function Variables changes summary: 0 Removed, 0 Changed, 0 Added variable
Quoted 6 lines
1 function with some indirect sub-type change:
[C] 'function int
rte_vfio_setup_device(const char*, const char*, int*, vfio_device_info*)' at eal_vfio.c:704:1 has some indirect sub-type changes:
parameter 5 of type 'unsigned char*' was addedError: ABI issue reported for 'abidiff --suppr ./devtools/libabigail.abignore --no-added-syms --headers-dir1 old_abi/include --headers-dir2 new_abi/include old_abi/dump/librte_eal.dump new_abi/dump/librte_eal.dump'
This is what I said: you need to add a rule to ignore internal symbols.
Re: [dpdk-dev] [PATCH v8 0/2] support for VFIO-PCI VF token interface
Quoted 58 lines
-----Original Message-----
From: Thomas Monjalon <[email protected]>
Sent: Tuesday, April 21, 2020 16:48
To: David Marchand <[email protected]>; Wang, Haiyue <[email protected]>
Cc: Neil Horman <[email protected]>; dev <[email protected]>; Burakov, Anatoly <[email protected]>; Vamsi Attunuru <[email protected]>; Jerin Jacob Kollanukkaran <[email protected]>; Yigit, Ferruh <[email protected]>; Richardson, Bruce <[email protected]>; Kinsella, Ray <[email protected]>
Subject: Re: [PATCH v8 0/2] support for VFIO-PCI VF token interface21/04/2020 04:52, Wang, Haiyue:
Quoted 19 lines
From: Thomas Monjalon <[email protected]>
Quoted 18 lines
21/04/2020 03:38, Wang, Haiyue:
Quoted 17 lines
From: Thomas Monjalon <[email protected]>
Quoted 16 lines
20/04/2020 19:37, Wang, Haiyue:
Quoted 15 lines
From: Thomas Monjalon <[email protected]>
Quoted 11 lines
20/04/2020 19:02, Wang, Haiyue:
Quoted 8 lines
From: David Marchand <[email protected]>
Quoted 2 lines
I had a look at the CI, I can see we are still missing bits to handle the ABI failure on
rte_vfio_setup_device.Yes, not handle it now.
If '
rte_vfio_setup_device' can be internal, not official DPDK API, then__rte_internalis the best way to handle ABI issue.Please could you help finishing integration of
__rte_internal?I thought it should be Neil ? "Yes, I'll get back to this today" ;-)
http://inbox.dpdk.org/dev/CAJFAV8ydLkV0afEHqbh6KeA3Box0Yxb3N0MNGtMD4S9bmSgT0A@mail.gmail.com/
Quoted 20 lines
Quoted 8 lines
Quoted 7 lines
Quoted 4 lines
It did not happen after several months.
If you want to unblock your patches, I think it is safer to finish yourself.Unfortunately, this
__rte_internalwill still make the ci fail to run when move the function to INTERNAL session:[...]
Quoted 9 lines
devtools/libabigail.abignore EXPERIMENTAL { · line 347
Quoted 6 lines
devtools/libabigail.abignore +6
-0+INTERNAL { + global: + + # added in 20.05 + rte_vfio_setup_device; +};Why do you need to move the symbol explicitly in .map?
The tool should ignore symbols moving to internal, as an exception until 20.11.If not move the symbol explicitly in .map, another kind of error happened.
Quoted 7 lines
./devtools/check-abi.shold_abinew_abi
Functions changes summary: 0 Removed, 1 Changed, 0 Added function Variables changes summary: 0 Removed, 0 Changed, 0 Added variable1 function with some indirect sub-type change:
[C] 'function int
rte_vfio_setup_device(const char*, const char*, int*, vfio_device_info*)' at
Quoted 9 lines
eal_vfio.c:704:1 has some indirect sub-type changes:
Quoted 3 lines
parameter 5 of type 'unsigned char*' was added
Error: ABI issue reported for 'abidiff --suppr ./devtools/libabigail.abignore --no-added-syms --
headers-dir1 old_abi/include --headers-dir2 new_abi/include old_abi/dump/librte_eal.dump new_abi/dump/librte_eal.dump'
This is what I said: you need to add a rule to ignore internal symbols.
Got the kind reply from Dodji, the maintainer of libabigail, his suggestion meets what we have done for 'EXPERIMENTAL'. So it seems that have to mark all needed functions as INTERNAL firstly, which obviously will break the CI as the public functions are removed....
"__PROJECT_INTERNAL_USE_ONLY_VERSION__ {
global: funA
};
...
Then, once you have all your internal functions marked in the ELF binary with the proper ELF version string, you can tell libabigail to suppress all functions that have that version by writing a suppression specification file that has this content:
[suppress_function]
symbol_version = __PROJECT_INTERNAL_USE_ONLY_VERSION__
"
v9 cover support for VFIO-PCI VF token interface
In reply to [email protected]v9: Rewrite the document.
v8: Update the document.
https://patchwork.dpdk.org/cover/68859/
v7: Add the Fixes tag in uuid, the release note and help
document.
https://patchwork.dpdk.org/cover/68845/v6: Drop the Fixes tag in uuid, since the file has been
moved to another place, not suitable to apply on stable. And this is not a bug, just some kind of enhancement.
https://patchwork.dpdk.org/cover/68367/
v5: 1. Add the VF token parse error handling.
2. Split into two patches for different logic module.
3. Add more comments into the code for explaining the design.
4. Drop the ABI change workaround, this patch set focuses on code review.
https://patchwork.dpdk.org/cover/68364/
v4: 1. Ignore rte_vfio_setup_device ABI check since it is
for Linux driver use.
https://patchwork.dpdk.org/patch/68255/
v3: Fix the Travis build failed:
(1). rte_uuid.h:97:55: error: unknown type name ‘size_t’ (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’
https://patchwork.dpdk.org/patch/68254/
v2: Fix the FreeBSD build error.
https://patchwork.dpdk.org/patch/68240/
v1: Update the commit message.
https://patchwork.dpdk.org/patch/68237/
RFC v2: https://patchwork.dpdk.org/patch/68114/
Based on Vamsi's RFC v1, and Alex's patch for Qemu
[https://lore.kernel.org/lkml/[email protected]/]:
Use the devarg to pass-down the VF token.
RFC v1: https://patchwork.dpdk.org/patch/66281/ by Vamsi.
Haiyue Wang (2):
eal: add uuid dependent header files explicitly
eal: support for VFIO-PCI VF token
devtools/libabigail.abignore | 2 + doc/guides/linux_gsg/linux_drivers.rst | 41 +++++++++++++- doc/guides/rel_notes/release_20_05.rst | 5 ++ drivers/bus/pci/linux/pci_vfio.c | 74 +++++++++++++++++++++++++- lib/librte_eal/freebsd/eal.c | 3 +- lib/librte_eal/include/rte_uuid.h | 2 + lib/librte_eal/include/rte_vfio.h | 24 ++++++++- lib/librte_eal/linux/eal_vfio.c | 20 +++++-- 8 files changed, 163 insertions(+), 8 deletions(-)
Signature · 1 line
2.26.2
PATCH v9 1/2 eal: add uuid dependent header files explicitly
Add the dependent header files explicitly, so that the user just needs to include the 'rte_uuid.h' header file directly to avoid compile error:
(1). rte_uuid.h:97:55: error: unknown type name ‘size_t’ (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’
--- lib/librte_eal/include/rte_uuid.h | 2 ++ 1 file changed, 2 insertions(+)
lib/librte_eal/include/rte_uuid.h +2 -0
@@ -15,6 +15,8 @@ extern "C" {#endif #include <stdbool.h> +#include <stddef.h> +#include <string.h> /** * Struct describing a Universal Unique Identifier
Signature · 1 line
2.26.2
PATCH v9 2/2 eal: support for VFIO-PCI VF token
In reply to Haiyue WangThe kernel module vfio-pci introduces the VF token to enable SR-IOV support since 5.7.
The VF token can be set by a vfio-pci based PF driver and must be known by the vfio-pci based VF driver in order to gain access to the device.
--- devtools/libabigail.abignore | 2 + doc/guides/linux_gsg/linux_drivers.rst | 41 +++++++++++++- doc/guides/rel_notes/release_20_05.rst | 5 ++ drivers/bus/pci/linux/pci_vfio.c | 74 +++++++++++++++++++++++++- lib/librte_eal/freebsd/eal.c | 3 +- lib/librte_eal/include/rte_vfio.h | 24 ++++++++- lib/librte_eal/linux/eal_vfio.c | 20 +++++-- 7 files changed, 161 insertions(+), 8 deletions(-)
devtools/libabigail.abignore +2 -0
@@ -6,6 +6,8 @@ ; Explicit ignore for driver-only ABI [suppress_type] name = rte_cryptodev_ops +[suppress_function] + name = rte_vfio_setup_device ; Ignore this enum update as it is part of an experimental API [suppress_type] type_kind = enum
doc/guides/linux_gsg/linux_drivers.rst +40 -1
51 lines not on this page · raw message
doc/guides/rel_notes/release_20_05.rst +5 -0
@@ -97,6 +97,11 @@ New Features by making use of the event device capabilities. The event mode currently supports only inline IPsec protocol offload. +* **Added the support for vfio-pci new VF token interface.** + + Since Linux version 5.7, vfio-pci supports a shared VF token (UUID) to represent + the trust between SR-IOV PF and the created VFs. Update the method to gain access + to the device by appending the VF token. Removed Items -------------
drivers/bus/pci/linux/pci_vfio.c +72 -2
112 lines not on this page · raw message
lib/librte_eal/freebsd/eal.c +2 -1
@@ -995,7 +995,8 @@ rte_eal_vfio_intr_mode(void) int rte_vfio_setup_device(__rte_unused const char *sysfs_base, __rte_unused const char *dev_addr, __rte_unused int *vfio_dev_fd, - __rte_unused struct vfio_device_info *device_info) + __rte_unused struct vfio_device_info *device_info, + __rte_unused rte_uuid_t vf_token) {return -1 ; }
lib/librte_eal/include/rte_vfio.h +23 -1
@@ -16,6 +16,8 @@ extern "C" {#include <stdint.h> +#include <rte_uuid.h> +/* * determine if VFIO is present on the system */
@@ -102,13 +104,33 @@ struct vfio_device_info;* @param device_info * Device information. * +* @param vf_token +* Before linux 5.7, the PF bound to vfio-pci doesn't support SR-IOV to +* create VFs for security reason. Now the VF token is introduced to work +* as some degree of trust or collaboration between PF and VFs. +* +* A). as VF device, if the PF is a vfio device and it is bound to the +* vfio-pci driver, the user needs to provide a VF token to access the +* device, in the form of appending a vf_token to the device name, for +* example: +* "-w 04:10.0,vf_token=bd8d9d2b-5a5f-4f5a-a211-f591514ba1f3" +* +* B). as PF device, When presented with a PF which has VFs in use, the +* user must also provide the current VF token to prove collaboration with +* existing VF users. If VFs are not in use, the VF token provided for the +* PF device will act to set the VF token. +* +* The vf_token can be zero uuid, which will be ignored to pass into the +* vfio-pci module. +* * @return * 0 on success. * <0 on failure. * >1 if the device cannot be managed this way. */ int rte_vfio_setup_device (const char *sysfs_base,const char *dev_addr, -int *vfio_dev_fd,struct vfio_device_info *device_info); +int *vfio_dev_fd,struct vfio_device_info *device_info, +rte_uuid_t vf_token);/** * Release a device mapped to a VFIO-managed I/O MMU group.
lib/librte_eal/linux/eal_vfio.c +17 -3
@@ -702,7 +702,8 @@ rte_vfio_clear_group(int vfio_group_fd)int rte_vfio_setup_device (const char *sysfs_base,const char *dev_addr, -int *vfio_dev_fd,struct vfio_device_info *device_info) +int *vfio_dev_fd,struct vfio_device_info *device_info, +rte_uuid_t vf_token) {struct vfio_group_status group_status = { .argsz =sizeof (group_status)
@@ -712,6 +713,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,int vfio_container_fd;int vfio_group_fd;int iommu_group_num; +char dev[PATH_MAX ];int i, ret;/* get group number */
@@ -895,8 +897,19 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr, t->type_id, t->name); } +if (!rte_uuid_is_null (vf_token)) { +char vf_token_str[RTE_UUID_STRLEN ]; + +rte_uuid_unparse (vf_token, vf_token_str,sizeof (vf_token_str)); +snprintf (dev,sizeof (dev), +"%s vf_token=%s" , dev_addr, vf_token_str); + }else { +snprintf (dev,sizeof (dev), +"%s" , dev_addr); + } +/* get a file descriptor for the device */ - *vfio_dev_fd =ioctl (vfio_group_fd,VFIO_GROUP_GET_DEVICE_FD , dev_addr); + *vfio_dev_fd =ioctl (vfio_group_fd,VFIO_GROUP_GET_DEVICE_FD , dev);if (*vfio_dev_fd <0 ) {/* if we cannot get a device fd, this implies a problem with * the VFIO group or the container not having IOMMU configured.
@@ -2083,7 +2096,8 @@ int rte_vfio_setup_device(__rte_unused const char *sysfs_base, __rte_unused const char *dev_addr, __rte_unused int *vfio_dev_fd, - __rte_unused struct vfio_device_info *device_info) + __rte_unused struct vfio_device_info *device_info, + __rte_unused rte_uuid_t vf_token) { return -1; }
Signature · 1 line
2.26.2
Re: [dpdk-dev] [PATCH v9 2/2] eal: support for VFIO-PCI VF token
22/04/2020 07:08, Haiyue Wang:
devtools/libabigail.abignore line 10
Quoted 4 lines
devtools/libabigail.abignore +2
-0+[suppress_function] + name = rte_vfio_setup_device
There is no comment to justify ignoring this symbol.
The current policy is to reject any change in libabigail.abignore which could be better handled with (unfinished) rte_internal mark.
Re: [dpdk-dev] [PATCH v9 2/2] eal: support for VFIO-PCI VF token
Quoted 16 lines
-----Original Message-----
From: Thomas Monjalon <[email protected]>
Sent: Wednesday, April 22, 2020 16:24
To: Wang, Haiyue <[email protected]>
Cc: [email protected]; Burakov, Anatoly <[email protected]>; [email protected]; [email protected]; [email protected]; [email protected]
Subject: Re: [PATCH v9 2/2] eal: support for VFIO-PCI VF token22/04/2020 07:08, Haiyue Wang:
devtools/libabigail.abignore line 10
Quoted 4 lines
devtools/libabigail.abignore +2
-0+[suppress_function] + name = rte_vfio_setup_deviceThere is no comment to justify ignoring this symbol.
Quoted 4 lines
The current policy is to reject any change in libabigail.abignore which could be better handled with (unfinished)
rte_internalmark.
While updating the vfio patch, also doing the ABI things. Found that:
1). [suppress_function]
name = rte_vfio_setup_device
can make CI happy, but not scale.
2). rte_internal is beautiful, but make CI cry : https://patchwork.dpdk.org/patch/69109/
1 Removed function:
[D] 'function int rte_vfio_setup_device(const char*, const char*, int*, vfio_device_info*)' {rte_vfio_setup_device@@DPDK_20.0}
v10 cover support for VFIO-PCI VF token interface
In reply to [email protected]v10: Use the __rte_internal to mark the internal API changing.
v9: Rewrite the document.
v8: Update the document.
v7: Add the Fixes tag in uuid, the release note and help
document.
v6: Drop the Fixes tag in uuid, since the file has been
moved to another place, not suitable to apply on stable. And this is not a bug, just some kind of enhancement.
v5: 1. Add the VF token parse error handling.
2. Split into two patches for different logic module.
3. Add more comments into the code for explaining the design.
4. Drop the ABI change workaround, this patch set focuses on code review.
v4: 1. Ignore rte_vfio_setup_device ABI check since it is
for Linux driver use.
v3: Fix the Travis build failed:
(1). rte_uuid.h:97:55: error: unknown type name ‘size_t’ (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’
v2: Fix the FreeBSD build error.
v1: Update the commit message.
RFC v2:
Based on Vamsi's RFC v1, and Alex's patch for Qemu
[https://lore.kernel.org/lkml/[email protected]/]:
Use the devarg to pass-down the VF token.
RFC v1: https://patchwork.dpdk.org/patch/66281/ by Vamsi.
Haiyue Wang (2):
eal: add uuid dependent header files explicitly
eal: support for VFIO-PCI VF token
doc/guides/linux_gsg/linux_drivers.rst | 41 +++++++++++++- doc/guides/rel_notes/release_20_05.rst | 6 +++ drivers/bus/pci/linux/pci_vfio.c | 74 +++++++++++++++++++++++++- lib/librte_eal/freebsd/eal.c | 3 +- lib/librte_eal/include/rte_uuid.h | 2 + lib/librte_eal/include/rte_vfio.h | 26 ++++++++- lib/librte_eal/linux/eal_vfio.c | 20 +++++-- lib/librte_eal/rte_eal_version.map | 8 ++- 8 files changed, 171 insertions(+), 9 deletions(-)
Signature · 1 line
2.26.2
PATCH v10 1/2 eal: add uuid dependent header files explicitly
Add the dependent header files explicitly, so that the user just needs to include the 'rte_uuid.h' header file directly to avoid compile error:
(1). rte_uuid.h:97:55: error: unknown type name ‘size_t’ (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’
--- lib/librte_eal/include/rte_uuid.h | 2 ++ 1 file changed, 2 insertions(+)
lib/librte_eal/include/rte_uuid.h +2 -0
@@ -15,6 +15,8 @@ extern "C" {#endif #include <stdbool.h> +#include <stddef.h> +#include <string.h> /** * Struct describing a Universal Unique Identifier
Signature · 1 line
2.26.2
PATCH v10 2/2 eal: support for VFIO-PCI VF token
In reply to Haiyue WangThe kernel module vfio-pci introduces the VF token to enable SR-IOV support since 5.7.
The VF token can be set by a vfio-pci based PF driver and must be known by the vfio-pci based VF driver in order to gain access to the device.
--- doc/guides/linux_gsg/linux_drivers.rst | 41 +++++++++++++- doc/guides/rel_notes/release_20_05.rst | 6 +++ drivers/bus/pci/linux/pci_vfio.c | 74 +++++++++++++++++++++++++- lib/librte_eal/freebsd/eal.c | 3 +- lib/librte_eal/include/rte_vfio.h | 26 ++++++++- lib/librte_eal/linux/eal_vfio.c | 20 +++++-- lib/librte_eal/rte_eal_version.map | 8 ++- 7 files changed, 169 insertions(+), 9 deletions(-)
doc/guides/linux_gsg/linux_drivers.rst +40 -1
@@ -72,11 +72,50 @@ Note that in order to use VFIO, your kernel must support it. VFIO kernel modules have been included in the Linux kernel since version 3.6.0 and are usually present by default, however please consult your distributions documentation to make sure that is the case. +The ``vfio-pci`` module since Linux version 5.7 supports the creation of virtual +functions. After the PF is bound to vfio-pci module, the user can create the VFs +by sysfs interface, and these VFs are bound to vfio-pci module automatically. + +When the PF is bound to vfio-pci, it has initial VF token generated by random. For +security reason, this token is write only, the user can't read it from the kernel +directly. For accessing the VF, the user needs to start the PF with token parameter +to setup a VF token (uuid format), then the VF can be accessed with this new known +VF token. + +Also if the DPDK application running on the PF device exits, the user wants to start +the PF with another different VF token value, it has no issue if no application like +DPDK or KVM runs on VFs. Otherwise, the PF will fail to start until all VFs are free +to use, after that, the user can select a new VF token to start the PF device. + +DPDK will use the keyword ``vf_token`` as the device argument to pass the VF token +value to PF and its related VFs, the PMD should not use it, and this argument will +be pruned from the device argument list, so the PMD can parse its own valid device +arguments successfully. + +.. code-block:: console + + 1. Generate the VF token by uuid command + 14d63f20-8445-11ea-8900-1f9ce7d5650d + + 2. sudo modprobe vfio-pci enable_sriov=1 + + 2. ./usertools/dpdk-devbind.py -b vfio-pci 0000:86:00.0 + + 3. echo 2 > /sys/bus/pci/devices/0000:86:00.0/sriov_numvfs + + 4. Start the PF: + ./x86_64-native-linux-gcc/app/testpmd -l 22-25 -n 4 \ + -w 86:00.0,vf_token=14d63f20-8445-11ea-8900-1f9ce7d5650d --file-prefix=pf -- -i + + 5. Start the VF: + ./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 \ + -w 86:02.0,vf_token=14d63f20-8445-11ea-8900-1f9ce7d5650d --file-prefix=vf0 -- -i + Also, to use VFIO, both kernel and BIOS must support and be configured to use IO virtualization (such as Intel® VT-d). .. note:: - ``vfio-pci`` module doesn't support the creation of virtual functions. + ``vfio-pci`` module doesn't support the creation of virtual functions before Linux version 5.7. For proper operation of VFIO when running DPDK applications as a non-privileged user, correct permissions should also be set up. This can be done by using the DPDK setup script (called dpdk-setup.sh and located in the usertools directory).
doc/guides/rel_notes/release_20_05.rst +6 -0
@@ -93,6 +93,12 @@ New Features * Added new query: ``rte_flow_get_aged_flows`` to get the aged-out flows contexts from the port. +* **Added the support for vfio-pci new VF token interface.** + + Since Linux version 5.7, vfio-pci supports a shared VF token (UUID) to represent + the trust between SR-IOV PF and the created VFs. Update the method to gain access + to the PF and VFs devices by appending the VF token parameter. + * **Updated Amazon ena driver.** Updated ena PMD with new features and improvements, including:
drivers/bus/pci/linux/pci_vfio.c +72 -2
112 lines not on this page · raw message
lib/librte_eal/freebsd/eal.c +2 -1
@@ -1005,7 +1005,8 @@ rte_eal_vfio_intr_mode(void) int rte_vfio_setup_device(__rte_unused const char *sysfs_base, __rte_unused const char *dev_addr, __rte_unused int *vfio_dev_fd, - __rte_unused struct vfio_device_info *device_info) + __rte_unused struct vfio_device_info *device_info, + __rte_unused rte_uuid_t vf_token) {return -1 ; }
lib/librte_eal/include/rte_vfio.h +25 -1
@@ -16,6 +16,9 @@ extern "C" {#include <stdint.h> +#include <rte_compat.h> +#include <rte_uuid.h> +/* * determine if VFIO is present on the system */
@@ -102,13 +105,34 @@ struct vfio_device_info;* @param device_info * Device information. * +* @param vf_token +* Before linux 5.7, the PF bound to vfio-pci doesn't support SR-IOV to +* create VFs for security reason. Now the VF token is introduced to work +* as some degree of trust or collaboration between PF and VFs. +* +* A). as VF device, if the PF is a vfio device and it is bound to the +* vfio-pci driver, the user needs to provide a VF token to access the +* device, in the form of appending a vf_token to the device name, for +* example: +* "-w 04:10.0,vf_token=bd8d9d2b-5a5f-4f5a-a211-f591514ba1f3" +* +* B). as PF device, When presented with a PF which has VFs in use, the +* user must also provide the current VF token to prove collaboration with +* existing VF users. If VFs are not in use, the VF token provided for the +* PF device will act to set the VF token. +* +* The vf_token can be zero uuid, which will be ignored to pass into the +* vfio-pci module. +* * @return * 0 on success. * <0 on failure. * >1 if the device cannot be managed this way. */ +__rte_internal int rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr, -int *vfio_dev_fd,struct vfio_device_info *device_info); +int *vfio_dev_fd,struct vfio_device_info *device_info, +rte_uuid_t vf_token);/** * Release a device mapped to a VFIO-managed I/O MMU group.
lib/librte_eal/linux/eal_vfio.c +17 -3
@@ -702,7 +702,8 @@ rte_vfio_clear_group(int vfio_group_fd)int rte_vfio_setup_device (const char *sysfs_base,const char *dev_addr, -int *vfio_dev_fd,struct vfio_device_info *device_info) +int *vfio_dev_fd,struct vfio_device_info *device_info, +rte_uuid_t vf_token) {struct vfio_group_status group_status = { .argsz =sizeof (group_status)
@@ -712,6 +713,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,int vfio_container_fd;int vfio_group_fd;int iommu_group_num; +char dev[PATH_MAX ];int i, ret;/* get group number */
@@ -895,8 +897,19 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr, t->type_id, t->name); } +if (!rte_uuid_is_null (vf_token)) { +char vf_token_str[RTE_UUID_STRLEN ]; + +rte_uuid_unparse (vf_token, vf_token_str,sizeof (vf_token_str)); +snprintf (dev,sizeof (dev), +"%s vf_token=%s" , dev_addr, vf_token_str); + }else { +snprintf (dev,sizeof (dev), +"%s" , dev_addr); + } +/* get a file descriptor for the device */ - *vfio_dev_fd =ioctl (vfio_group_fd,VFIO_GROUP_GET_DEVICE_FD , dev_addr); + *vfio_dev_fd =ioctl (vfio_group_fd,VFIO_GROUP_GET_DEVICE_FD , dev);if (*vfio_dev_fd <0 ) {/* if we cannot get a device fd, this implies a problem with * the VFIO group or the container not having IOMMU configured.
@@ -2083,7 +2096,8 @@ int rte_vfio_setup_device(__rte_unused const char *sysfs_base, __rte_unused const char *dev_addr, __rte_unused int *vfio_dev_fd, - __rte_unused struct vfio_device_info *device_info) + __rte_unused struct vfio_device_info *device_info, + __rte_unused rte_uuid_t vf_token) { return -1; }
lib/librte_eal/rte_eal_version.map +7 -1
@@ -213,7 +213,6 @@ DPDK_20.0 { rte_vfio_is_enabled; rte_vfio_noiommu_is_enabled; rte_vfio_release_device; - rte_vfio_setup_device; rte_vlog; rte_zmalloc; rte_zmalloc_socket;
@@ -388,3 +387,10 @@ EXPERIMENTAL { rte_trace_regexp; rte_trace_save; }; + +INTERNAL { + global: + + # added in 20.05 + rte_vfio_setup_device; +};
Signature · 1 line
2.26.2
Re: [dpdk-dev] [PATCH v10 2/2] eal: support for VFIO-PCI VF token
Quoted 10 lines · Haiyue Wang
The kernel module vfio-pci introduces the VF token to enable SR-IOV support since 5.7.
The VF token can be set by a vfio-pci based PF driver and must be known by the vfio-pci based VF driver in order to gain access to the device.
Signed-off-by Haiyue Wang <[email protected]>Acked-by Vamsi Attunuru <[email protected]>Tested-by Vamsi Attunuru <[email protected]>---
LGTM
Signature · 2 lines
Thanks, Anatoly
Re: [dpdk-dev] [PATCH v10 2/2] eal: support for VFIO-PCI VF token
In reply to Haiyue WangQuoted 9 lines · Haiyue Wang
The kernel module vfio-pci introduces the VF token to enable SR-IOV support since 5.7.
The VF token can be set by a vfio-pci based PF driver and must be known by the vfio-pci based VF driver in order to gain access to the device.
Signed-off-by Haiyue Wang <[email protected]>Acked-by Vamsi Attunuru <[email protected]>Tested-by Vamsi Attunuru <[email protected]>
Few points below, anyway:
[snip]
Quoted 7 lines
doc/guides/linux_gsg/linux_drivers.rst +0
-0@@ -72,11 +72,50 @@ Note that in order to use VFIO, your kernel must support it. VFIO kernel modules have been included in the Linux kernel since version 3.6.0 and are usually present by default, however please consult your distributions documentation to make sure that is the case.
doc/guides/linux_gsg/linux_drivers.rst Note that in order to use VFIO, your kernel must support it. · line 83
Quoted 10 lines
doc/guides/linux_gsg/linux_drivers.rst +9
-0+The ``vfio-pci`` module since Linux version 5.7 supports the creation of virtual +functions. After the PF is bound to vfio-pci module, the user can create the VFs +by sysfs interface, and these VFs are bound to vfio-pci module automatically. + +When the PF is bound to vfio-pci, it has initial VF token generated by random. For +security reason, this token is write only, the user can't read it from the kernel +directly. For accessing the VF, the user needs to start the PF with token parameter +to setup a VF token (uuid format), then the VF can be accessed with this new known +VF token.
If token is write-only in kernel sysfs, shouldn't we make it invisible in ps output (procfs)? I.e. substitute with something like xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx.
However, it does not look easy. Just a thought.
[snip]
Quoted 19 lines
drivers/bus/pci/linux/pci_vfio.c +4
-0@@ -11,6 +11,7 @@#include <sys/mman.h> #include <stdbool.h> +#include <rte_devargs.h> #include <rte_log.h> #include <rte_pci.h> #include <rte_bus_pci.h> @@ -644,12 +645,72 @@ pci_vfio_msix_is_mappable(int vfio_dev_fd, int msix_region)return ret; } +static int +vfio_pci_vf_token_arg (struct rte_devargs *devargs,rte_uuid_t uuid) +{
drivers/bus/pci/linux/pci_vfio.c pci_vfio_msix_is_mappable(int vfio_dev_fd, int msix_region) · line 660
Quoted 10 lines
drivers/bus/pci/linux/pci_vfio.c +10
-0+#define VF_TOKEN_ARG "vf_token=" +char c, *p, *vf_token; + +memset (uuid,0 ,sizeof (rte_uuid_t)); + +if (devargs ==NULL ) +return 0 ; + + p =strstr (devargs->args,VF_TOKEN_ARG ); +if (!p)
Please, p == NULL as you do few lines above.
[snip]
v11 cover support for VFIO-PCI VF token interface
In reply to [email protected]v11: Use the eal parameter to pass the VF token, then not every PCI
device needs to be specified with this token. Also no ABI issue now.
v10: Use the __rte_internal to mark the internal API changing.
v9: Rewrite the document.
v8: Update the document.
v7: Add the Fixes tag in uuid, the release note and help
document.
v6: Drop the Fixes tag in uuid, since the file has been
moved to another place, not suitable to apply on stable. And this is not a bug, just some kind of enhancement.
v5: 1. Add the VF token parse error handling.
2. Split into two patches for different logic module.
3. Add more comments into the code for explaining the design.
4. Drop the ABI change workaround, this patch set focuses on code review.
v4: 1. Ignore rte_vfio_setup_device ABI check since it is
for Linux driver use.
v3: Fix the Travis build failed:
(1). rte_uuid.h:97:55: error: unknown type name ‘size_t’ (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’
v2: Fix the FreeBSD build error.
v1: Update the commit message.
RFC v2:
Based on Vamsi's RFC v1, and Alex's patch for Qemu
[https://lore.kernel.org/lkml/[email protected]/]:
Use the devarg to pass-down the VF token.
RFC v1: https://patchwork.dpdk.org/patch/66281/ by Vamsi.
Haiyue Wang (2):
eal: add uuid dependent header files explicitly
eal: support for VFIO-PCI VF token
doc/guides/linux_gsg/linux_drivers.rst | 35 ++++++++++++++++++- doc/guides/linux_gsg/linux_eal_parameters.rst | 4 +++ doc/guides/rel_notes/release_20_05.rst | 6 ++++ lib/librte_eal/common/eal_common_options.c | 2 ++ lib/librte_eal/common/eal_internal_cfg.h | 2 ++ lib/librte_eal/common/eal_options.h | 2 ++ lib/librte_eal/freebsd/eal.c | 5 +++ lib/librte_eal/include/rte_eal.h | 12 +++++++ lib/librte_eal/include/rte_uuid.h | 2 ++ lib/librte_eal/linux/eal.c | 29 +++++++++++++++ lib/librte_eal/linux/eal_vfio.c | 16 ++++++++- lib/librte_eal/rte_eal_version.map | 1 + 12 files changed, 114 insertions(+), 2 deletions(-)
Signature · 1 line
2.26.2
PATCH v11 1/2 eal: add uuid dependent header files explicitly
Add the dependent header files explicitly, so that the user just needs to include the 'rte_uuid.h' header file directly to avoid compile error:
(1). rte_uuid.h:97:55: error: unknown type name ‘size_t’ (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’
--- lib/librte_eal/include/rte_uuid.h | 2 ++ 1 file changed, 2 insertions(+)
lib/librte_eal/include/rte_uuid.h +2 -0
@@ -15,6 +15,8 @@ extern "C" {#endif #include <stdbool.h> +#include <stddef.h> +#include <string.h> /** * Struct describing a Universal Unique Identifier
Signature · 1 line
2.26.2
PATCH v11 2/2 eal: support for VFIO-PCI VF token
In reply to Haiyue WangThe kernel module vfio-pci introduces the VF token to enable SR-IOV support since 5.7.
The VF token can be set by a vfio-pci based PF driver and must be known by the vfio-pci based VF driver in order to gain access to the device.
--- doc/guides/linux_gsg/linux_drivers.rst | 35 ++++++++++++++++++- doc/guides/linux_gsg/linux_eal_parameters.rst | 4 +++ doc/guides/rel_notes/release_20_05.rst | 6 ++++ lib/librte_eal/common/eal_common_options.c | 2 ++ lib/librte_eal/common/eal_internal_cfg.h | 2 ++ lib/librte_eal/common/eal_options.h | 2 ++ lib/librte_eal/freebsd/eal.c | 5 +++ lib/librte_eal/include/rte_eal.h | 12 +++++++ lib/librte_eal/linux/eal.c | 29 +++++++++++++++ lib/librte_eal/linux/eal_vfio.c | 16 ++++++++- lib/librte_eal/rte_eal_version.map | 1 + 11 files changed, 112 insertions(+), 2 deletions(-)
doc/guides/linux_gsg/linux_drivers.rst +34 -1
@@ -72,11 +72,44 @@ Note that in order to use VFIO, your kernel must support it. VFIO kernel modules have been included in the Linux kernel since version 3.6.0 and are usually present by default, however please consult your distributions documentation to make sure that is the case. +The ``vfio-pci`` module since Linux version 5.7 supports the creation of virtual +functions. After the PF is bound to vfio-pci module, the user can create the VFs +by sysfs interface, and these VFs are bound to vfio-pci module automatically. + +When the PF is bound to vfio-pci, it has initial VF token generated by random. For +security reason, this token is write only, the user can't read it from the kernel +directly. For accessing the VF, the user needs to start the PF with token parameter +to setup a VF token (uuid format), then the VF can be accessed with this new known +VF token. + +DPDK will use the EAL parameter ``--vfio-vf-token`` to specify the VF token value to +PF and its related VFs, this VF token will be shared in all VFIO devices, including +the different PFs. + +.. code-block:: console + + 1. Generate the VF token by uuid command + 14d63f20-8445-11ea-8900-1f9ce7d5650d + + 2. sudo modprobe vfio-pci enable_sriov=1 + + 2. ./usertools/dpdk-devbind.py -b vfio-pci 0000:86:00.0 + + 3. echo 2 > /sys/bus/pci/devices/0000:86:00.0/sriov_numvfs + + 4. Start the PF: + ./x86_64-native-linux-gcc/app/testpmd -l 22-25 -n 4 -w 86:00.0 \ + --vfio-vf-token=14d63f20-8445-11ea-8900-1f9ce7d5650d --file-prefix=pf -- -i + + 5. Start the VF: + ./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 -w 86:02.0 \ + --vfio-vf-token=14d63f20-8445-11ea-8900-1f9ce7d5650d --file-prefix=vf0 -- -i + Also, to use VFIO, both kernel and BIOS must support and be configured to use IO virtualization (such as Intel® VT-d). .. note:: - ``vfio-pci`` module doesn't support the creation of virtual functions. + ``vfio-pci`` module doesn't support the creation of virtual functions before Linux version 5.7. For proper operation of VFIO when running DPDK applications as a non-privileged user, correct permissions should also be set up. This can be done by using the DPDK setup script (called dpdk-setup.sh and located in the usertools directory).
doc/guides/linux_gsg/linux_eal_parameters.rst +4 -0
@@ -40,6 +40,10 @@ Device-related options Use specified interrupt mode for devices bound to VFIO kernel driver. +* ``--vfio-vf-token <uuid>`` + + Use specified VF token for devices bound to VFIO kernel driver. + Multiprocessing-related options ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
doc/guides/rel_notes/release_20_05.rst +6 -0
@@ -93,6 +93,12 @@ New Features * Added new query: ``rte_flow_get_aged_flows`` to get the aged-out flows contexts from the port. +* **Added the support for vfio-pci new VF token interface.** + + Since Linux version 5.7, vfio-pci supports a shared VF token (UUID) to represent + the trust between SR-IOV PF and the created VFs. Update the method to gain access + to the PF and VFs devices by appending the VF token parameter. + * **Updated Amazon ena driver.** Updated ena PMD with new features and improvements, including:
lib/librte_eal/common/eal_common_options.c +2 -0
@@ -89,6 +89,7 @@ eal_long_options[] = { {OPT_SYSLOG, 1, NULL, OPT_SYSLOG_NUM }, {OPT_VDEV, 1, NULL, OPT_VDEV_NUM }, {OPT_VFIO_INTR, 1, NULL, OPT_VFIO_INTR_NUM }, + {OPT_VFIO_VF_TOKEN, 1, NULL, OPT_VFIO_VF_TOKEN_NUM }, {OPT_VMWARE_TSC_MAP, 0, NULL, OPT_VMWARE_TSC_MAP_NUM }, {OPT_LEGACY_MEM, 0, NULL, OPT_LEGACY_MEM_NUM }, {OPT_SINGLE_FILE_SEGMENTS, 0, NULL, OPT_SINGLE_FILE_SEGMENTS_NUM},
@@ -222,6 +223,7 @@ eal_reset_internal_config(struct internal_config *internal_cfg)/* if set to NONE, interrupt mode is determined automatically */ internal_cfg->vfio_intr_mode =RTE_INTR_MODE_NONE ; +memset (internal_cfg->vfio_vf_token,0 ,sizeof (rte_uuid_t));#ifdef RTE_LIBEAL_USE_HPET internal_cfg->no_hpet =0 ;
lib/librte_eal/common/eal_internal_cfg.h +2 -0
@@ -72,6 +72,8 @@ struct internal_config {volatile int syslog_facility;/**< facility passed to openlog() */ /** default interrupt mode for VFIO */ volatile enum rte_intr_mode vfio_intr_mode; +/** the shared VF token for VFIO-PCI bound PF and VFs devices */ +rte_uuid_t vfio_vf_token;char *hugefile_prefix;/**< the base filename of hugetlbfs files */ char *hugepage_dir;/**< specific hugetlbfs directory to use */ char *user_mbuf_pool_ops_name;
lib/librte_eal/common/eal_options.h +2 -0
@@ -67,6 +67,8 @@ enum {OPT_VDEV_NUM , #define OPT_VFIO_INTR "vfio-intr"OPT_VFIO_INTR_NUM , +#define OPT_VFIO_VF_TOKEN "vfio-vf-token" +OPT_VFIO_VF_TOKEN_NUM , #define OPT_VMWARE_TSC_MAP "vmware-tsc-map"OPT_VMWARE_TSC_MAP_NUM , #define OPT_LEGACY_MEM "legacy-mem"
lib/librte_eal/freebsd/eal.c +5 -0
@@ -1002,6 +1002,11 @@ rte_eal_vfio_intr_mode(void)return RTE_INTR_MODE_NONE ; } +void rte_eal_vfio_vf_token (rte_uuid_t vf_token) +{ +memset (vf_token,0 ,sizeof (rte_uuid_t)); +} + int rte_vfio_setup_device(__rte_unused const char *sysfs_base, __rte_unused const char *dev_addr, __rte_unused int *vfio_dev_fd,
lib/librte_eal/include/rte_eal.h +12 -0
@@ -21,6 +21,7 @@#include <rte_bus.h> #include <rte_pci_dev_feature_defs.h> +#include <rte_uuid.h> #ifdef __cplusplusextern "C" {
@@ -438,6 +439,17 @@ int rte_eal_create_uio_dev(void);*/ enum rte_intr_mode rte_eal_vfio_intr_mode (void ); + +/** +* The user-configured vfio VF token. +* +* @param vf_token +* vfio VF token configured with the command line is copied +* into this parameter, zero uuid by default. +*/ +__rte_experimental +void rte_eal_vfio_vf_token(rte_uuid_t vf_token); +/** * A wrap API for syscall gettid. *
lib/librte_eal/linux/eal.c +29 -0
@@ -558,6 +558,7 @@ eal_usage(const char *prgname)" --" OPT_FILE_PREFIX " Prefix for hugepage filenames\n" " --" OPT_CREATE_UIO_DEV " Create /dev/uioX (usually done by hotplug)\n" " --" OPT_VFIO_INTR " Interrupt mode for VFIO (legacy|msi|msix)\n" +" --" OPT_VFIO_VF_TOKEN " VF token (UUID) shared between SR-IOV PF and VFs\n" " --" OPT_LEGACY_MEM " Legacy memory mode (no dynamic allocation, contiguous segments)\n" " --" OPT_SINGLE_FILE_SEGMENTS " Put all hugepage memory in single files\n" " --" OPT_MATCH_ALLOCATIONS " Free hugepages exactly as allocated\n"
@@ -649,6 +650,19 @@ eal_parse_vfio_intr(const char *mode)return -1 ; } +static int +eal_parse_vfio_vf_token (const char *vf_token) +{ +rte_uuid_t uuid; + +if (!rte_uuid_parse (vf_token, uuid)) { +rte_uuid_copy (internal_config.vfio_vf_token, uuid); +return 0 ; + } + +return -1 ; +} +/* Parse the arguments for --log-level only */ static void eal_log_level_parse (int argc,char **argv)
@@ -795,6 +809,16 @@ eal_parse_args(int argc, char **argv) }break ; +case OPT_VFIO_VF_TOKEN_NUM : +if (eal_parse_vfio_vf_token (optarg) <0 ) { +RTE_LOG (ERR ,EAL ,"invalid parameters for --" +OPT_VFIO_VF_TOKEN "\n" ); +eal_usage (prgname); + ret =-1 ; + goto out; + } +break ; +case OPT_CREATE_UIO_DEV_NUM : internal_config.create_uio_dev =1 ;break ;
@@ -1367,6 +1391,11 @@ rte_eal_vfio_intr_mode(void)return internal_config.vfio_intr_mode; } +void rte_eal_vfio_vf_token (rte_uuid_t vf_token) +{ +rte_uuid_copy (vf_token, internal_config.vfio_vf_token); +} +int rte_eal_check_module (const char *module_name) {
lib/librte_eal/linux/eal_vfio.c +15 -1
@@ -712,6 +712,8 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,int vfio_container_fd;int vfio_group_fd;int iommu_group_num; +rte_uuid_t vf_token; +char dev[PATH_MAX ];int i, ret;/* get group number */
@@ -895,8 +897,20 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr, t->type_id, t->name); } +rte_eal_vfio_vf_token (vf_token); +if (!rte_uuid_is_null (vf_token)) { +char vf_token_str[RTE_UUID_STRLEN ]; + +rte_uuid_unparse (vf_token, vf_token_str,sizeof (vf_token_str)); +snprintf (dev,sizeof (dev), +"%s vf_token=%s" , dev_addr, vf_token_str); + }else { +snprintf (dev,sizeof (dev), +"%s" , dev_addr); + } +/* get a file descriptor for the device */ - *vfio_dev_fd =ioctl (vfio_group_fd,VFIO_GROUP_GET_DEVICE_FD , dev_addr); + *vfio_dev_fd =ioctl (vfio_group_fd,VFIO_GROUP_GET_DEVICE_FD , dev);if (*vfio_dev_fd <0 ) {/* if we cannot get a device fd, this implies a problem with * the VFIO group or the container not having IOMMU configured.
lib/librte_eal/rte_eal_version.map +1 -0
@@ -373,6 +373,7 @@ EXPERIMENTAL { __rte_trace_point_register; per_lcore_trace_mem; per_lcore_trace_point_sz; + rte_eal_vfio_vf_token; rte_log_can_log; rte_thread_getname; rte_trace_dump;
Signature · 1 line
2.26.2
v12 cover support for VFIO-PCI VF token interface
In reply to [email protected]v12: support to vfio devices with VF token and no token.
v11: Use the eal parameter to pass the VF token, then not every PCI
device needs to be specified with this token. Also no ABI issue now.
v10: Use the __rte_internal to mark the internal API changing.
v9: Rewrite the document.
v8: Update the document.
v7: Add the Fixes tag in uuid, the release note and help
document.
v6: Drop the Fixes tag in uuid, since the file has been
moved to another place, not suitable to apply on stable. And this is not a bug, just some kind of enhancement.
v5: 1. Add the VF token parse error handling.
2. Split into two patches for different logic module.
3. Add more comments into the code for explaining the design.
4. Drop the ABI change workaround, this patch set focuses on code review.
v4: 1. Ignore rte_vfio_setup_device ABI check since it is
for Linux driver use.
v3: Fix the Travis build failed:
(1). rte_uuid.h:97:55: error: unknown type name ‘size_t’ (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’
v2: Fix the FreeBSD build error.
v1: Update the commit message.
RFC v2:
Based on Vamsi's RFC v1, and Alex's patch for Qemu
[https://lore.kernel.org/lkml/[email protected]/]:
Use the devarg to pass-down the VF token.
RFC v1: https://patchwork.dpdk.org/patch/66281/ by Vamsi.
Haiyue Wang (2):
eal: add uuid dependent header files explicitly
eal: support for VFIO-PCI VF token
doc/guides/linux_gsg/linux_drivers.rst | 35 ++++++++++++++++++- doc/guides/linux_gsg/linux_eal_parameters.rst | 4 +++ doc/guides/rel_notes/release_20_05.rst | 6 ++++ lib/librte_eal/common/eal_common_options.c | 2 ++ lib/librte_eal/common/eal_internal_cfg.h | 2 ++ lib/librte_eal/common/eal_options.h | 2 ++ lib/librte_eal/freebsd/eal.c | 5 +++ lib/librte_eal/include/rte_eal.h | 12 +++++++ lib/librte_eal/include/rte_uuid.h | 2 ++ lib/librte_eal/linux/eal.c | 29 +++++++++++++++ lib/librte_eal/linux/eal_vfio.c | 19 ++++++++++ lib/librte_eal/rte_eal_version.map | 1 + 12 files changed, 118 insertions(+), 1 deletion(-)
Signature · 1 line
2.26.2
PATCH v12 1/2 eal: add uuid dependent header files explicitly
Add the dependent header files explicitly, so that the user just needs to include the 'rte_uuid.h' header file directly to avoid compile error:
(1). rte_uuid.h:97:55: error: unknown type name ‘size_t’ (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’
--- lib/librte_eal/include/rte_uuid.h | 2 ++ 1 file changed, 2 insertions(+)
lib/librte_eal/include/rte_uuid.h +2 -0
@@ -15,6 +15,8 @@ extern "C" {#endif #include <stdbool.h> +#include <stddef.h> +#include <string.h> /** * Struct describing a Universal Unique Identifier
Signature · 1 line
2.26.2
PATCH v12 2/2 eal: support for VFIO-PCI VF token
In reply to Haiyue WangThe kernel module vfio-pci introduces the VF token to enable SR-IOV support since 5.7.
The VF token can be set by a vfio-pci based PF driver and must be known by the vfio-pci based VF driver in order to gain access to the device.
--- doc/guides/linux_gsg/linux_drivers.rst | 35 ++++++++++++++++++- doc/guides/linux_gsg/linux_eal_parameters.rst | 4 +++ doc/guides/rel_notes/release_20_05.rst | 6 ++++ lib/librte_eal/common/eal_common_options.c | 2 ++ lib/librte_eal/common/eal_internal_cfg.h | 2 ++ lib/librte_eal/common/eal_options.h | 2 ++ lib/librte_eal/freebsd/eal.c | 5 +++ lib/librte_eal/include/rte_eal.h | 12 +++++++ lib/librte_eal/linux/eal.c | 29 +++++++++++++++ lib/librte_eal/linux/eal_vfio.c | 19 ++++++++++ lib/librte_eal/rte_eal_version.map | 1 + 11 files changed, 116 insertions(+), 1 deletion(-)
doc/guides/linux_gsg/linux_drivers.rst +34 -1
@@ -72,11 +72,44 @@ Note that in order to use VFIO, your kernel must support it. VFIO kernel modules have been included in the Linux kernel since version 3.6.0 and are usually present by default, however please consult your distributions documentation to make sure that is the case. +The ``vfio-pci`` module since Linux version 5.7 supports the creation of virtual +functions. After the PF is bound to vfio-pci module, the user can create the VFs +by sysfs interface, and these VFs are bound to vfio-pci module automatically. + +When the PF is bound to vfio-pci, it has initial VF token generated by random. For +security reason, this token is write only, the user can't read it from the kernel +directly. For accessing the VF, the user needs to start the PF with token parameter +to setup a VF token (uuid format), then the VF can be accessed with this new known +VF token. + +DPDK will use the EAL parameter ``--vfio-vf-token`` to specify the VF token value to +PF and its related VFs, this VF token will be shared in all VFIO devices, including +the different PFs. + +.. code-block:: console + + 1. Generate the VF token by uuid command + 14d63f20-8445-11ea-8900-1f9ce7d5650d + + 2. sudo modprobe vfio-pci enable_sriov=1 + + 2. ./usertools/dpdk-devbind.py -b vfio-pci 0000:86:00.0 + + 3. echo 2 > /sys/bus/pci/devices/0000:86:00.0/sriov_numvfs + + 4. Start the PF: + ./x86_64-native-linux-gcc/app/testpmd -l 22-25 -n 4 -w 86:00.0 \ + --vfio-vf-token=14d63f20-8445-11ea-8900-1f9ce7d5650d --file-prefix=pf -- -i + + 5. Start the VF: + ./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 -w 86:02.0 \ + --vfio-vf-token=14d63f20-8445-11ea-8900-1f9ce7d5650d --file-prefix=vf0 -- -i + Also, to use VFIO, both kernel and BIOS must support and be configured to use IO virtualization (such as Intel® VT-d). .. note:: - ``vfio-pci`` module doesn't support the creation of virtual functions. + ``vfio-pci`` module doesn't support the creation of virtual functions before Linux version 5.7. For proper operation of VFIO when running DPDK applications as a non-privileged user, correct permissions should also be set up. This can be done by using the DPDK setup script (called dpdk-setup.sh and located in the usertools directory).
doc/guides/linux_gsg/linux_eal_parameters.rst +4 -0
@@ -40,6 +40,10 @@ Device-related options Use specified interrupt mode for devices bound to VFIO kernel driver. +* ``--vfio-vf-token <uuid>`` + + Use specified VF token for devices bound to VFIO kernel driver. + Multiprocessing-related options ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
doc/guides/rel_notes/release_20_05.rst +6 -0
@@ -93,6 +93,12 @@ New Features * Added new query: ``rte_flow_get_aged_flows`` to get the aged-out flows contexts from the port. +* **Added the support for vfio-pci new VF token interface.** + + Since Linux version 5.7, vfio-pci supports a shared VF token (UUID) to represent + the trust between SR-IOV PF and the created VFs. Update the method to gain access + to the PF and VFs devices by appending the VF token parameter. + * **Updated Amazon ena driver.** Updated ena PMD with new features and improvements, including:
lib/librte_eal/common/eal_common_options.c +2 -0
@@ -89,6 +89,7 @@ eal_long_options[] = { {OPT_SYSLOG, 1, NULL, OPT_SYSLOG_NUM }, {OPT_VDEV, 1, NULL, OPT_VDEV_NUM }, {OPT_VFIO_INTR, 1, NULL, OPT_VFIO_INTR_NUM }, + {OPT_VFIO_VF_TOKEN, 1, NULL, OPT_VFIO_VF_TOKEN_NUM }, {OPT_VMWARE_TSC_MAP, 0, NULL, OPT_VMWARE_TSC_MAP_NUM }, {OPT_LEGACY_MEM, 0, NULL, OPT_LEGACY_MEM_NUM }, {OPT_SINGLE_FILE_SEGMENTS, 0, NULL, OPT_SINGLE_FILE_SEGMENTS_NUM},
@@ -222,6 +223,7 @@ eal_reset_internal_config(struct internal_config *internal_cfg)/* if set to NONE, interrupt mode is determined automatically */ internal_cfg->vfio_intr_mode =RTE_INTR_MODE_NONE ; +memset (internal_cfg->vfio_vf_token,0 ,sizeof (rte_uuid_t));#ifdef RTE_LIBEAL_USE_HPET internal_cfg->no_hpet =0 ;
lib/librte_eal/common/eal_internal_cfg.h +2 -0
@@ -72,6 +72,8 @@ struct internal_config {volatile int syslog_facility;/**< facility passed to openlog() */ /** default interrupt mode for VFIO */ volatile enum rte_intr_mode vfio_intr_mode; +/** the shared VF token for VFIO-PCI bound PF and VFs devices */ +rte_uuid_t vfio_vf_token;char *hugefile_prefix;/**< the base filename of hugetlbfs files */ char *hugepage_dir;/**< specific hugetlbfs directory to use */ char *user_mbuf_pool_ops_name;
lib/librte_eal/common/eal_options.h +2 -0
@@ -67,6 +67,8 @@ enum {OPT_VDEV_NUM , #define OPT_VFIO_INTR "vfio-intr"OPT_VFIO_INTR_NUM , +#define OPT_VFIO_VF_TOKEN "vfio-vf-token" +OPT_VFIO_VF_TOKEN_NUM , #define OPT_VMWARE_TSC_MAP "vmware-tsc-map"OPT_VMWARE_TSC_MAP_NUM , #define OPT_LEGACY_MEM "legacy-mem"
lib/librte_eal/freebsd/eal.c +5 -0
@@ -1002,6 +1002,11 @@ rte_eal_vfio_intr_mode(void)return RTE_INTR_MODE_NONE ; } +void rte_eal_vfio_vf_token (rte_uuid_t vf_token) +{ +memset (vf_token,0 ,sizeof (rte_uuid_t)); +} + int rte_vfio_setup_device(__rte_unused const char *sysfs_base, __rte_unused const char *dev_addr, __rte_unused int *vfio_dev_fd,
lib/librte_eal/include/rte_eal.h +12 -0
@@ -21,6 +21,7 @@#include <rte_bus.h> #include <rte_pci_dev_feature_defs.h> +#include <rte_uuid.h> #ifdef __cplusplusextern "C" {
@@ -438,6 +439,17 @@ int rte_eal_create_uio_dev(void);*/ enum rte_intr_mode rte_eal_vfio_intr_mode (void ); + +/** +* The user-configured vfio VF token. +* +* @param vf_token +* vfio VF token configured with the command line is copied +* into this parameter, zero uuid by default. +*/ +__rte_experimental +void rte_eal_vfio_vf_token(rte_uuid_t vf_token); +/** * A wrap API for syscall gettid. *
lib/librte_eal/linux/eal.c +29 -0
@@ -558,6 +558,7 @@ eal_usage(const char *prgname)" --" OPT_FILE_PREFIX " Prefix for hugepage filenames\n" " --" OPT_CREATE_UIO_DEV " Create /dev/uioX (usually done by hotplug)\n" " --" OPT_VFIO_INTR " Interrupt mode for VFIO (legacy|msi|msix)\n" +" --" OPT_VFIO_VF_TOKEN " VF token (UUID) shared between SR-IOV PF and VFs\n" " --" OPT_LEGACY_MEM " Legacy memory mode (no dynamic allocation, contiguous segments)\n" " --" OPT_SINGLE_FILE_SEGMENTS " Put all hugepage memory in single files\n" " --" OPT_MATCH_ALLOCATIONS " Free hugepages exactly as allocated\n"
@@ -649,6 +650,19 @@ eal_parse_vfio_intr(const char *mode)return -1 ; } +static int +eal_parse_vfio_vf_token (const char *vf_token) +{ +rte_uuid_t uuid; + +if (!rte_uuid_parse (vf_token, uuid)) { +rte_uuid_copy (internal_config.vfio_vf_token, uuid); +return 0 ; + } + +return -1 ; +} +/* Parse the arguments for --log-level only */ static void eal_log_level_parse (int argc,char **argv)
@@ -795,6 +809,16 @@ eal_parse_args(int argc, char **argv) }break ; +case OPT_VFIO_VF_TOKEN_NUM : +if (eal_parse_vfio_vf_token (optarg) <0 ) { +RTE_LOG (ERR ,EAL ,"invalid parameters for --" +OPT_VFIO_VF_TOKEN "\n" ); +eal_usage (prgname); + ret =-1 ; + goto out; + } +break ; +case OPT_CREATE_UIO_DEV_NUM : internal_config.create_uio_dev =1 ;break ;
@@ -1367,6 +1391,11 @@ rte_eal_vfio_intr_mode(void)return internal_config.vfio_intr_mode; } +void rte_eal_vfio_vf_token (rte_uuid_t vf_token) +{ +rte_uuid_copy (vf_token, internal_config.vfio_vf_token); +} +int rte_eal_check_module (const char *module_name) {
lib/librte_eal/linux/eal_vfio.c +19 -0
@@ -712,6 +712,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,int vfio_container_fd;int vfio_group_fd;int iommu_group_num; +rte_uuid_t vf_token;int i, ret;/* get group number */
@@ -895,6 +896,23 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr, t->type_id, t->name); } +rte_eal_vfio_vf_token (vf_token); + +/* get a file descriptor for the device with VF token firstly */ +if (!rte_uuid_is_null (vf_token)) { +char vf_token_str[RTE_UUID_STRLEN ]; +char dev[PATH_MAX ]; + +rte_uuid_unparse (vf_token, vf_token_str,sizeof (vf_token_str)); +snprintf (dev,sizeof (dev), +"%s vf_token=%s" , dev_addr, vf_token_str); + + *vfio_dev_fd =ioctl (vfio_group_fd,VFIO_GROUP_GET_DEVICE_FD , + dev); +if (*vfio_dev_fd >=0 ) + goto dev_get_info; + } +/* get a file descriptor for the device */ *vfio_dev_fd =ioctl (vfio_group_fd,VFIO_GROUP_GET_DEVICE_FD , dev_addr);if (*vfio_dev_fd <0 ) {
@@ -909,6 +927,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,return -1 ; } +dev_get_info:/* test and setup the device */ ret =ioctl (*vfio_dev_fd,VFIO_DEVICE_GET_INFO , device_info);if (ret) {
lib/librte_eal/rte_eal_version.map +1 -0
@@ -373,6 +373,7 @@ EXPERIMENTAL { __rte_trace_point_register; per_lcore_trace_mem; per_lcore_trace_point_sz; + rte_eal_vfio_vf_token; rte_log_can_log; rte_thread_getname; rte_trace_dump;
Signature · 1 line
2.26.2
Re: [dpdk-dev] [PATCH v12 2/2] eal: support for VFIO-PCI VF token
Quoted 8 lines · Haiyue Wang
The kernel module vfio-pci introduces the VF token to enable SR-IOV support since 5.7.
The VF token can be set by a vfio-pci based PF driver and must be known by the vfio-pci based VF driver in order to gain access to the device.
Signed-off-by Haiyue Wang <[email protected]>---
<snip>
lib/librte_eal/freebsd/eal.c rte_eal_vfio_intr_mode(void) · line 1008
Quoted 12 lines
lib/librte_eal/freebsd/eal.c +4
-0@@ -1002,6 +1002,11 @@ rte_eal_vfio_intr_mode(void)return RTE_INTR_MODE_NONE ; } +void rte_eal_vfio_vf_token (rte_uuid_t vf_token) +{ +memset (vf_token,0 ,sizeof (rte_uuid_t)); +}
What's the purpose of memset(0) here? Presumably, if the API is not supposed to be supported, the function should have no effect?
Quoted 17 lines
lib/librte_eal/freebsd/eal.c +1
-0+ int rte_vfio_setup_device(__rte_unused const char *sysfs_base, __rte_unused const char *dev_addr, __rte_unused int *vfio_dev_fd,lib/librte_eal/include/rte_eal.h +1
-0@@ -21,6 +21,7 @@#include <rte_bus.h> #include <rte_pci_dev_feature_defs.h> +#include <rte_uuid.h> #ifdef __cplusplusextern "C" {
lib/librte_eal/include/rte_eal.h int rte_eal_create_uio_dev(void); · line 444
Quoted 6 lines
lib/librte_eal/include/rte_eal.h +3
-0@@ -438,6 +439,17 @@ int rte_eal_create_uio_dev(void);*/ enum rte_intr_mode rte_eal_vfio_intr_mode (void ); + +/** +* The user-configured vfio VF token.
The description is unclear. Suggested rewording:
Copy the user-configured vfio VF token.
lib/librte_eal/include/rte_eal.h int rte_eal_create_uio_dev(void); · line 451
Quoted 7 lines
lib/librte_eal/include/rte_eal.h +7
-0+* +* @param vf_token +* vfio VF token configured with the command line is copied +* into this parameter, zero uuid by default. +*/ +__rte_experimental +void rte_eal_vfio_vf_token(rte_uuid_t vf_token);
Maybe rte_eal_vfio_get_vf_token()? I imagine this would be a more descriptive name for this API.
Once all of the above is addressed (or explained),
Signature · 2 lines
Thanks, Anatoly
Re: [dpdk-dev] [PATCH v12 2/2] eal: support for VFIO-PCI VF token
Hi Anatoly,
Quoted 27 lines
-----Original Message-----
From: Burakov, Anatoly <[email protected]>
Sent: Wednesday, May 6, 2020 18:27
To: Wang, Haiyue <[email protected]>; [email protected]; [email protected]; [email protected]; [email protected]
Subject: Re: [PATCH v12 2/2] eal: support for VFIO-PCI VF tokenQuoted 8 lines · Haiyue Wang
The kernel module vfio-pci introduces the VF token to enable SR-IOV support since 5.7.
The VF token can be set by a vfio-pci based PF driver and must be known by the vfio-pci based VF driver in order to gain access to the device.
Signed-off-by Haiyue Wang <[email protected]>---
<snip>
Quoted 8 lines
lib/librte_eal/freebsd/eal.c +0
-0@@ -1002,6 +1002,11 @@ rte_eal_vfio_intr_mode(void)return RTE_INTR_MODE_NONE ; }
Quoted 8 lines
lib/librte_eal/freebsd/eal.c rte_eal_vfio_intr_mode(void) · line 1008
Quoted 4 lines
lib/librte_eal/freebsd/eal.c +4
-0+void rte_eal_vfio_vf_token (rte_uuid_t vf_token) +{ +memset (vf_token,0 ,sizeof (rte_uuid_t)); +}What's the purpose of memset(0) here? Presumably, if the API is not supposed to be supported, the function should have no effect?
Yes, originally I meant to no parse the input, then return zero UUID. I changed it to the same as Linux:
rte_uuid_copy(vf_token, internal_config.vfio_vf_token);
Since it is inited to zero.
Quoted 20 lines
Quoted 20 lines
lib/librte_eal/freebsd/eal.c +1
-0+ int rte_vfio_setup_device(__rte_unused const char *sysfs_base, __rte_unused const char *dev_addr, __rte_unused int *vfio_dev_fd,lib/librte_eal/include/rte_eal.h +1
-0@@ -21,6 +21,7 @@#include <rte_bus.h> #include <rte_pci_dev_feature_defs.h> +#include <rte_uuid.h> #ifdef __cplusplusextern "C" {@@ -438,6 +439,17 @@ int rte_eal_create_uio_dev(void);*/ enum rte_intr_mode rte_eal_vfio_intr_mode (void );
Quoted 8 lines
lib/librte_eal/include/rte_eal.h int rte_eal_create_uio_dev(void); · line 444
Quoted 3 lines
lib/librte_eal/include/rte_eal.h +3
-0+ +/** +* The user-configured vfio VF token. The description is unclear. Suggested rewording:
Copy the user-configured vfio VF token.
+1.
Quoted 11 lines
lib/librte_eal/include/rte_eal.h int rte_eal_create_uio_dev(void); · line 451
Quoted 7 lines
lib/librte_eal/include/rte_eal.h +7
-0+* +* @param vf_token +* vfio VF token configured with the command line is copied +* into this parameter, zero uuid by default. +*/ +__rte_experimental +void rte_eal_vfio_vf_token(rte_uuid_t vf_token);Maybe
rte_eal_vfio_get_vf_token()? I imagine this would be a more descriptive name for this API.
Yes, this looks better. Fixed.
Quoted 7 lines
Once all of the above is addressed (or explained),
Acked-by Anatoly Burakov <[email protected]>--
Thanks,
Anatoly
Re: [dpdk-dev] [PATCH v12 2/2] eal: support for VFIO-PCI VF token
Quoted 38 lines · Haiyue
Hi Anatoly,
Quoted 27 lines
-----Original Message-----
From: Burakov, Anatoly <[email protected]>
Sent: Wednesday, May 6, 2020 18:27
To: Wang, Haiyue <[email protected]>; [email protected]; [email protected]; [email protected]; [email protected]
Subject: Re: [PATCH v12 2/2] eal: support for VFIO-PCI VF tokenQuoted 8 lines · Haiyue Wang
The kernel module vfio-pci introduces the VF token to enable SR-IOV support since 5.7.
The VF token can be set by a vfio-pci based PF driver and must be known by the vfio-pci based VF driver in order to gain access to the device.
Signed-off-by Haiyue Wang <[email protected]>---
<snip>
Quoted 8 lines
lib/librte_eal/freebsd/eal.c +0
-0@@ -1002,6 +1002,11 @@ rte_eal_vfio_intr_mode(void)return RTE_INTR_MODE_NONE ; }Quoted 8 lines
lib/librte_eal/freebsd/eal.c rte_eal_vfio_intr_mode(void) · line 1008
Quoted 4 lines
lib/librte_eal/freebsd/eal.c +4
-0+void rte_eal_vfio_vf_token (rte_uuid_t vf_token) +{ +memset (vf_token,0 ,sizeof (rte_uuid_t)); +}What's the purpose of memset(0) here? Presumably, if the API is not supposed to be supported, the function should have no effect?
Quoted 4 lines
Yes, originally I meant to no parse the input, then return zero UUID. I changed it to the same as Linux:
rte_uuid_copy(vf_token, internal_config.vfio_vf_token);Since it is inited to zero.
Yes but on Linux it has effect because Linux supports it. On FreeBSD there's no VFIO, so the function should IMO have no effect. However, i don't have a strong opinion on this, so i'm fine either way.
Signature · 2 lines
Thanks, Anatoly
Re: [dpdk-dev] [PATCH v12 2/2] eal: support for VFIO-PCI VF token
Quoted 46 lines
-----Original Message-----
From: Burakov, Anatoly <[email protected]>
Sent: Wednesday, May 6, 2020 19:40
To: Wang, Haiyue <[email protected]>; [email protected]; [email protected]; [email protected]; [email protected]
Subject: Re: [PATCH v12 2/2] eal: support for VFIO-PCI VF tokenQuoted 38 lines · Haiyue
Hi Anatoly,
Quoted 35 lines
-----Original Message-----
From: Burakov, Anatoly <[email protected]>
Sent: Wednesday, May 6, 2020 18:27
To: Wang, Haiyue <[email protected]>; [email protected]; [email protected]; [email protected]; [email protected]
Subject: Re: [PATCH v12 2/2] eal: support for VFIO-PCI VF tokenQuoted 8 lines · Haiyue Wang
The kernel module vfio-pci introduces the VF token to enable SR-IOV support since 5.7.
The VF token can be set by a vfio-pci based PF driver and must be known by the vfio-pci based VF driver in order to gain access to the device.
Signed-off-by Haiyue Wang <[email protected]>---
<snip>
lib/librte_eal/freebsd/eal.c rte_eal_vfio_intr_mode(void) · line 1008
Quoted 12 lines
lib/librte_eal/freebsd/eal.c +4
-0@@ -1002,6 +1002,11 @@ rte_eal_vfio_intr_mode(void)return RTE_INTR_MODE_NONE ; } +void rte_eal_vfio_vf_token (rte_uuid_t vf_token) +{ +memset (vf_token,0 ,sizeof (rte_uuid_t)); +}What's the purpose of memset(0) here? Presumably, if the API is not supposed to be supported, the function should have no effect?
Quoted 9 lines
Quoted 4 lines
Yes, originally I meant to no parse the input, then return zero UUID. I changed it to the same as Linux:
rte_uuid_copy(vf_token, internal_config.vfio_vf_token);Since it is inited to zero.
Yes but on Linux it has effect because Linux supports it. On FreeBSD there's no VFIO, so the function should IMO have no effect. However, i don't have a strong opinion on this, so i'm fine either way.
Uh, seems empty function should be good one. I will change it. ;-)
Quoted 3 lines
--
Thanks,
Anatoly
v13 cover support for VFIO-PCI VF token interface
In reply to [email protected]v13: Rename the EAL get VF token function, and leave the freebsd type as empty.
v12: support to vfio devices with VF token and no token.
v11: Use the eal parameter to pass the VF token, then not every PCI
device needs to be specified with this token. Also no ABI issue now.
v10: Use the __rte_internal to mark the internal API changing.
v9: Rewrite the document.
v8: Update the document.
v7: Add the Fixes tag in uuid, the release note and help
document.
v6: Drop the Fixes tag in uuid, since the file has been
moved to another place, not suitable to apply on stable. And this is not a bug, just some kind of enhancement.
v5: 1. Add the VF token parse error handling.
2. Split into two patches for different logic module.
3. Add more comments into the code for explaining the design.
4. Drop the ABI change workaround, this patch set focuses on code review.
v4: 1. Ignore rte_vfio_setup_device ABI check since it is
for Linux driver use.
v3: Fix the Travis build failed:
(1). rte_uuid.h:97:55: error: unknown type name ‘size_t’ (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’
v2: Fix the FreeBSD build error.
v1: Update the commit message.
RFC v2:
Based on Vamsi's RFC v1, and Alex's patch for Qemu
[https://lore.kernel.org/lkml/[email protected]/]:
Use the devarg to pass-down the VF token.
RFC v1: https://patchwork.dpdk.org/patch/66281/ by Vamsi.
Haiyue Wang (2):
eal: add uuid dependent header files explicitly
eal: support for VFIO-PCI VF token
doc/guides/linux_gsg/linux_drivers.rst | 35 ++++++++++++++++++- doc/guides/linux_gsg/linux_eal_parameters.rst | 4 +++ doc/guides/rel_notes/release_20_05.rst | 6 ++++ lib/librte_eal/common/eal_common_options.c | 2 ++ lib/librte_eal/common/eal_internal_cfg.h | 2 ++ lib/librte_eal/common/eal_options.h | 2 ++ lib/librte_eal/freebsd/eal.c | 4 +++ lib/librte_eal/include/rte_eal.h | 12 +++++++ lib/librte_eal/include/rte_uuid.h | 2 ++ lib/librte_eal/linux/eal.c | 29 +++++++++++++++ lib/librte_eal/linux/eal_vfio.c | 19 ++++++++++ lib/librte_eal/rte_eal_version.map | 1 + 12 files changed, 117 insertions(+), 1 deletion(-)
Signature · 1 line
2.26.2
PATCH v13 1/2 eal: add uuid dependent header files explicitly
Add the dependent header files explicitly, so that the user just needs to include the 'rte_uuid.h' header file directly to avoid compile error:
(1). rte_uuid.h:97:55: error: unknown type name ‘size_t’ (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’
--- lib/librte_eal/include/rte_uuid.h | 2 ++ 1 file changed, 2 insertions(+)
lib/librte_eal/include/rte_uuid.h +2 -0
@@ -15,6 +15,8 @@ extern "C" {#endif #include <stdbool.h> +#include <stddef.h> +#include <string.h> /** * Struct describing a Universal Unique Identifier
Signature · 1 line
2.26.2
PATCH v13 2/2 eal: support for VFIO-PCI VF token
In reply to Haiyue WangThe kernel module vfio-pci introduces the VF token to enable SR-IOV support since 5.7.
The VF token can be set by a vfio-pci based PF driver and must be known by the vfio-pci based VF driver in order to gain access to the device.
--- doc/guides/linux_gsg/linux_drivers.rst | 35 ++++++++++++++++++- doc/guides/linux_gsg/linux_eal_parameters.rst | 4 +++ doc/guides/rel_notes/release_20_05.rst | 6 ++++ lib/librte_eal/common/eal_common_options.c | 2 ++ lib/librte_eal/common/eal_internal_cfg.h | 2 ++ lib/librte_eal/common/eal_options.h | 2 ++ lib/librte_eal/freebsd/eal.c | 4 +++ lib/librte_eal/include/rte_eal.h | 12 +++++++ lib/librte_eal/linux/eal.c | 29 +++++++++++++++ lib/librte_eal/linux/eal_vfio.c | 19 ++++++++++ lib/librte_eal/rte_eal_version.map | 1 + 11 files changed, 115 insertions(+), 1 deletion(-)
doc/guides/linux_gsg/linux_drivers.rst +34 -1
@@ -72,11 +72,44 @@ Note that in order to use VFIO, your kernel must support it. VFIO kernel modules have been included in the Linux kernel since version 3.6.0 and are usually present by default, however please consult your distributions documentation to make sure that is the case. +The ``vfio-pci`` module since Linux version 5.7 supports the creation of virtual +functions. After the PF is bound to vfio-pci module, the user can create the VFs +by sysfs interface, and these VFs are bound to vfio-pci module automatically. + +When the PF is bound to vfio-pci, it has initial VF token generated by random. For +security reason, this token is write only, the user can't read it from the kernel +directly. For accessing the VF, the user needs to start the PF with token parameter +to setup a VF token (uuid format), then the VF can be accessed with this new known +VF token. + +DPDK will use the EAL parameter ``--vfio-vf-token`` to specify the VF token value to +PF and its related VFs, this VF token will be shared in all VFIO devices, including +the different PFs. + +.. code-block:: console + + 1. Generate the VF token by uuid command + 14d63f20-8445-11ea-8900-1f9ce7d5650d + + 2. sudo modprobe vfio-pci enable_sriov=1 + + 2. ./usertools/dpdk-devbind.py -b vfio-pci 0000:86:00.0 + + 3. echo 2 > /sys/bus/pci/devices/0000:86:00.0/sriov_numvfs + + 4. Start the PF: + ./x86_64-native-linux-gcc/app/testpmd -l 22-25 -n 4 -w 86:00.0 \ + --vfio-vf-token=14d63f20-8445-11ea-8900-1f9ce7d5650d --file-prefix=pf -- -i + + 5. Start the VF: + ./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 -w 86:02.0 \ + --vfio-vf-token=14d63f20-8445-11ea-8900-1f9ce7d5650d --file-prefix=vf0 -- -i + Also, to use VFIO, both kernel and BIOS must support and be configured to use IO virtualization (such as Intel® VT-d). .. note:: - ``vfio-pci`` module doesn't support the creation of virtual functions. + ``vfio-pci`` module doesn't support the creation of virtual functions before Linux version 5.7. For proper operation of VFIO when running DPDK applications as a non-privileged user, correct permissions should also be set up. This can be done by using the DPDK setup script (called dpdk-setup.sh and located in the usertools directory).
doc/guides/linux_gsg/linux_eal_parameters.rst +4 -0
@@ -40,6 +40,10 @@ Device-related options Use specified interrupt mode for devices bound to VFIO kernel driver. +* ``--vfio-vf-token <uuid>`` + + Use specified VF token for devices bound to VFIO kernel driver. + Multiprocessing-related options ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
doc/guides/rel_notes/release_20_05.rst +6 -0
@@ -93,6 +93,12 @@ New Features * Added new query: ``rte_flow_get_aged_flows`` to get the aged-out flows contexts from the port. +* **Added the support for vfio-pci new VF token interface.** + + Since Linux version 5.7, vfio-pci supports a shared VF token (UUID) to represent + the trust between SR-IOV PF and the created VFs. Update the method to gain access + to the PF and VFs devices by appending the VF token parameter. + * **Updated Amazon ena driver.** Updated ena PMD with new features and improvements, including:
lib/librte_eal/common/eal_common_options.c +2 -0
@@ -89,6 +89,7 @@ eal_long_options[] = { {OPT_SYSLOG, 1, NULL, OPT_SYSLOG_NUM }, {OPT_VDEV, 1, NULL, OPT_VDEV_NUM }, {OPT_VFIO_INTR, 1, NULL, OPT_VFIO_INTR_NUM }, + {OPT_VFIO_VF_TOKEN, 1, NULL, OPT_VFIO_VF_TOKEN_NUM }, {OPT_VMWARE_TSC_MAP, 0, NULL, OPT_VMWARE_TSC_MAP_NUM }, {OPT_LEGACY_MEM, 0, NULL, OPT_LEGACY_MEM_NUM }, {OPT_SINGLE_FILE_SEGMENTS, 0, NULL, OPT_SINGLE_FILE_SEGMENTS_NUM},
@@ -222,6 +223,7 @@ eal_reset_internal_config(struct internal_config *internal_cfg)/* if set to NONE, interrupt mode is determined automatically */ internal_cfg->vfio_intr_mode =RTE_INTR_MODE_NONE ; +memset (internal_cfg->vfio_vf_token,0 ,sizeof (rte_uuid_t));#ifdef RTE_LIBEAL_USE_HPET internal_cfg->no_hpet =0 ;
lib/librte_eal/common/eal_internal_cfg.h +2 -0
@@ -72,6 +72,8 @@ struct internal_config {volatile int syslog_facility;/**< facility passed to openlog() */ /** default interrupt mode for VFIO */ volatile enum rte_intr_mode vfio_intr_mode; +/** the shared VF token for VFIO-PCI bound PF and VFs devices */ +rte_uuid_t vfio_vf_token;char *hugefile_prefix;/**< the base filename of hugetlbfs files */ char *hugepage_dir;/**< specific hugetlbfs directory to use */ char *user_mbuf_pool_ops_name;
lib/librte_eal/common/eal_options.h +2 -0
@@ -67,6 +67,8 @@ enum {OPT_VDEV_NUM , #define OPT_VFIO_INTR "vfio-intr"OPT_VFIO_INTR_NUM , +#define OPT_VFIO_VF_TOKEN "vfio-vf-token" +OPT_VFIO_VF_TOKEN_NUM , #define OPT_VMWARE_TSC_MAP "vmware-tsc-map"OPT_VMWARE_TSC_MAP_NUM , #define OPT_LEGACY_MEM "legacy-mem"
lib/librte_eal/freebsd/eal.c +4 -0
@@ -1002,6 +1002,10 @@ rte_eal_vfio_intr_mode(void)return RTE_INTR_MODE_NONE ; } +void rte_eal_vfio_get_vf_token(__rte_unused rte_uuid_t vf_token) +{ +} + int rte_vfio_setup_device(__rte_unused const char *sysfs_base, __rte_unused const char *dev_addr, __rte_unused int *vfio_dev_fd,
lib/librte_eal/include/rte_eal.h +12 -0
@@ -21,6 +21,7 @@#include <rte_bus.h> #include <rte_pci_dev_feature_defs.h> +#include <rte_uuid.h> #ifdef __cplusplusextern "C" {
@@ -438,6 +439,17 @@ int rte_eal_create_uio_dev(void);*/ enum rte_intr_mode rte_eal_vfio_intr_mode (void ); + +/** +* Copy the user-configured vfio VF token. +* +* @param vf_token +* vfio VF token configured with the command line is copied +* into this parameter, zero uuid by default. +*/ +__rte_experimental +void rte_eal_vfio_get_vf_token(rte_uuid_t vf_token); +/** * A wrap API for syscall gettid. *
lib/librte_eal/linux/eal.c +29 -0
@@ -558,6 +558,7 @@ eal_usage(const char *prgname)" --" OPT_FILE_PREFIX " Prefix for hugepage filenames\n" " --" OPT_CREATE_UIO_DEV " Create /dev/uioX (usually done by hotplug)\n" " --" OPT_VFIO_INTR " Interrupt mode for VFIO (legacy|msi|msix)\n" +" --" OPT_VFIO_VF_TOKEN " VF token (UUID) shared between SR-IOV PF and VFs\n" " --" OPT_LEGACY_MEM " Legacy memory mode (no dynamic allocation, contiguous segments)\n" " --" OPT_SINGLE_FILE_SEGMENTS " Put all hugepage memory in single files\n" " --" OPT_MATCH_ALLOCATIONS " Free hugepages exactly as allocated\n"
@@ -649,6 +650,19 @@ eal_parse_vfio_intr(const char *mode)return -1 ; } +static int +eal_parse_vfio_vf_token (const char *vf_token) +{ +rte_uuid_t uuid; + +if (!rte_uuid_parse (vf_token, uuid)) { +rte_uuid_copy (internal_config.vfio_vf_token, uuid); +return 0 ; + } + +return -1 ; +} +/* Parse the arguments for --log-level only */ static void eal_log_level_parse (int argc,char **argv)
@@ -795,6 +809,16 @@ eal_parse_args(int argc, char **argv) }break ; +case OPT_VFIO_VF_TOKEN_NUM : +if (eal_parse_vfio_vf_token (optarg) <0 ) { +RTE_LOG (ERR ,EAL ,"invalid parameters for --" +OPT_VFIO_VF_TOKEN "\n" ); +eal_usage (prgname); + ret =-1 ; + goto out; + } +break ; +case OPT_CREATE_UIO_DEV_NUM : internal_config.create_uio_dev =1 ;break ;
@@ -1367,6 +1391,11 @@ rte_eal_vfio_intr_mode(void)return internal_config.vfio_intr_mode; } +void rte_eal_vfio_get_vf_token (rte_uuid_t vf_token) +{ +rte_uuid_copy (vf_token, internal_config.vfio_vf_token); +} +int rte_eal_check_module (const char *module_name) {
lib/librte_eal/linux/eal_vfio.c +19 -0
@@ -712,6 +712,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,int vfio_container_fd;int vfio_group_fd;int iommu_group_num; +rte_uuid_t vf_token;int i, ret;/* get group number */
@@ -895,6 +896,23 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr, t->type_id, t->name); } +rte_eal_vfio_get_vf_token (vf_token); + +/* get a file descriptor for the device with VF token firstly */ +if (!rte_uuid_is_null (vf_token)) { +char vf_token_str[RTE_UUID_STRLEN ]; +char dev[PATH_MAX ]; + +rte_uuid_unparse (vf_token, vf_token_str,sizeof (vf_token_str)); +snprintf (dev,sizeof (dev), +"%s vf_token=%s" , dev_addr, vf_token_str); + + *vfio_dev_fd =ioctl (vfio_group_fd,VFIO_GROUP_GET_DEVICE_FD , + dev); +if (*vfio_dev_fd >=0 ) + goto dev_get_info; + } +/* get a file descriptor for the device */ *vfio_dev_fd =ioctl (vfio_group_fd,VFIO_GROUP_GET_DEVICE_FD , dev_addr);if (*vfio_dev_fd <0 ) {
@@ -909,6 +927,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,return -1 ; } +dev_get_info:/* test and setup the device */ ret =ioctl (*vfio_dev_fd,VFIO_DEVICE_GET_INFO , device_info);if (ret) {
lib/librte_eal/rte_eal_version.map +1 -0
@@ -373,6 +373,7 @@ EXPERIMENTAL { __rte_trace_point_register; per_lcore_trace_mem; per_lcore_trace_point_sz; + rte_eal_vfio_get_vf_token; rte_log_can_log; rte_thread_getname; rte_trace_dump;
Signature · 1 line
2.26.2
Re: [dpdk-dev] [PATCH v13 2/2] eal: support for VFIO-PCI VF token
Quoted 8 lines · Haiyue Wang
The kernel module vfio-pci introduces the VF token to enable SR-IOV support since 5.7.
The VF token can be set by a vfio-pci based PF driver and must be known by the vfio-pci based VF driver in order to gain access to the device.
Signed-off-by Haiyue Wang <[email protected]>Acked-by Anatoly Burakov <[email protected]>
Sorry, lost from my view new versions of the patch series.
Quoted 7 lines
doc/guides/linux_gsg/linux_drivers.rst +0
-0@@ -72,11 +72,44 @@ Note that in order to use VFIO, your kernel must support it. VFIO kernel modules have been included in the Linux kernel since version 3.6.0 and are usually present by default, however please consult your distributions documentation to make sure that is the case.
doc/guides/linux_gsg/linux_drivers.rst Note that in order to use VFIO, your kernel must support it. · line 83
Quoted 10 lines
doc/guides/linux_gsg/linux_drivers.rst +9
-0+The ``vfio-pci`` module since Linux version 5.7 supports the creation of virtual +functions. After the PF is bound to vfio-pci module, the user can create the VFs +by sysfs interface, and these VFs are bound to vfio-pci module automatically. + +When the PF is bound to vfio-pci, it has initial VF token generated by random. For +security reason, this token is write only, the user can't read it from the kernel +directly. For accessing the VF, the user needs to start the PF with token parameter +to setup a VF token (uuid format), then the VF can be accessed with this new known +VF token.
If token is write-only in kernel sysfs, shouldn't we make it invisible in ps output? I.e. substitute with something like xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx.
It is a bit easier with the new design. Just a thought.
[snip]
Re: [dpdk-dev] [PATCH v13 2/2] eal: support for VFIO-PCI VF token
Quoted 30 lines
-----Original Message-----
From: Andrew Rybchenko <[email protected]>
Sent: Thursday, May 7, 2020 00:51
To: Wang, Haiyue <[email protected]>; [email protected]; Burakov, Anatoly <[email protected]>; [email protected]; [email protected]; [email protected]
Subject: Re: [dpdk-dev] [PATCH v13 2/2] eal: support for VFIO-PCI VF tokenQuoted 8 lines · Haiyue Wang
The kernel module vfio-pci introduces the VF token to enable SR-IOV support since 5.7.
The VF token can be set by a vfio-pci based PF driver and must be known by the vfio-pci based VF driver in order to gain access to the device.
Signed-off-by Haiyue Wang <[email protected]>Acked-by Anatoly Burakov <[email protected]>Sorry, lost from my view new versions of the patch series.
Acked-by Andrew Rybchenko <[email protected]>doc/guides/linux_gsg/linux_drivers.rst Note that in order to use VFIO, your kernel must support it. · line 72
Quoted 6 lines
doc/guides/linux_gsg/linux_drivers.rst +0
-0@@ -72,11 +72,44 @@ Note that in order to use VFIO, your kernel must support it. VFIO kernel modules have been included in the Linux kernel since version 3.6.0 and are usuallypresent by default,
Quoted 2 lines
doc/guides/linux_gsg/linux_drivers.rst +0
-0however please consult your distributions documentation to make sure that is the case.
Quoted 15 lines
doc/guides/linux_gsg/linux_drivers.rst Note that in order to use VFIO, your kernel must support it. · line 83
Quoted 9 lines
doc/guides/linux_gsg/linux_drivers.rst +9
-0+The ``vfio-pci`` module since Linux version 5.7 supports the creation of virtual +functions. After the PF is bound to vfio-pci module, the user can create the VFs +by sysfs interface, and these VFs are bound to vfio-pci module automatically. + +When the PF is bound to vfio-pci, it has initial VF token generated by random. For +security reason, this token is write only, the user can't read it from the kernel +directly. For accessing the VF, the user needs to start the PF with token parameter +to setup a VF token (uuid format), then the VF can be accessed with this new known +VF token.If token is write-only in kernel sysfs, shouldn't we make it invisible in ps output? I.e. substitute with something like xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx.
It is a bit easier with the new design. Just a thought.
In fact, no sysfs for VF token, just write-only IOCTL. ;-)
Quoted 1 line
[snip]
Re: [dpdk-dev] [PATCH v13 2/2] eal: support for VFIO-PCI VF token
Quoted 46 lines · Haiyue
Quoted 30 lines
-----Original Message-----
From: Andrew Rybchenko <[email protected]>
Sent: Thursday, May 7, 2020 00:51
To: Wang, Haiyue <[email protected]>; [email protected]; Burakov, Anatoly <[email protected]>; [email protected]; [email protected]; [email protected]
Subject: Re: [dpdk-dev] [PATCH v13 2/2] eal: support for VFIO-PCI VF tokenQuoted 8 lines · Haiyue Wang
The kernel module vfio-pci introduces the VF token to enable SR-IOV support since 5.7.
The VF token can be set by a vfio-pci based PF driver and must be known by the vfio-pci based VF driver in order to gain access to the device.
Signed-off-by Haiyue Wang <[email protected]>Acked-by Anatoly Burakov <[email protected]>Sorry, lost from my view new versions of the patch series.
Acked-by Andrew Rybchenko <[email protected]>doc/guides/linux_gsg/linux_drivers.rst Note that in order to use VFIO, your kernel must support it. · line 72
Quoted 6 lines
doc/guides/linux_gsg/linux_drivers.rst +0
-0@@ -72,11 +72,44 @@ Note that in order to use VFIO, your kernel must support it. VFIO kernel modules have been included in the Linux kernel since version 3.6.0 and are usuallypresent by default,
Quoted 2 lines
doc/guides/linux_gsg/linux_drivers.rst +0
-0however please consult your distributions documentation to make sure that is the case.Quoted 15 lines
doc/guides/linux_gsg/linux_drivers.rst Note that in order to use VFIO, your kernel must support it. · line 83
Quoted 9 lines
doc/guides/linux_gsg/linux_drivers.rst +9
-0+The ``vfio-pci`` module since Linux version 5.7 supports the creation of virtual +functions. After the PF is bound to vfio-pci module, the user can create the VFs +by sysfs interface, and these VFs are bound to vfio-pci module automatically. + +When the PF is bound to vfio-pci, it has initial VF token generated by random. For +security reason, this token is write only, the user can't read it from the kernel +directly. For accessing the VF, the user needs to start the PF with token parameter +to setup a VF token (uuid format), then the VF can be accessed with this new known +VF token.If token is write-only in kernel sysfs, shouldn't we make it invisible in ps output? I.e. substitute with something like xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx.
It is a bit easier with the new design. Just a thought.
Quoted 1 line
In fact, no sysfs for VF token, just write-only IOCTL. ;-)
OK, got it. The question remains anyway. Should it be treated as a secret with at least minimal security precaution?
Re: [dpdk-dev] [PATCH v13 2/2] eal: support for VFIO-PCI VF token
+Alex
Quoted 55 lines
-----Original Message-----
From: Andrew Rybchenko <[email protected]>
Sent: Thursday, May 7, 2020 00:59
To: Wang, Haiyue <[email protected]>; [email protected]; Burakov, Anatoly <[email protected]>; [email protected]; [email protected]; [email protected]
Subject: Re: [dpdk-dev] [PATCH v13 2/2] eal: support for VFIO-PCI VF tokenQuoted 4 lines · Haiyue
Quoted 4 lines
-----Original Message-----
From: Andrew Rybchenko <[email protected]>
Sent: Thursday, May 7, 2020 00:51
To: Wang, Haiyue <[email protected]>; [email protected]; Burakov, AnatolyQuoted 42 lines
Quoted 41 lines
[email protected]; [email protected]; [email protected]
Subject: Re: [dpdk-dev] [PATCH v13 2/2] eal: support for VFIO-PCI VF tokenQuoted 8 lines · Haiyue Wang
The kernel module vfio-pci introduces the VF token to enable SR-IOV support since 5.7.
The VF token can be set by a vfio-pci based PF driver and must be known by the vfio-pci based VF driver in order to gain access to the device.
Signed-off-by Haiyue Wang <[email protected]>Acked-by Anatoly Burakov <[email protected]>Sorry, lost from my view new versions of the patch series.
Acked-by Andrew Rybchenko <[email protected]>doc/guides/linux_gsg/linux_drivers.rst Note that in order to use VFIO, your kernel must support it. · line 72
Quoted 6 lines
doc/guides/linux_gsg/linux_drivers.rst +0
-0@@ -72,11 +72,44 @@ Note that in order to use VFIO, your kernel must support it. VFIO kernel modules have been included in the Linux kernel since version 3.6.0 and are usuallypresent by default,
doc/guides/linux_gsg/linux_drivers.rst Note that in order to use VFIO, your kernel must support it. · line 83
Quoted 11 lines
doc/guides/linux_gsg/linux_drivers.rst +9
-0however please consult your distributions documentation to make sure that is the case. +The ``vfio-pci`` module since Linux version 5.7 supports the creation of virtual +functions. After the PF is bound to vfio-pci module, the user can create the VFs +by sysfs interface, and these VFs are bound to vfio-pci module automatically. + +When the PF is bound to vfio-pci, it has initial VF token generated by random. For +security reason, this token is write only, the user can't read it from the kernel +directly. For accessing the VF, the user needs to start the PF with token parameter +to setup a VF token (uuid format), then the VF can be accessed with this new known +VF token.If token is write-only in kernel sysfs, shouldn't we make it invisible in ps output? I.e. substitute with something like xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx.
It is a bit easier with the new design. Just a thought.
Quoted 5 lines
Quoted 1 line
In fact, no sysfs for VF token, just write-only IOCTL. ;-)
OK, got it. The question remains anyway. Should it be treated as a secret with at least minimal security precaution?
Sounds yes, and also it looks like be more friendly for user to check whether this PF/VF have a VF token required or not by cat /sys/...
@Alex may consider this design.
v14 cover support for VFIO-PCI VF token interface
In reply to [email protected]v14: Rebase the patch for 20.08 release note.
v13: Rename the EAL get VF token function, and leave the freebsd type as empty.
v12: support to vfio devices with VF token and no token.
v11: Use the eal parameter to pass the VF token, then not every PCI
device needs to be specified with this token. Also no ABI issue now.
v10: Use the __rte_internal to mark the internal API changing.
v9: Rewrite the document.
v8: Update the document.
v7: Add the Fixes tag in uuid, the release note and help
document.
v6: Drop the Fixes tag in uuid, since the file has been
moved to another place, not suitable to apply on stable. And this is not a bug, just some kind of enhancement.
v5: 1. Add the VF token parse error handling.
2. Split into two patches for different logic module.
3. Add more comments into the code for explaining the design.
4. Drop the ABI change workaround, this patch set focuses on code review.
v4: 1. Ignore rte_vfio_setup_device ABI check since it is
for Linux driver use.
v3: Fix the Travis build failed:
(1). rte_uuid.h:97:55: error: unknown type name ‘size_t’ (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’
v2: Fix the FreeBSD build error.
v1: Update the commit message.
RFC v2:
Based on Vamsi's RFC v1, and Alex's patch for Qemu
[https://lore.kernel.org/lkml/[email protected]/]:
Use the devarg to pass-down the VF token.
RFC v1: https://patchwork.dpdk.org/patch/66281/ by Vamsi.
Haiyue Wang (2):
eal: add uuid dependent header files explicitly
eal: support for VFIO-PCI VF token
doc/guides/linux_gsg/linux_drivers.rst | 35 ++++++++++++++++++- doc/guides/linux_gsg/linux_eal_parameters.rst | 4 +++ doc/guides/rel_notes/release_20_08.rst | 5 +++ lib/librte_eal/common/eal_common_options.c | 2 ++ lib/librte_eal/common/eal_internal_cfg.h | 2 ++ lib/librte_eal/common/eal_options.h | 2 ++ lib/librte_eal/freebsd/eal.c | 4 +++ lib/librte_eal/include/rte_eal.h | 12 +++++++ lib/librte_eal/include/rte_uuid.h | 2 ++ lib/librte_eal/linux/eal.c | 29 +++++++++++++++ lib/librte_eal/linux/eal_vfio.c | 19 ++++++++++ lib/librte_eal/rte_eal_version.map | 1 + 12 files changed, 116 insertions(+), 1 deletion(-)
Signature · 1 line
2.26.2
PATCH v14 1/2 eal: add uuid dependent header files explicitly
Add the dependent header files explicitly, so that the user just needs to include the 'rte_uuid.h' header file directly to avoid compile error:
(1). rte_uuid.h:97:55: error: unknown type name ‘size_t’ (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’
--- lib/librte_eal/include/rte_uuid.h | 2 ++ 1 file changed, 2 insertions(+)
lib/librte_eal/include/rte_uuid.h +2 -0
@@ -15,6 +15,8 @@ extern "C" {#endif #include <stdbool.h> +#include <stddef.h> +#include <string.h> /** * Struct describing a Universal Unique Identifier
Signature · 1 line
2.26.2
PATCH v14 2/2 eal: support for VFIO-PCI VF token
In reply to Haiyue WangThe kernel module vfio-pci introduces the VF token to enable SR-IOV support since 5.7.
The VF token can be set by a vfio-pci based PF driver and must be known by the vfio-pci based VF driver in order to gain access to the device.
--- doc/guides/linux_gsg/linux_drivers.rst | 35 ++++++++++++++++++- doc/guides/linux_gsg/linux_eal_parameters.rst | 4 +++ doc/guides/rel_notes/release_20_08.rst | 5 +++ lib/librte_eal/common/eal_common_options.c | 2 ++ lib/librte_eal/common/eal_internal_cfg.h | 2 ++ lib/librte_eal/common/eal_options.h | 2 ++ lib/librte_eal/freebsd/eal.c | 4 +++ lib/librte_eal/include/rte_eal.h | 12 +++++++ lib/librte_eal/linux/eal.c | 29 +++++++++++++++ lib/librte_eal/linux/eal_vfio.c | 19 ++++++++++ lib/librte_eal/rte_eal_version.map | 1 + 11 files changed, 114 insertions(+), 1 deletion(-)
doc/guides/linux_gsg/linux_drivers.rst +34 -1
@@ -78,11 +78,44 @@ Note that in order to use VFIO, your kernel must support it. VFIO kernel modules have been included in the Linux kernel since version 3.6.0 and are usually present by default, however please consult your distributions documentation to make sure that is the case. +The ``vfio-pci`` module since Linux version 5.7 supports the creation of virtual +functions. After the PF is bound to vfio-pci module, the user can create the VFs +by sysfs interface, and these VFs are bound to vfio-pci module automatically. + +When the PF is bound to vfio-pci, it has initial VF token generated by random. For +security reason, this token is write only, the user can't read it from the kernel +directly. For accessing the VF, the user needs to start the PF with token parameter +to setup a VF token (uuid format), then the VF can be accessed with this new known +VF token. + +DPDK will use the EAL parameter ``--vfio-vf-token`` to specify the VF token value to +PF and its related VFs, this VF token will be shared in all VFIO devices, including +the different PFs. + +.. code-block:: console + + 1. Generate the VF token by uuid command + 14d63f20-8445-11ea-8900-1f9ce7d5650d + + 2. sudo modprobe vfio-pci enable_sriov=1 + + 2. ./usertools/dpdk-devbind.py -b vfio-pci 0000:86:00.0 + + 3. echo 2 > /sys/bus/pci/devices/0000:86:00.0/sriov_numvfs + + 4. Start the PF: + ./x86_64-native-linux-gcc/app/testpmd -l 22-25 -n 4 -w 86:00.0 \ + --vfio-vf-token=14d63f20-8445-11ea-8900-1f9ce7d5650d --file-prefix=pf -- -i + + 5. Start the VF: + ./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 -w 86:02.0 \ + --vfio-vf-token=14d63f20-8445-11ea-8900-1f9ce7d5650d --file-prefix=vf0 -- -i + Also, to use VFIO, both kernel and BIOS must support and be configured to use IO virtualization (such as Intel® VT-d). .. note:: - ``vfio-pci`` module doesn't support the creation of virtual functions. + ``vfio-pci`` module doesn't support the creation of virtual functions before Linux version 5.7. For proper operation of VFIO when running DPDK applications as a non-privileged user, correct permissions should also be set up. This can be done by using the DPDK setup script (called dpdk-setup.sh and located in the usertools directory).
doc/guides/linux_gsg/linux_eal_parameters.rst +4 -0
@@ -40,6 +40,10 @@ Device-related options Use specified interrupt mode for devices bound to VFIO kernel driver. +* ``--vfio-vf-token <uuid>`` + + Use specified VF token for devices bound to VFIO kernel driver. + Multiprocessing-related options ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
doc/guides/rel_notes/release_20_08.rst +5 -0
@@ -56,6 +56,11 @@ New Features Also, make sure to start the actual text at the margin. ========================================================= +* **Added the support for vfio-pci new VF token interface.** + + Since Linux version 5.7, vfio-pci supports a shared VF token (UUID) to represent + the trust between SR-IOV PF and the created VFs. Update the method to gain access + to the PF and VFs devices by appending the VF token parameter. Removed Items -------------
lib/librte_eal/common/eal_common_options.c +2 -0
@@ -92,6 +92,7 @@ eal_long_options[] = { {OPT_SYSLOG, 1, NULL, OPT_SYSLOG_NUM }, {OPT_VDEV, 1, NULL, OPT_VDEV_NUM }, {OPT_VFIO_INTR, 1, NULL, OPT_VFIO_INTR_NUM }, + {OPT_VFIO_VF_TOKEN, 1, NULL, OPT_VFIO_VF_TOKEN_NUM }, {OPT_VMWARE_TSC_MAP, 0, NULL, OPT_VMWARE_TSC_MAP_NUM }, {OPT_LEGACY_MEM, 0, NULL, OPT_LEGACY_MEM_NUM }, {OPT_SINGLE_FILE_SEGMENTS, 0, NULL, OPT_SINGLE_FILE_SEGMENTS_NUM},
@@ -297,6 +298,7 @@ eal_reset_internal_config(struct internal_config *internal_cfg)/* if set to NONE, interrupt mode is determined automatically */ internal_cfg->vfio_intr_mode =RTE_INTR_MODE_NONE ; +memset (internal_cfg->vfio_vf_token,0 ,sizeof (rte_uuid_t));#ifdef RTE_LIBEAL_USE_HPET internal_cfg->no_hpet =0 ;
lib/librte_eal/common/eal_internal_cfg.h +2 -0
@@ -72,6 +72,8 @@ struct internal_config {volatile int syslog_facility;/**< facility passed to openlog() */ /** default interrupt mode for VFIO */ volatile enum rte_intr_mode vfio_intr_mode; +/** the shared VF token for VFIO-PCI bound PF and VFs devices */ +rte_uuid_t vfio_vf_token;char *hugefile_prefix;/**< the base filename of hugetlbfs files */ char *hugepage_dir;/**< specific hugetlbfs directory to use */ char *user_mbuf_pool_ops_name;
lib/librte_eal/common/eal_options.h +2 -0
@@ -69,6 +69,8 @@ enum {OPT_VDEV_NUM , #define OPT_VFIO_INTR "vfio-intr"OPT_VFIO_INTR_NUM , +#define OPT_VFIO_VF_TOKEN "vfio-vf-token" +OPT_VFIO_VF_TOKEN_NUM , #define OPT_VMWARE_TSC_MAP "vmware-tsc-map"OPT_VMWARE_TSC_MAP_NUM , #define OPT_LEGACY_MEM "legacy-mem"
lib/librte_eal/freebsd/eal.c +4 -0
@@ -1005,6 +1005,10 @@ rte_eal_vfio_intr_mode(void)return RTE_INTR_MODE_NONE ; } +void rte_eal_vfio_get_vf_token(__rte_unused rte_uuid_t vf_token) +{ +} + int rte_vfio_setup_device(__rte_unused const char *sysfs_base, __rte_unused const char *dev_addr, __rte_unused int *vfio_dev_fd,
lib/librte_eal/include/rte_eal.h +12 -0
@@ -21,6 +21,7 @@#include <rte_bus.h> #include <rte_pci_dev_feature_defs.h> +#include <rte_uuid.h> #ifdef __cplusplusextern "C" {
@@ -438,6 +439,17 @@ int rte_eal_create_uio_dev(void);*/ enum rte_intr_mode rte_eal_vfio_intr_mode (void ); + +/** +* Copy the user-configured vfio VF token. +* +* @param vf_token +* vfio VF token configured with the command line is copied +* into this parameter, zero uuid by default. +*/ +__rte_experimental +void rte_eal_vfio_get_vf_token(rte_uuid_t vf_token); +/** * A wrap API for syscall gettid. *
lib/librte_eal/linux/eal.c +29 -0
@@ -558,6 +558,7 @@ eal_usage(const char *prgname)" --" OPT_FILE_PREFIX " Prefix for hugepage filenames\n" " --" OPT_CREATE_UIO_DEV " Create /dev/uioX (usually done by hotplug)\n" " --" OPT_VFIO_INTR " Interrupt mode for VFIO (legacy|msi|msix)\n" +" --" OPT_VFIO_VF_TOKEN " VF token (UUID) shared between SR-IOV PF and VFs\n" " --" OPT_LEGACY_MEM " Legacy memory mode (no dynamic allocation, contiguous segments)\n" " --" OPT_SINGLE_FILE_SEGMENTS " Put all hugepage memory in single files\n" " --" OPT_MATCH_ALLOCATIONS " Free hugepages exactly as allocated\n"
@@ -649,6 +650,19 @@ eal_parse_vfio_intr(const char *mode)return -1 ; } +static int +eal_parse_vfio_vf_token (const char *vf_token) +{ +rte_uuid_t uuid; + +if (!rte_uuid_parse (vf_token, uuid)) { +rte_uuid_copy (internal_config.vfio_vf_token, uuid); +return 0 ; + } + +return -1 ; +} +/* Parse the arguments for --log-level only */ static void eal_log_level_parse (int argc,char **argv)
@@ -787,6 +801,16 @@ eal_parse_args(int argc, char **argv) }break ; +case OPT_VFIO_VF_TOKEN_NUM : +if (eal_parse_vfio_vf_token (optarg) <0 ) { +RTE_LOG (ERR ,EAL ,"invalid parameters for --" +OPT_VFIO_VF_TOKEN "\n" ); +eal_usage (prgname); + ret =-1 ; + goto out; + } +break ; +case OPT_CREATE_UIO_DEV_NUM : internal_config.create_uio_dev =1 ;break ;
@@ -1370,6 +1394,11 @@ rte_eal_vfio_intr_mode(void)return internal_config.vfio_intr_mode; } +void rte_eal_vfio_get_vf_token (rte_uuid_t vf_token) +{ +rte_uuid_copy (vf_token, internal_config.vfio_vf_token); +} +int rte_eal_check_module (const char *module_name) {
lib/librte_eal/linux/eal_vfio.c +19 -0
@@ -712,6 +712,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,int vfio_container_fd;int vfio_group_fd;int iommu_group_num; +rte_uuid_t vf_token;int i, ret;/* get group number */
@@ -895,6 +896,23 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr, t->type_id, t->name); } +rte_eal_vfio_get_vf_token (vf_token); + +/* get a file descriptor for the device with VF token firstly */ +if (!rte_uuid_is_null (vf_token)) { +char vf_token_str[RTE_UUID_STRLEN ]; +char dev[PATH_MAX ]; + +rte_uuid_unparse (vf_token, vf_token_str,sizeof (vf_token_str)); +snprintf (dev,sizeof (dev), +"%s vf_token=%s" , dev_addr, vf_token_str); + + *vfio_dev_fd =ioctl (vfio_group_fd,VFIO_GROUP_GET_DEVICE_FD , + dev); +if (*vfio_dev_fd >=0 ) + goto dev_get_info; + } +/* get a file descriptor for the device */ *vfio_dev_fd =ioctl (vfio_group_fd,VFIO_GROUP_GET_DEVICE_FD , dev_addr);if (*vfio_dev_fd <0 ) {
@@ -909,6 +927,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,return -1 ; } +dev_get_info:/* test and setup the device */ ret =ioctl (*vfio_dev_fd,VFIO_DEVICE_GET_INFO , device_info);if (ret) {
lib/librte_eal/rte_eal_version.map +1 -0
@@ -372,6 +372,7 @@ EXPERIMENTAL { __rte_trace_point_register; per_lcore_trace_mem; per_lcore_trace_point_sz; + rte_eal_vfio_get_vf_token; rte_log_can_log; rte_thread_getname; rte_trace_dump;
Signature · 1 line
2.26.2
Re: [dpdk-dev] [PATCH v14 2/2] eal: support for VFIO-PCI VF token
Quoted 10 lines · Haiyue Wang
The kernel module vfio-pci introduces the VF token to enable SR-IOV support since 5.7.
The VF token can be set by a vfio-pci based PF driver and must be known by the vfio-pci based VF driver in order to gain access to the device.
Signed-off-by Haiyue Wang <[email protected]>Acked-by Anatoly Burakov <[email protected]>Acked-by Andrew Rybchenko <[email protected]>---
[snip]
lib/librte_eal/include/rte_eal.h int rte_eal_create_uio_dev(void); · line 452
Quoted 11 lines
lib/librte_eal/include/rte_eal.h +11
-0+ +/** +* Copy the user-configured vfio VF token. +* +* @param vf_token +* vfio VF token configured with the command line is copied +* into this parameter, zero uuid by default. +*/ +__rte_experimental +void rte_eal_vfio_get_vf_token(rte_uuid_t vf_token); +
Minor nit, missing the EXPERIMENTAL warning int the comment.
* @warning
* @b EXPERIMENTAL: this API may change without prior notice *
Re: [dpdk-dev] [PATCH v14 2/2] eal: support for VFIO-PCI VF token
Quoted 22 lines
-----Original Message-----
From: Ye, Xiaolong <[email protected]>
Sent: Friday, May 29, 2020 09:26
To: Wang, Haiyue <[email protected]>
Cc: [email protected]; Burakov, Anatoly <[email protected]>; [email protected]; [email protected]; [email protected]; [email protected]
Subject: Re: [dpdk-dev] [PATCH v14 2/2] eal: support for VFIO-PCI VF tokenQuoted 10 lines · Haiyue Wang
The kernel module vfio-pci introduces the VF token to enable SR-IOV support since 5.7.
The VF token can be set by a vfio-pci based PF driver and must be known by the vfio-pci based VF driver in order to gain access to the device.
Signed-off-by Haiyue Wang <[email protected]>Acked-by Anatoly Burakov <[email protected]>Acked-by Andrew Rybchenko <[email protected]>---
[snip]
Quoted 14 lines
lib/librte_eal/include/rte_eal.h int rte_eal_create_uio_dev(void); · line 452
Quoted 11 lines
lib/librte_eal/include/rte_eal.h +11
-0+ +/** +* Copy the user-configured vfio VF token. +* +* @param vf_token +* vfio VF token configured with the command line is copied +* into this parameter, zero uuid by default. +*/ +__rte_experimental +void rte_eal_vfio_get_vf_token(rte_uuid_t vf_token); +Minor nit, missing the EXPERIMENTAL warning int the comment.
Yes, missed, it is good for kindly reminding. ;-)
Quoted 3 lines
* @warning
* @b EXPERIMENTAL: this API may change without prior notice *
v15 cover support for VFIO-PCI VF token interface
In reply to [email protected]v15: Add the missed EXPERIMENTAL warning for API doxgen.
v14: Rebase the patch for 20.08 release note.
v13: Rename the EAL get VF token function, and leave the freebsd type as empty.
v12: support to vfio devices with VF token and no token.
v11: Use the eal parameter to pass the VF token, then not every PCI
device needs to be specified with this token. Also no ABI issue now.
v10: Use the __rte_internal to mark the internal API changing.
v9: Rewrite the document.
v8: Update the document.
v7: Add the Fixes tag in uuid, the release note and help
document.
v6: Drop the Fixes tag in uuid, since the file has been
moved to another place, not suitable to apply on stable. And this is not a bug, just some kind of enhancement.
v5: 1. Add the VF token parse error handling.
2. Split into two patches for different logic module.
3. Add more comments into the code for explaining the design.
4. Drop the ABI change workaround, this patch set focuses on code review.
v4: 1. Ignore rte_vfio_setup_device ABI check since it is
for Linux driver use.
v3: Fix the Travis build failed:
(1). rte_uuid.h:97:55: error: unknown type name ‘size_t’ (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’
v2: Fix the FreeBSD build error.
v1: Update the commit message.
RFC v2:
Based on Vamsi's RFC v1, and Alex's patch for Qemu
[https://lore.kernel.org/lkml/[email protected]/]:
Use the devarg to pass-down the VF token.
RFC v1: https://patchwork.dpdk.org/patch/66281/ by Vamsi.
Haiyue Wang (2):
eal: add uuid dependent header files explicitly
eal: support for VFIO-PCI VF token
doc/guides/linux_gsg/linux_drivers.rst | 35 ++++++++++++++++++- doc/guides/linux_gsg/linux_eal_parameters.rst | 4 +++ doc/guides/rel_notes/release_20_08.rst | 5 +++ lib/librte_eal/common/eal_common_options.c | 2 ++ lib/librte_eal/common/eal_internal_cfg.h | 2 ++ lib/librte_eal/common/eal_options.h | 2 ++ lib/librte_eal/freebsd/eal.c | 4 +++ lib/librte_eal/include/rte_eal.h | 15 ++++++++ lib/librte_eal/include/rte_uuid.h | 2 ++ lib/librte_eal/linux/eal.c | 29 +++++++++++++++ lib/librte_eal/linux/eal_vfio.c | 19 ++++++++++ lib/librte_eal/rte_eal_version.map | 1 + 12 files changed, 119 insertions(+), 1 deletion(-)
Signature · 1 line
2.26.2
PATCH v15 1/2 eal: add uuid dependent header files explicitly
Add the dependent header files explicitly, so that the user just needs to include the 'rte_uuid.h' header file directly to avoid compile error:
(1). rte_uuid.h:97:55: error: unknown type name ‘size_t’ (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’
--- lib/librte_eal/include/rte_uuid.h | 2 ++ 1 file changed, 2 insertions(+)
lib/librte_eal/include/rte_uuid.h +2 -0
@@ -15,6 +15,8 @@ extern "C" {#endif #include <stdbool.h> +#include <stddef.h> +#include <string.h> /** * Struct describing a Universal Unique Identifier
Signature · 1 line
2.26.2
PATCH v15 2/2 eal: support for VFIO-PCI VF token
In reply to Haiyue WangThe kernel module vfio-pci introduces the VF token to enable SR-IOV support since 5.7.
The VF token can be set by a vfio-pci based PF driver and must be known by the vfio-pci based VF driver in order to gain access to the device.
--- doc/guides/linux_gsg/linux_drivers.rst | 35 ++++++++++++++++++- doc/guides/linux_gsg/linux_eal_parameters.rst | 4 +++ doc/guides/rel_notes/release_20_08.rst | 5 +++ lib/librte_eal/common/eal_common_options.c | 2 ++ lib/librte_eal/common/eal_internal_cfg.h | 2 ++ lib/librte_eal/common/eal_options.h | 2 ++ lib/librte_eal/freebsd/eal.c | 4 +++ lib/librte_eal/include/rte_eal.h | 15 ++++++++ lib/librte_eal/linux/eal.c | 29 +++++++++++++++ lib/librte_eal/linux/eal_vfio.c | 19 ++++++++++ lib/librte_eal/rte_eal_version.map | 1 + 11 files changed, 117 insertions(+), 1 deletion(-)
doc/guides/linux_gsg/linux_drivers.rst +34 -1
@@ -78,11 +78,44 @@ Note that in order to use VFIO, your kernel must support it. VFIO kernel modules have been included in the Linux kernel since version 3.6.0 and are usually present by default, however please consult your distributions documentation to make sure that is the case. +The ``vfio-pci`` module since Linux version 5.7 supports the creation of virtual +functions. After the PF is bound to vfio-pci module, the user can create the VFs +by sysfs interface, and these VFs are bound to vfio-pci module automatically. + +When the PF is bound to vfio-pci, it has initial VF token generated by random. For +security reason, this token is write only, the user can't read it from the kernel +directly. For accessing the VF, the user needs to start the PF with token parameter +to setup a VF token (uuid format), then the VF can be accessed with this new known +VF token. + +DPDK will use the EAL parameter ``--vfio-vf-token`` to specify the VF token value to +PF and its related VFs, this VF token will be shared in all VFIO devices, including +the different PFs. + +.. code-block:: console + + 1. Generate the VF token by uuid command + 14d63f20-8445-11ea-8900-1f9ce7d5650d + + 2. sudo modprobe vfio-pci enable_sriov=1 + + 2. ./usertools/dpdk-devbind.py -b vfio-pci 0000:86:00.0 + + 3. echo 2 > /sys/bus/pci/devices/0000:86:00.0/sriov_numvfs + + 4. Start the PF: + ./x86_64-native-linux-gcc/app/testpmd -l 22-25 -n 4 -w 86:00.0 \ + --vfio-vf-token=14d63f20-8445-11ea-8900-1f9ce7d5650d --file-prefix=pf -- -i + + 5. Start the VF: + ./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 -w 86:02.0 \ + --vfio-vf-token=14d63f20-8445-11ea-8900-1f9ce7d5650d --file-prefix=vf0 -- -i + Also, to use VFIO, both kernel and BIOS must support and be configured to use IO virtualization (such as Intel® VT-d). .. note:: - ``vfio-pci`` module doesn't support the creation of virtual functions. + ``vfio-pci`` module doesn't support the creation of virtual functions before Linux version 5.7. For proper operation of VFIO when running DPDK applications as a non-privileged user, correct permissions should also be set up. This can be done by using the DPDK setup script (called dpdk-setup.sh and located in the usertools directory).
doc/guides/linux_gsg/linux_eal_parameters.rst +4 -0
@@ -40,6 +40,10 @@ Device-related options Use specified interrupt mode for devices bound to VFIO kernel driver. +* ``--vfio-vf-token <uuid>`` + + Use specified VF token for devices bound to VFIO kernel driver. + Multiprocessing-related options ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
doc/guides/rel_notes/release_20_08.rst +5 -0
@@ -56,6 +56,11 @@ New Features Also, make sure to start the actual text at the margin. ========================================================= +* **Added the support for vfio-pci new VF token interface.** + + Since Linux version 5.7, vfio-pci supports a shared VF token (UUID) to represent + the trust between SR-IOV PF and the created VFs. Update the method to gain access + to the PF and VFs devices by appending the VF token parameter. Removed Items -------------
lib/librte_eal/common/eal_common_options.c +2 -0
@@ -92,6 +92,7 @@ eal_long_options[] = { {OPT_SYSLOG, 1, NULL, OPT_SYSLOG_NUM }, {OPT_VDEV, 1, NULL, OPT_VDEV_NUM }, {OPT_VFIO_INTR, 1, NULL, OPT_VFIO_INTR_NUM }, + {OPT_VFIO_VF_TOKEN, 1, NULL, OPT_VFIO_VF_TOKEN_NUM }, {OPT_VMWARE_TSC_MAP, 0, NULL, OPT_VMWARE_TSC_MAP_NUM }, {OPT_LEGACY_MEM, 0, NULL, OPT_LEGACY_MEM_NUM }, {OPT_SINGLE_FILE_SEGMENTS, 0, NULL, OPT_SINGLE_FILE_SEGMENTS_NUM},
@@ -297,6 +298,7 @@ eal_reset_internal_config(struct internal_config *internal_cfg)/* if set to NONE, interrupt mode is determined automatically */ internal_cfg->vfio_intr_mode =RTE_INTR_MODE_NONE ; +memset (internal_cfg->vfio_vf_token,0 ,sizeof (rte_uuid_t));#ifdef RTE_LIBEAL_USE_HPET internal_cfg->no_hpet =0 ;
lib/librte_eal/common/eal_internal_cfg.h +2 -0
@@ -72,6 +72,8 @@ struct internal_config {volatile int syslog_facility;/**< facility passed to openlog() */ /** default interrupt mode for VFIO */ volatile enum rte_intr_mode vfio_intr_mode; +/** the shared VF token for VFIO-PCI bound PF and VFs devices */ +rte_uuid_t vfio_vf_token;char *hugefile_prefix;/**< the base filename of hugetlbfs files */ char *hugepage_dir;/**< specific hugetlbfs directory to use */ char *user_mbuf_pool_ops_name;
lib/librte_eal/common/eal_options.h +2 -0
@@ -69,6 +69,8 @@ enum {OPT_VDEV_NUM , #define OPT_VFIO_INTR "vfio-intr"OPT_VFIO_INTR_NUM , +#define OPT_VFIO_VF_TOKEN "vfio-vf-token" +OPT_VFIO_VF_TOKEN_NUM , #define OPT_VMWARE_TSC_MAP "vmware-tsc-map"OPT_VMWARE_TSC_MAP_NUM , #define OPT_LEGACY_MEM "legacy-mem"
lib/librte_eal/freebsd/eal.c +4 -0
@@ -1005,6 +1005,10 @@ rte_eal_vfio_intr_mode(void)return RTE_INTR_MODE_NONE ; } +void rte_eal_vfio_get_vf_token(__rte_unused rte_uuid_t vf_token) +{ +} + int rte_vfio_setup_device(__rte_unused const char *sysfs_base, __rte_unused const char *dev_addr, __rte_unused int *vfio_dev_fd,
lib/librte_eal/include/rte_eal.h +15 -0
@@ -21,6 +21,7 @@#include <rte_bus.h> #include <rte_pci_dev_feature_defs.h> +#include <rte_uuid.h> #ifdef __cplusplusextern "C" {
@@ -438,6 +439,20 @@ int rte_eal_create_uio_dev(void);*/ enum rte_intr_mode rte_eal_vfio_intr_mode (void ); + +/** +* @warning +* @b EXPERIMENTAL: this API may change without prior notice +* +* Copy the user-configured vfio VF token. +* +* @param vf_token +* vfio VF token configured with the command line is copied +* into this parameter, zero uuid by default. +*/ +__rte_experimental +void rte_eal_vfio_get_vf_token(rte_uuid_t vf_token); +/** * A wrap API for syscall gettid. *
lib/librte_eal/linux/eal.c +29 -0
@@ -558,6 +558,7 @@ eal_usage(const char *prgname)" --" OPT_FILE_PREFIX " Prefix for hugepage filenames\n" " --" OPT_CREATE_UIO_DEV " Create /dev/uioX (usually done by hotplug)\n" " --" OPT_VFIO_INTR " Interrupt mode for VFIO (legacy|msi|msix)\n" +" --" OPT_VFIO_VF_TOKEN " VF token (UUID) shared between SR-IOV PF and VFs\n" " --" OPT_LEGACY_MEM " Legacy memory mode (no dynamic allocation, contiguous segments)\n" " --" OPT_SINGLE_FILE_SEGMENTS " Put all hugepage memory in single files\n" " --" OPT_MATCH_ALLOCATIONS " Free hugepages exactly as allocated\n"
@@ -649,6 +650,19 @@ eal_parse_vfio_intr(const char *mode)return -1 ; } +static int +eal_parse_vfio_vf_token (const char *vf_token) +{ +rte_uuid_t uuid; + +if (!rte_uuid_parse (vf_token, uuid)) { +rte_uuid_copy (internal_config.vfio_vf_token, uuid); +return 0 ; + } + +return -1 ; +} +/* Parse the arguments for --log-level only */ static void eal_log_level_parse (int argc,char **argv)
@@ -787,6 +801,16 @@ eal_parse_args(int argc, char **argv) }break ; +case OPT_VFIO_VF_TOKEN_NUM : +if (eal_parse_vfio_vf_token (optarg) <0 ) { +RTE_LOG (ERR ,EAL ,"invalid parameters for --" +OPT_VFIO_VF_TOKEN "\n" ); +eal_usage (prgname); + ret =-1 ; + goto out; + } +break ; +case OPT_CREATE_UIO_DEV_NUM : internal_config.create_uio_dev =1 ;break ;
@@ -1370,6 +1394,11 @@ rte_eal_vfio_intr_mode(void)return internal_config.vfio_intr_mode; } +void rte_eal_vfio_get_vf_token (rte_uuid_t vf_token) +{ +rte_uuid_copy (vf_token, internal_config.vfio_vf_token); +} +int rte_eal_check_module (const char *module_name) {
lib/librte_eal/linux/eal_vfio.c +19 -0
@@ -712,6 +712,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,int vfio_container_fd;int vfio_group_fd;int iommu_group_num; +rte_uuid_t vf_token;int i, ret;/* get group number */
@@ -895,6 +896,23 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr, t->type_id, t->name); } +rte_eal_vfio_get_vf_token (vf_token); + +/* get a file descriptor for the device with VF token firstly */ +if (!rte_uuid_is_null (vf_token)) { +char vf_token_str[RTE_UUID_STRLEN ]; +char dev[PATH_MAX ]; + +rte_uuid_unparse (vf_token, vf_token_str,sizeof (vf_token_str)); +snprintf (dev,sizeof (dev), +"%s vf_token=%s" , dev_addr, vf_token_str); + + *vfio_dev_fd =ioctl (vfio_group_fd,VFIO_GROUP_GET_DEVICE_FD , + dev); +if (*vfio_dev_fd >=0 ) + goto dev_get_info; + } +/* get a file descriptor for the device */ *vfio_dev_fd =ioctl (vfio_group_fd,VFIO_GROUP_GET_DEVICE_FD , dev_addr);if (*vfio_dev_fd <0 ) {
@@ -909,6 +927,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,return -1 ; } +dev_get_info:/* test and setup the device */ ret =ioctl (*vfio_dev_fd,VFIO_DEVICE_GET_INFO , device_info);if (ret) {
lib/librte_eal/rte_eal_version.map +1 -0
@@ -372,6 +372,7 @@ EXPERIMENTAL { __rte_trace_point_register; per_lcore_trace_mem; per_lcore_trace_point_sz; + rte_eal_vfio_get_vf_token; rte_log_can_log; rte_thread_getname; rte_trace_dump;
Signature · 1 line
2.26.2
v16 cover support for VFIO-PCI VF token interface
In reply to [email protected]v16: Rebase the patch for 20.08 release note.
v15: Add the missed EXPERIMENTAL warning for API doxgen.
v14: Rebase the patch for 20.08 release note.
v13: Rename the EAL get VF token function, and leave the freebsd type as empty.
v12: support to vfio devices with VF token and no token.
v11: Use the eal parameter to pass the VF token, then not every PCI
device needs to be specified with this token. Also no ABI issue now.
v10: Use the __rte_internal to mark the internal API changing.
v9: Rewrite the document.
v8: Update the document.
v7: Add the Fixes tag in uuid, the release note and help
document.
v6: Drop the Fixes tag in uuid, since the file has been
moved to another place, not suitable to apply on stable. And this is not a bug, just some kind of enhancement.
v5: 1. Add the VF token parse error handling.
2. Split into two patches for different logic module.
3. Add more comments into the code for explaining the design.
4. Drop the ABI change workaround, this patch set focuses on code review.
v4: 1. Ignore rte_vfio_setup_device ABI check since it is
for Linux driver use.
v3: Fix the Travis build failed:
(1). rte_uuid.h:97:55: error: unknown type name ‘size_t’ (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’
v2: Fix the FreeBSD build error.
v1: Update the commit message.
RFC v2:
Based on Vamsi's RFC v1, and Alex's patch for Qemu
[https://lore.kernel.org/lkml/[email protected]/]:
Use the devarg to pass-down the VF token.
RFC v1: https://patchwork.dpdk.org/patch/66281/ by Vamsi.
Haiyue Wang (2):
eal: add uuid dependent header files explicitly
eal: support for VFIO-PCI VF token
doc/guides/linux_gsg/linux_drivers.rst | 35 ++++++++++++++++++- doc/guides/linux_gsg/linux_eal_parameters.rst | 4 +++ doc/guides/rel_notes/release_20_08.rst | 5 +++ lib/librte_eal/common/eal_common_options.c | 2 ++ lib/librte_eal/common/eal_internal_cfg.h | 2 ++ lib/librte_eal/common/eal_options.h | 2 ++ lib/librte_eal/freebsd/eal.c | 4 +++ lib/librte_eal/include/rte_eal.h | 15 ++++++++ lib/librte_eal/include/rte_uuid.h | 2 ++ lib/librte_eal/linux/eal.c | 29 +++++++++++++++ lib/librte_eal/linux/eal_vfio.c | 19 ++++++++++ lib/librte_eal/rte_eal_version.map | 1 + 12 files changed, 119 insertions(+), 1 deletion(-)
Signature · 1 line
2.27.0
PATCH v16 1/2 eal: add uuid dependent header files explicitly
Add the dependent header files explicitly, so that the user just needs to include the 'rte_uuid.h' header file directly to avoid compile error:
(1). rte_uuid.h:97:55: error: unknown type name ‘size_t’ (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’
--- lib/librte_eal/include/rte_uuid.h | 2 ++ 1 file changed, 2 insertions(+)
lib/librte_eal/include/rte_uuid.h +2 -0
@@ -15,6 +15,8 @@ extern "C" {#endif #include <stdbool.h> +#include <stddef.h> +#include <string.h> /** * Struct describing a Universal Unique Identifier
Signature · 1 line
2.27.0
PATCH v16 2/2 eal: support for VFIO-PCI VF token
In reply to Haiyue WangThe kernel module vfio-pci introduces the VF token to enable SR-IOV support since 5.7.
The VF token can be set by a vfio-pci based PF driver and must be known by the vfio-pci based VF driver in order to gain access to the device.
--- doc/guides/linux_gsg/linux_drivers.rst | 35 ++++++++++++++++++- doc/guides/linux_gsg/linux_eal_parameters.rst | 4 +++ doc/guides/rel_notes/release_20_08.rst | 5 +++ lib/librte_eal/common/eal_common_options.c | 2 ++ lib/librte_eal/common/eal_internal_cfg.h | 2 ++ lib/librte_eal/common/eal_options.h | 2 ++ lib/librte_eal/freebsd/eal.c | 4 +++ lib/librte_eal/include/rte_eal.h | 15 ++++++++ lib/librte_eal/linux/eal.c | 29 +++++++++++++++ lib/librte_eal/linux/eal_vfio.c | 19 ++++++++++ lib/librte_eal/rte_eal_version.map | 1 + 11 files changed, 117 insertions(+), 1 deletion(-)
doc/guides/linux_gsg/linux_drivers.rst +34 -1
@@ -78,11 +78,44 @@ Note that in order to use VFIO, your kernel must support it. VFIO kernel modules have been included in the Linux kernel since version 3.6.0 and are usually present by default, however please consult your distributions documentation to make sure that is the case. +The ``vfio-pci`` module since Linux version 5.7 supports the creation of virtual +functions. After the PF is bound to vfio-pci module, the user can create the VFs +by sysfs interface, and these VFs are bound to vfio-pci module automatically. + +When the PF is bound to vfio-pci, it has initial VF token generated by random. For +security reason, this token is write only, the user can't read it from the kernel +directly. For accessing the VF, the user needs to start the PF with token parameter +to setup a VF token (uuid format), then the VF can be accessed with this new known +VF token. + +DPDK will use the EAL parameter ``--vfio-vf-token`` to specify the VF token value to +PF and its related VFs, this VF token will be shared in all VFIO devices, including +the different PFs. + +.. code-block:: console + + 1. Generate the VF token by uuid command + 14d63f20-8445-11ea-8900-1f9ce7d5650d + + 2. sudo modprobe vfio-pci enable_sriov=1 + + 2. ./usertools/dpdk-devbind.py -b vfio-pci 0000:86:00.0 + + 3. echo 2 > /sys/bus/pci/devices/0000:86:00.0/sriov_numvfs + + 4. Start the PF: + ./x86_64-native-linux-gcc/app/testpmd -l 22-25 -n 4 -w 86:00.0 \ + --vfio-vf-token=14d63f20-8445-11ea-8900-1f9ce7d5650d --file-prefix=pf -- -i + + 5. Start the VF: + ./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 -w 86:02.0 \ + --vfio-vf-token=14d63f20-8445-11ea-8900-1f9ce7d5650d --file-prefix=vf0 -- -i + Also, to use VFIO, both kernel and BIOS must support and be configured to use IO virtualization (such as Intel® VT-d). .. note:: - ``vfio-pci`` module doesn't support the creation of virtual functions. + ``vfio-pci`` module doesn't support the creation of virtual functions before Linux version 5.7. For proper operation of VFIO when running DPDK applications as a non-privileged user, correct permissions should also be set up. This can be done by using the DPDK setup script (called dpdk-setup.sh and located in the usertools directory).
doc/guides/linux_gsg/linux_eal_parameters.rst +4 -0
@@ -40,6 +40,10 @@ Device-related options Use specified interrupt mode for devices bound to VFIO kernel driver. +* ``--vfio-vf-token <uuid>`` + + Use specified VF token for devices bound to VFIO kernel driver. + Multiprocessing-related options ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
doc/guides/rel_notes/release_20_08.rst +5 -0
@@ -62,6 +62,11 @@ New Features * Added new PMD devarg ``reclaim_mem_mode``. +* **Added the support for vfio-pci new VF token interface.** + + Since Linux version 5.7, vfio-pci supports a shared VF token (UUID) to represent + the trust between SR-IOV PF and the created VFs. Update the method to gain access + to the PF and VFs devices by appending the VF token parameter. Removed Items -------------
lib/librte_eal/common/eal_common_options.c +2 -0
@@ -94,6 +94,7 @@ eal_long_options[] = { {OPT_SYSLOG, 1, NULL, OPT_SYSLOG_NUM }, {OPT_VDEV, 1, NULL, OPT_VDEV_NUM }, {OPT_VFIO_INTR, 1, NULL, OPT_VFIO_INTR_NUM }, + {OPT_VFIO_VF_TOKEN, 1, NULL, OPT_VFIO_VF_TOKEN_NUM }, {OPT_VMWARE_TSC_MAP, 0, NULL, OPT_VMWARE_TSC_MAP_NUM }, {OPT_LEGACY_MEM, 0, NULL, OPT_LEGACY_MEM_NUM }, {OPT_SINGLE_FILE_SEGMENTS, 0, NULL, OPT_SINGLE_FILE_SEGMENTS_NUM},
@@ -301,6 +302,7 @@ eal_reset_internal_config(struct internal_config *internal_cfg)/* if set to NONE, interrupt mode is determined automatically */ internal_cfg->vfio_intr_mode =RTE_INTR_MODE_NONE ; +memset (internal_cfg->vfio_vf_token,0 ,sizeof (rte_uuid_t));#ifdef RTE_LIBEAL_USE_HPET internal_cfg->no_hpet =0 ;
lib/librte_eal/common/eal_internal_cfg.h +2 -0
@@ -72,6 +72,8 @@ struct internal_config {volatile int syslog_facility;/**< facility passed to openlog() */ /** default interrupt mode for VFIO */ volatile enum rte_intr_mode vfio_intr_mode; +/** the shared VF token for VFIO-PCI bound PF and VFs devices */ +rte_uuid_t vfio_vf_token;char *hugefile_prefix;/**< the base filename of hugetlbfs files */ char *hugepage_dir;/**< specific hugetlbfs directory to use */ char *user_mbuf_pool_ops_name;
lib/librte_eal/common/eal_options.h +2 -0
@@ -69,6 +69,8 @@ enum {OPT_VDEV_NUM , #define OPT_VFIO_INTR "vfio-intr"OPT_VFIO_INTR_NUM , +#define OPT_VFIO_VF_TOKEN "vfio-vf-token" +OPT_VFIO_VF_TOKEN_NUM , #define OPT_VMWARE_TSC_MAP "vmware-tsc-map"OPT_VMWARE_TSC_MAP_NUM , #define OPT_LEGACY_MEM "legacy-mem"
lib/librte_eal/freebsd/eal.c +4 -0
@@ -1005,6 +1005,10 @@ rte_eal_vfio_intr_mode(void)return RTE_INTR_MODE_NONE ; } +void rte_eal_vfio_get_vf_token(__rte_unused rte_uuid_t vf_token) +{ +} + int rte_vfio_setup_device(__rte_unused const char *sysfs_base, __rte_unused const char *dev_addr, __rte_unused int *vfio_dev_fd,
lib/librte_eal/include/rte_eal.h +15 -0
@@ -21,6 +21,7 @@#include <rte_bus.h> #include <rte_pci_dev_feature_defs.h> +#include <rte_uuid.h> #ifdef __cplusplusextern "C" {
@@ -438,6 +439,20 @@ int rte_eal_create_uio_dev(void);*/ enum rte_intr_mode rte_eal_vfio_intr_mode (void ); + +/** +* @warning +* @b EXPERIMENTAL: this API may change without prior notice +* +* Copy the user-configured vfio VF token. +* +* @param vf_token +* vfio VF token configured with the command line is copied +* into this parameter, zero uuid by default. +*/ +__rte_experimental +void rte_eal_vfio_get_vf_token(rte_uuid_t vf_token); +/** * A wrap API for syscall gettid. *
lib/librte_eal/linux/eal.c +29 -0
@@ -558,6 +558,7 @@ eal_usage(const char *prgname)" --" OPT_FILE_PREFIX " Prefix for hugepage filenames\n" " --" OPT_CREATE_UIO_DEV " Create /dev/uioX (usually done by hotplug)\n" " --" OPT_VFIO_INTR " Interrupt mode for VFIO (legacy|msi|msix)\n" +" --" OPT_VFIO_VF_TOKEN " VF token (UUID) shared between SR-IOV PF and VFs\n" " --" OPT_LEGACY_MEM " Legacy memory mode (no dynamic allocation, contiguous segments)\n" " --" OPT_SINGLE_FILE_SEGMENTS " Put all hugepage memory in single files\n" " --" OPT_MATCH_ALLOCATIONS " Free hugepages exactly as allocated\n"
@@ -649,6 +650,19 @@ eal_parse_vfio_intr(const char *mode)return -1 ; } +static int +eal_parse_vfio_vf_token (const char *vf_token) +{ +rte_uuid_t uuid; + +if (!rte_uuid_parse (vf_token, uuid)) { +rte_uuid_copy (internal_config.vfio_vf_token, uuid); +return 0 ; + } + +return -1 ; +} +/* Parse the arguments for --log-level only */ static void eal_log_level_parse (int argc,char **argv)
@@ -787,6 +801,16 @@ eal_parse_args(int argc, char **argv) }break ; +case OPT_VFIO_VF_TOKEN_NUM : +if (eal_parse_vfio_vf_token (optarg) <0 ) { +RTE_LOG (ERR ,EAL ,"invalid parameters for --" +OPT_VFIO_VF_TOKEN "\n" ); +eal_usage (prgname); + ret =-1 ; + goto out; + } +break ; +case OPT_CREATE_UIO_DEV_NUM : internal_config.create_uio_dev =1 ;break ;
@@ -1370,6 +1394,11 @@ rte_eal_vfio_intr_mode(void)return internal_config.vfio_intr_mode; } +void rte_eal_vfio_get_vf_token (rte_uuid_t vf_token) +{ +rte_uuid_copy (vf_token, internal_config.vfio_vf_token); +} +int rte_eal_check_module (const char *module_name) {
lib/librte_eal/linux/eal_vfio.c +19 -0
@@ -712,6 +712,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,int vfio_container_fd;int vfio_group_fd;int iommu_group_num; +rte_uuid_t vf_token;int i, ret;/* get group number */
@@ -895,6 +896,23 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr, t->type_id, t->name); } +rte_eal_vfio_get_vf_token (vf_token); + +/* get a file descriptor for the device with VF token firstly */ +if (!rte_uuid_is_null (vf_token)) { +char vf_token_str[RTE_UUID_STRLEN ]; +char dev[PATH_MAX ]; + +rte_uuid_unparse (vf_token, vf_token_str,sizeof (vf_token_str)); +snprintf (dev,sizeof (dev), +"%s vf_token=%s" , dev_addr, vf_token_str); + + *vfio_dev_fd =ioctl (vfio_group_fd,VFIO_GROUP_GET_DEVICE_FD , + dev); +if (*vfio_dev_fd >=0 ) + goto dev_get_info; + } +/* get a file descriptor for the device */ *vfio_dev_fd =ioctl (vfio_group_fd,VFIO_GROUP_GET_DEVICE_FD , dev_addr);if (*vfio_dev_fd <0 ) {
@@ -909,6 +927,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,return -1 ; } +dev_get_info:/* test and setup the device */ ret =ioctl (*vfio_dev_fd,VFIO_DEVICE_GET_INFO , device_info);if (ret) {
lib/librte_eal/rte_eal_version.map +1 -0
@@ -372,6 +372,7 @@ EXPERIMENTAL { __rte_trace_point_register; per_lcore_trace_mem; per_lcore_trace_point_sz; + rte_eal_vfio_get_vf_token; rte_log_can_log; rte_thread_getname; rte_trace_dump;
Signature · 1 line
2.27.0
Re: [dpdk-dev] [PATCH v16 2/2] eal: support for VFIO-PCI VF token
Quoted 10 lines · Haiyue Wang
The kernel module vfio-pci introduces the VF token to enable SR-IOV support since 5.7.
The VF token can be set by a vfio-pci based PF driver and must be known by the vfio-pci based VF driver in order to gain access to the device.
Signed-off-by Haiyue Wang <[email protected]>Acked-by Anatoly Burakov <[email protected]>Acked-by Andrew Rybchenko <[email protected]>---
Quoted 308 lines
doc/guides/linux_gsg/linux_drivers.rst | 35 ++++++++++++++++++- doc/guides/linux_gsg/linux_eal_parameters.rst | 4 +++ doc/guides/rel_notes/release_20_08.rst | 5 +++ lib/librte_eal/common/eal_common_options.c | 2 ++ lib/librte_eal/common/eal_internal_cfg.h | 2 ++ lib/librte_eal/common/eal_options.h | 2 ++ lib/librte_eal/freebsd/eal.c | 4 +++ lib/librte_eal/include/rte_eal.h | 15 ++++++++ lib/librte_eal/linux/eal.c | 29 +++++++++++++++ lib/librte_eal/linux/eal_vfio.c | 19 ++++++++++ lib/librte_eal/rte_eal_version.map | 1 + 11 files changed, 117 insertions(+), 1 deletion(-)doc/guides/linux_gsg/linux_drivers.rst +34
-1@@ -78,11 +78,44 @@ Note that in order to use VFIO, your kernel must support it. VFIO kernel modules have been included in the Linux kernel since version 3.6.0 and are usually present by default, however please consult your distributions documentation to make sure that is the case. +The ``vfio-pci`` module since Linux version 5.7 supports the creation of virtual +functions. After the PF is bound to vfio-pci module, the user can create the VFs +by sysfs interface, and these VFs are bound to vfio-pci module automatically. + +When the PF is bound to vfio-pci, it has initial VF token generated by random. For +security reason, this token is write only, the user can't read it from the kernel +directly. For accessing the VF, the user needs to start the PF with token parameter +to setup a VF token (uuid format), then the VF can be accessed with this new known +VF token. + +DPDK will use the EAL parameter ``--vfio-vf-token`` to specify the VF token value to +PF and its related VFs, this VF token will be shared in all VFIO devices, including +the different PFs. + +.. code-block:: console + + 1. Generate the VF token by uuid command + 14d63f20-8445-11ea-8900-1f9ce7d5650d + + 2. sudo modprobe vfio-pci enable_sriov=1 + + 2. ./usertools/dpdk-devbind.py -b vfio-pci 0000:86:00.0 + + 3. echo 2 > /sys/bus/pci/devices/0000:86:00.0/sriov_numvfs + + 4. Start the PF: + ./x86_64-native-linux-gcc/app/testpmd -l 22-25 -n 4 -w 86:00.0 \ + --vfio-vf-token=14d63f20-8445-11ea-8900-1f9ce7d5650d --file-prefix=pf -- -i + + 5. Start the VF: + ./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 -w 86:02.0 \ + --vfio-vf-token=14d63f20-8445-11ea-8900-1f9ce7d5650d --file-prefix=vf0 -- -i + Also, to use VFIO, both kernel and BIOS must support and be configured to use IO virtualization (such as Intel® VT-d). .. note:: - ``vfio-pci`` module doesn't support the creation of virtual functions. + ``vfio-pci`` module doesn't support the creation of virtual functions before Linux version 5.7. For proper operation of VFIO when running DPDK applications as a non-privileged user, correct permissions should also be set up. This can be done by using the DPDK setup script (called dpdk-setup.sh and located in the usertools directory).doc/guides/linux_gsg/linux_eal_parameters.rst +4
-0@@ -40,6 +40,10 @@ Device-related options Use specified interrupt mode for devices bound to VFIO kernel driver. +* ``--vfio-vf-token <uuid>`` + + Use specified VF token for devices bound to VFIO kernel driver. + Multiprocessing-related options ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~doc/guides/rel_notes/release_20_08.rst +5
-0@@ -62,6 +62,11 @@ New Features * Added new PMD devarg ``reclaim_mem_mode``. +* **Added the support for vfio-pci new VF token interface.** + + Since Linux version 5.7, vfio-pci supports a shared VF token (UUID) to represent + the trust between SR-IOV PF and the created VFs. Update the method to gain access + to the PF and VFs devices by appending the VF token parameter. Removed Items -------------lib/librte_eal/common/eal_common_options.c +2
-0@@ -94,6 +94,7 @@ eal_long_options[] = { {OPT_SYSLOG, 1, NULL, OPT_SYSLOG_NUM }, {OPT_VDEV, 1, NULL, OPT_VDEV_NUM }, {OPT_VFIO_INTR, 1, NULL, OPT_VFIO_INTR_NUM }, + {OPT_VFIO_VF_TOKEN, 1, NULL, OPT_VFIO_VF_TOKEN_NUM }, {OPT_VMWARE_TSC_MAP, 0, NULL, OPT_VMWARE_TSC_MAP_NUM }, {OPT_LEGACY_MEM, 0, NULL, OPT_LEGACY_MEM_NUM }, {OPT_SINGLE_FILE_SEGMENTS, 0, NULL, OPT_SINGLE_FILE_SEGMENTS_NUM},@@ -301,6 +302,7 @@ eal_reset_internal_config(struct internal_config *internal_cfg)/* if set to NONE, interrupt mode is determined automatically */ internal_cfg->vfio_intr_mode =RTE_INTR_MODE_NONE ; +memset (internal_cfg->vfio_vf_token,0 ,sizeof (rte_uuid_t));#ifdef RTE_LIBEAL_USE_HPET internal_cfg->no_hpet =0 ;lib/librte_eal/common/eal_internal_cfg.h +2
-0@@ -72,6 +72,8 @@ struct internal_config {volatile int syslog_facility;/**< facility passed to openlog() */ /** default interrupt mode for VFIO */ volatile enum rte_intr_mode vfio_intr_mode; +/** the shared VF token for VFIO-PCI bound PF and VFs devices */ +rte_uuid_t vfio_vf_token;char *hugefile_prefix;/**< the base filename of hugetlbfs files */ char *hugepage_dir;/**< specific hugetlbfs directory to use */ char *user_mbuf_pool_ops_name;lib/librte_eal/common/eal_options.h +2
-0@@ -69,6 +69,8 @@ enum {OPT_VDEV_NUM , #define OPT_VFIO_INTR "vfio-intr"OPT_VFIO_INTR_NUM , +#define OPT_VFIO_VF_TOKEN "vfio-vf-token" +OPT_VFIO_VF_TOKEN_NUM , #define OPT_VMWARE_TSC_MAP "vmware-tsc-map"OPT_VMWARE_TSC_MAP_NUM , #define OPT_LEGACY_MEM "legacy-mem"lib/librte_eal/freebsd/eal.c +4
-0@@ -1005,6 +1005,10 @@ rte_eal_vfio_intr_mode(void)return RTE_INTR_MODE_NONE ; } +void rte_eal_vfio_get_vf_token(__rte_unused rte_uuid_t vf_token) +{ +} + int rte_vfio_setup_device(__rte_unused const char *sysfs_base, __rte_unused const char *dev_addr, __rte_unused int *vfio_dev_fd,lib/librte_eal/include/rte_eal.h +15
-0@@ -21,6 +21,7 @@#include <rte_bus.h> #include <rte_pci_dev_feature_defs.h> +#include <rte_uuid.h> #ifdef __cplusplusextern "C" {@@ -438,6 +439,20 @@ int rte_eal_create_uio_dev(void);*/ enum rte_intr_mode rte_eal_vfio_intr_mode (void ); + +/** +* @warning +* @b EXPERIMENTAL: this API may change without prior notice +* +* Copy the user-configured vfio VF token. +* +* @param vf_token +* vfio VF token configured with the command line is copied +* into this parameter, zero uuid by default. +*/ +__rte_experimental +void rte_eal_vfio_get_vf_token(rte_uuid_t vf_token); +/** * A wrap API for syscall gettid. * lib/librte_eal/linux/eal.c +29
-0@@ -558,6 +558,7 @@ eal_usage(const char *prgname)" --" OPT_FILE_PREFIX " Prefix for hugepage filenames\n" " --" OPT_CREATE_UIO_DEV " Create /dev/uioX (usually done by hotplug)\n" " --" OPT_VFIO_INTR " Interrupt mode for VFIO (legacy|msi|msix)\n" +" --" OPT_VFIO_VF_TOKEN " VF token (UUID) shared between SR-IOV PF and VFs\n" " --" OPT_LEGACY_MEM " Legacy memory mode (no dynamic allocation, contiguous segments)\n" " --" OPT_SINGLE_FILE_SEGMENTS " Put all hugepage memory in single files\n" " --" OPT_MATCH_ALLOCATIONS " Free hugepages exactly as allocated\n" @@ -649,6 +650,19 @@ eal_parse_vfio_intr(const char *mode)return -1 ; } +static int +eal_parse_vfio_vf_token (const char *vf_token) +{ +rte_uuid_t uuid; + +if (!rte_uuid_parse (vf_token, uuid)) { +rte_uuid_copy (internal_config.vfio_vf_token, uuid); +return 0 ; + } + +return -1 ; +} +/* Parse the arguments for --log-level only */ static void eal_log_level_parse (int argc,char **argv)@@ -787,6 +801,16 @@ eal_parse_args(int argc, char **argv) }break ; +case OPT_VFIO_VF_TOKEN_NUM : +if (eal_parse_vfio_vf_token (optarg) <0 ) { +RTE_LOG (ERR ,EAL ,"invalid parameters for --" +OPT_VFIO_VF_TOKEN "\n" ); +eal_usage (prgname); + ret =-1 ; + goto out; + } +break ; +case OPT_CREATE_UIO_DEV_NUM : internal_config.create_uio_dev =1 ;break ;@@ -1370,6 +1394,11 @@ rte_eal_vfio_intr_mode(void)return internal_config.vfio_intr_mode; } +void rte_eal_vfio_get_vf_token (rte_uuid_t vf_token) +{ +rte_uuid_copy (vf_token, internal_config.vfio_vf_token); +} +int rte_eal_check_module (const char *module_name) {lib/librte_eal/linux/eal_vfio.c +19
-0@@ -712,6 +712,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,int vfio_container_fd;int vfio_group_fd;int iommu_group_num; +rte_uuid_t vf_token;int i, ret;/* get group number */ @@ -895,6 +896,23 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr, t->type_id, t->name); } +rte_eal_vfio_get_vf_token (vf_token); + +/* get a file descriptor for the device with VF token firstly */ +if (!rte_uuid_is_null (vf_token)) { +char vf_token_str[RTE_UUID_STRLEN ]; +char dev[PATH_MAX ]; + +rte_uuid_unparse (vf_token, vf_token_str,sizeof (vf_token_str)); +snprintf (dev,sizeof (dev), +"%s vf_token=%s" , dev_addr, vf_token_str); + + *vfio_dev_fd =ioctl (vfio_group_fd,VFIO_GROUP_GET_DEVICE_FD , + dev); +if (*vfio_dev_fd >=0 ) + goto dev_get_info; + } +/* get a file descriptor for the device */ *vfio_dev_fd =ioctl (vfio_group_fd,VFIO_GROUP_GET_DEVICE_FD , dev_addr);if (*vfio_dev_fd <0 ) {@@ -909,6 +927,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,return -1 ; } +dev_get_info:/* test and setup the device */ ret =ioctl (*vfio_dev_fd,VFIO_DEVICE_GET_INFO , device_info);if (ret) {lib/librte_eal/rte_eal_version.map +1
-0@@ -372,6 +372,7 @@ EXPERIMENTAL { __rte_trace_point_register; per_lcore_trace_mem; per_lcore_trace_point_sz; + rte_eal_vfio_get_vf_token; rte_log_can_log; rte_thread_getname; rte_trace_dump;Signature · 1 line
2.27.0
Re: [dpdk-dev] [PATCH v16 2/2] eal: support for VFIO-PCI VF token
Quoted 8 lines · Harman Kalra
Quoted 6 lines · Haiyue Wang
The kernel module vfio-pci introduces the VF token to enable SR-IOV support since 5.7.
The VF token can be set by a vfio-pci based PF driver and must be known by the vfio-pci based VF driver in order to gain access to the device.
Quoted 6 lines
Quoted 4 lines
Signed-off-by Haiyue Wang <[email protected]>Acked-by Anatoly Burakov <[email protected]>Acked-by Andrew Rybchenko <[email protected]>---
Tested-by Harman Kalra <[email protected]>
Thanks for the test Harman.
I can see no complaint on using a single token for all devices, which is the only concern I would have with the last revision. If everyone is ok with this choice, I will take this for -rc1.
Signature · 1 line
David Marchand
Re: [dpdk-dev] [PATCH v16 2/2] eal: support for VFIO-PCI VF token
Quoted 23 lines
-----Original Message-----
From: David Marchand <[email protected]>
Sent: Thursday, June 25, 2020 15:33
To: Harman Kalra <[email protected]>; Jerin Jacob Kollanukkaran <[email protected]>
Cc: Wang, Haiyue <[email protected]>; dev <[email protected]>; Burakov, Anatoly <[email protected]>; Thomas Monjalon <[email protected]>; Andrew Rybchenko <[email protected]>; Maxime Coquelin <[email protected]>
Subject: Re: [dpdk-dev] [PATCH v16 2/2] eal: support for VFIO-PCI VF tokenQuoted 13 lines · Harman Kalra
Quoted 10 lines · Haiyue Wang
The kernel module vfio-pci introduces the VF token to enable SR-IOV support since 5.7.
The VF token can be set by a vfio-pci based PF driver and must be known by the vfio-pci based VF driver in order to gain access to the device.
Signed-off-by Haiyue Wang <[email protected]>Acked-by Anatoly Burakov <[email protected]>Acked-by Andrew Rybchenko <[email protected]>---
Quoted 5 lines
Quoted 1 line
Tested-by Harman Kalra <[email protected]>Thanks for the test Harman.
I can see no complaint on using a single token for all devices, which
Yeah, not the best, but may meet with the most needed: create VFs by vfio-pci ;-)
Since devarg was a private option for ALL kind of devices, and have to break the ABI policy to implement the design, so I drop this revision, choose to use it as a global option as "vfio-intr" option does.
Quoted 6 lines
is the only concern I would have with the last revision. If everyone is ok with this choice, I will take this for -rc1.
--
David Marchand
v17 cover support for VFIO-PCI VF token interface
In reply to [email protected]v17: Rebase for new EAL config API, update the commit message and doc.
v16: Rebase the patch for 20.08 release note.
v15: Add the missed EXPERIMENTAL warning for API doxgen.
v14: Rebase the patch for 20.08 release note.
v13: Rename the EAL get VF token function, and leave the freebsd type as empty.
v12: support to vfio devices with VF token and no token.
v11: Use the eal parameter to pass the VF token, then not every PCI
device needs to be specified with this token. Also no ABI issue now.
v10: Use the __rte_internal to mark the internal API changing.
v9: Rewrite the document.
v8: Update the document.
v7: Add the Fixes tag in uuid, the release note and help
document.
v6: Drop the Fixes tag in uuid, since the file has been
moved to another place, not suitable to apply on stable. And this is not a bug, just some kind of enhancement.
v5: 1. Add the VF token parse error handling.
2. Split into two patches for different logic module.
3. Add more comments into the code for explaining the design.
4. Drop the ABI change workaround, this patch set focuses on code review.
v4: 1. Ignore rte_vfio_setup_device ABI check since it is
for Linux driver use.
v3: Fix the Travis build failed:
(1). rte_uuid.h:97:55: error: unknown type name ‘size_t’ (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’
v2: Fix the FreeBSD build error.
v1: Update the commit message.
RFC v2:
Based on Vamsi's RFC v1, and Alex's patch for Qemu
[https://lore.kernel.org/lkml/[email protected]/]:
Use the devarg to pass-down the VF token.
RFC v1: https://patchwork.dpdk.org/patch/66281/ by Vamsi.
Haiyue Wang (2):
eal: add uuid dependent header files explicitly
eal: support for VFIO-PCI VF token
doc/guides/linux_gsg/linux_drivers.rst | 35 ++++++++++++++++++- doc/guides/linux_gsg/linux_eal_parameters.rst | 4 +++ doc/guides/rel_notes/release_20_08.rst | 6 ++++ lib/librte_eal/common/eal_common_options.c | 3 ++ lib/librte_eal/common/eal_internal_cfg.h | 2 ++ lib/librte_eal/common/eal_options.h | 2 ++ lib/librte_eal/freebsd/eal.c | 5 +++ lib/librte_eal/include/rte_eal.h | 14 ++++++++ lib/librte_eal/include/rte_uuid.h | 2 ++ lib/librte_eal/linux/eal.c | 33 +++++++++++++++++ lib/librte_eal/linux/eal_vfio.c | 19 ++++++++++ lib/librte_eal/rte_eal_version.map | 3 ++ 12 files changed, 127 insertions(+), 1 deletion(-)
Signature · 1 line
2.27.0
PATCH v17 1/2 eal: add uuid dependent header files explicitly
Add the dependent header files explicitly, so that the user just needs to include the 'rte_uuid.h' header file directly to avoid compile error:
(1). rte_uuid.h:97:55: error: unknown type name ‘size_t’ (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’
--- lib/librte_eal/include/rte_uuid.h | 2 ++ 1 file changed, 2 insertions(+)
lib/librte_eal/include/rte_uuid.h +2 -0
@@ -15,6 +15,8 @@ extern "C" {#endif #include <stdbool.h> +#include <stddef.h> +#include <string.h> /** * Struct describing a Universal Unique Identifier
Signature · 1 line
2.27.0
Re: [dpdk-dev] [PATCH v17 1/2] eal: add uuid dependent header files explicitly
Quoted 28 lines · Haiyue Wang
Add the dependent header files explicitly, so that the user just needs to include the '
rte_uuid.h' header file directly to avoid compile error:
(1). rte_uuid.h:97:55: error: unknown type name ‘size_t’ (2). rte_uuid.h:58:2: error: implicit declaration of function ‘memcpy’Fixes 6bc67c497a51 ("eal: add uuid API")Signed-off-by Haiyue Wang <[email protected]>--- lib/librte_eal/include/rte_uuid.h | 2 ++ 1 file changed, 2 insertions(+)lib/librte_eal/include/rte_uuid.h +2
-1@@ -15,6 +15,8 @@ extern "C" {#endif #include <stdbool.h> +#include <stddef.h> +#include <string.h> /** * Struct describing a Universal Unique Identifier --
Quoted 2 lines
2.27.0
Signature · 1 line
David Marchand
PATCH v17 2/2 eal: support for VFIO-PCI VF token
In reply to Haiyue WangThe kernel module vfio-pci introduces the VF token to enable SR-IOV support since 5.7.
The VF token can be set by a vfio-pci based PF driver and must be known by the vfio-pci based VF driver in order to gain access to the device.
Since the vfio-pci module uses the VF token as internal data to provide the collaboration between SR-IOV PF and VFs, so DPDK can use the same VF token for all PF devices by specifying the related EAL option.
--- doc/guides/linux_gsg/linux_drivers.rst | 35 ++++++++++++++++++- doc/guides/linux_gsg/linux_eal_parameters.rst | 4 +++ doc/guides/rel_notes/release_20_08.rst | 6 ++++ lib/librte_eal/common/eal_common_options.c | 3 ++ lib/librte_eal/common/eal_internal_cfg.h | 2 ++ lib/librte_eal/common/eal_options.h | 2 ++ lib/librte_eal/freebsd/eal.c | 5 +++ lib/librte_eal/include/rte_eal.h | 14 ++++++++ lib/librte_eal/linux/eal.c | 33 +++++++++++++++++ lib/librte_eal/linux/eal_vfio.c | 19 ++++++++++ lib/librte_eal/rte_eal_version.map | 3 ++ 11 files changed, 125 insertions(+), 1 deletion(-)
doc/guides/linux_gsg/linux_drivers.rst +34 -1
@@ -78,11 +78,44 @@ Note that in order to use VFIO, your kernel must support it. VFIO kernel modules have been included in the Linux kernel since version 3.6.0 and are usually present by default, however please consult your distributions documentation to make sure that is the case. +The ``vfio-pci`` module since Linux version 5.7 supports the creation of virtual +functions. After the PF is bound to vfio-pci module, the user can create the VFs +by sysfs interface, and these VFs are bound to vfio-pci module automatically. + +When the PF is bound to vfio-pci, it has initial VF token generated by random. For +security reason, this token is write only, the user can't read it from the kernel +directly. To access the VF, the user needs to start the PF with token parameter to +setup a VF token in UUID format, then the VF can be accessed with this new token. + +Since the ``vfio-pci`` module uses the VF token as internal data to provide the +collaboration between SR-IOV PF and VFs, so DPDK can use the same VF token for all +PF devices which bound to one application. This VF token can be specified by the EAL +parameter ``--vfio-vf-token``. + +.. code-block:: console + + 1. Generate the VF token by uuid command + 14d63f20-8445-11ea-8900-1f9ce7d5650d + + 2. sudo modprobe vfio-pci enable_sriov=1 + + 2. ./usertools/dpdk-devbind.py -b vfio-pci 0000:86:00.0 + + 3. echo 2 > /sys/bus/pci/devices/0000:86:00.0/sriov_numvfs + + 4. Start the PF: + ./x86_64-native-linux-gcc/app/testpmd -l 22-25 -n 4 -w 86:00.0 \ + --vfio-vf-token=14d63f20-8445-11ea-8900-1f9ce7d5650d --file-prefix=pf -- -i + + 5. Start the VF: + ./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 -w 86:02.0 \ + --vfio-vf-token=14d63f20-8445-11ea-8900-1f9ce7d5650d --file-prefix=vf0 -- -i + Also, to use VFIO, both kernel and BIOS must support and be configured to use IO virtualization (such as Intel® VT-d). .. note:: - ``vfio-pci`` module doesn't support the creation of virtual functions. + ``vfio-pci`` module doesn't support the creation of virtual functions before Linux version 5.7. For proper operation of VFIO when running DPDK applications as a non-privileged user, correct permissions should also be set up. This can be done by using the DPDK setup script (called dpdk-setup.sh and located in the usertools directory).
doc/guides/linux_gsg/linux_eal_parameters.rst +4 -0
@@ -40,6 +40,10 @@ Device-related options Use specified interrupt mode for devices bound to VFIO kernel driver. +* ``--vfio-vf-token <uuid>`` + + Use specified VF token for devices bound to VFIO kernel driver. + Multiprocessing-related options ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
doc/guides/rel_notes/release_20_08.rst +6 -0
@@ -107,6 +107,12 @@ New Features * Dump ``rte_flow`` memory consumption. * Measure packet per second forwarding. +* **Added the support for vfio-pci new VF token interface.** + + From Linux 5.7, vfio-pci supports to bind both SR-IOV PF and the created VFs, + it uses a shared VF token (UUID) to represent the collaboration between PF + and VFs. Update DPDK PCI driver to gain the access to the PF and VFs devices + by appending the VF token parameter. Removed Items -------------
lib/librte_eal/common/eal_common_options.c +3 -0
@@ -94,6 +94,7 @@ eal_long_options[] = { {OPT_SYSLOG, 1, NULL, OPT_SYSLOG_NUM }, {OPT_VDEV, 1, NULL, OPT_VDEV_NUM }, {OPT_VFIO_INTR, 1, NULL, OPT_VFIO_INTR_NUM }, + {OPT_VFIO_VF_TOKEN, 1, NULL, OPT_VFIO_VF_TOKEN_NUM }, {OPT_VMWARE_TSC_MAP, 0, NULL, OPT_VMWARE_TSC_MAP_NUM }, {OPT_LEGACY_MEM, 0, NULL, OPT_LEGACY_MEM_NUM }, {OPT_SINGLE_FILE_SEGMENTS, 0, NULL, OPT_SINGLE_FILE_SEGMENTS_NUM},
@@ -327,6 +328,8 @@ eal_reset_internal_config(struct internal_config *internal_cfg)/* if set to NONE, interrupt mode is determined automatically */ internal_cfg->vfio_intr_mode =RTE_INTR_MODE_NONE ; +memset (internal_cfg->vfio_vf_token,0 , +sizeof (internal_cfg->vfio_vf_token));#ifdef RTE_LIBEAL_USE_HPET internal_cfg->no_hpet =0 ;
lib/librte_eal/common/eal_internal_cfg.h +2 -0
@@ -72,6 +72,8 @@ struct internal_config {volatile int syslog_facility;/**< facility passed to openlog() */ /** default interrupt mode for VFIO */ volatile enum rte_intr_mode vfio_intr_mode; +/** the shared VF token for VFIO-PCI bound PF and VFs devices */ +rte_uuid_t vfio_vf_token;char *hugefile_prefix;/**< the base filename of hugetlbfs files */ char *hugepage_dir;/**< specific hugetlbfs directory to use */ char *user_mbuf_pool_ops_name;
lib/librte_eal/common/eal_options.h +2 -0
@@ -69,6 +69,8 @@ enum {OPT_VDEV_NUM , #define OPT_VFIO_INTR "vfio-intr"OPT_VFIO_INTR_NUM , +#define OPT_VFIO_VF_TOKEN "vfio-vf-token" +OPT_VFIO_VF_TOKEN_NUM , #define OPT_VMWARE_TSC_MAP "vmware-tsc-map"OPT_VMWARE_TSC_MAP_NUM , #define OPT_LEGACY_MEM "legacy-mem"
lib/librte_eal/freebsd/eal.c +5 -0
@@ -966,6 +966,11 @@ rte_eal_vfio_intr_mode(void)return RTE_INTR_MODE_NONE ; } +void +rte_eal_vfio_get_vf_token(__rte_unused rte_uuid_t vf_token) +{ +} + int rte_vfio_setup_device(__rte_unused const char *sysfs_base, __rte_unused const char *dev_addr, __rte_unused int *vfio_dev_fd,
lib/librte_eal/include/rte_eal.h +14 -0
@@ -19,6 +19,7 @@#include <rte_compat.h> #include <rte_per_lcore.h> #include <rte_bus.h> +#include <rte_uuid.h> #include <rte_pci_dev_feature_defs.h>
@@ -438,6 +439,19 @@ int rte_eal_create_uio_dev(void);*/ enum rte_intr_mode rte_eal_vfio_intr_mode (void ); +/** +* @warning +* @b EXPERIMENTAL: this API may change without prior notice +* +* Copy the user-configured vfio VF token. +* +* @param vf_token +* vfio VF token configured with the command line is copied +* into this parameter, zero uuid by default. +*/ +__rte_experimental +void rte_eal_vfio_get_vf_token(rte_uuid_t vf_token); +/** * A wrap API for syscall gettid. *
lib/librte_eal/linux/eal.c +33 -0
@@ -540,6 +540,7 @@ eal_usage(const char *prgname)" --" OPT_FILE_PREFIX " Prefix for hugepage filenames\n" " --" OPT_CREATE_UIO_DEV " Create /dev/uioX (usually done by hotplug)\n" " --" OPT_VFIO_INTR " Interrupt mode for VFIO (legacy|msi|msix)\n" +" --" OPT_VFIO_VF_TOKEN " VF token (UUID) shared between SR-IOV PF and VFs\n" " --" OPT_LEGACY_MEM " Legacy memory mode (no dynamic allocation, contiguous segments)\n" " --" OPT_SINGLE_FILE_SEGMENTS " Put all hugepage memory in single files\n" " --" OPT_MATCH_ALLOCATIONS " Free hugepages exactly as allocated\n"
@@ -620,6 +621,20 @@ eal_parse_vfio_intr(const char *mode)return -1 ; } +static int +eal_parse_vfio_vf_token (const char *vf_token) +{ +struct internal_config *cfg =eal_get_internal_configuration (); +rte_uuid_t uuid; + +if (!rte_uuid_parse (vf_token, uuid)) { +rte_uuid_copy (cfg->vfio_vf_token, uuid); +return 0 ; + } + +return -1 ; +} +/* Parse the arguments for --log-level only */ static void eal_log_level_parse (int argc,char **argv)
@@ -762,6 +777,16 @@ eal_parse_args(int argc, char **argv) }break ; +case OPT_VFIO_VF_TOKEN_NUM : +if (eal_parse_vfio_vf_token (optarg) <0 ) { +RTE_LOG (ERR ,EAL ,"invalid parameters for --" +OPT_VFIO_VF_TOKEN "\n" ); +eal_usage (prgname); + ret =-1 ; + goto out; + } +break ; +case OPT_CREATE_UIO_DEV_NUM : internal_conf->create_uio_dev =1 ;break ;
@@ -1342,6 +1367,14 @@ rte_eal_vfio_intr_mode(void)return internal_conf->vfio_intr_mode; } +void +rte_eal_vfio_get_vf_token (rte_uuid_t vf_token) +{ +struct internal_config *cfg =eal_get_internal_configuration (); + +rte_uuid_copy (vf_token, cfg->vfio_vf_token); +} +int rte_eal_check_module (const char *module_name) {
lib/librte_eal/linux/eal_vfio.c +19 -0
@@ -714,6 +714,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,int vfio_container_fd;int vfio_group_fd;int iommu_group_num; +rte_uuid_t vf_token;int i, ret;const struct internal_config *internal_conf =eal_get_internal_configuration ();
@@ -899,6 +900,23 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr, t->type_id, t->name); } +rte_eal_vfio_get_vf_token (vf_token); + +/* get a file descriptor for the device with VF token firstly */ +if (!rte_uuid_is_null (vf_token)) { +char vf_token_str[RTE_UUID_STRLEN ]; +char dev[PATH_MAX ]; + +rte_uuid_unparse (vf_token, vf_token_str,sizeof (vf_token_str)); +snprintf (dev,sizeof (dev), +"%s vf_token=%s" , dev_addr, vf_token_str); + + *vfio_dev_fd =ioctl (vfio_group_fd,VFIO_GROUP_GET_DEVICE_FD , + dev); +if (*vfio_dev_fd >=0 ) + goto dev_get_info; + } +/* get a file descriptor for the device */ *vfio_dev_fd =ioctl (vfio_group_fd,VFIO_GROUP_GET_DEVICE_FD , dev_addr);if (*vfio_dev_fd <0 ) {
@@ -914,6 +932,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr, }/* test and setup the device */ +dev_get_info: ret =ioctl (*vfio_dev_fd,VFIO_DEVICE_GET_INFO , device_info);if (ret) { RTE_LOG(ERR, EAL, " %s cannot get device info, "
lib/librte_eal/rte_eal_version.map +3 -0
@@ -386,6 +386,9 @@ EXPERIMENTAL { rte_trace_point_lookup; rte_trace_regexp; rte_trace_save; + + # added in 20.08 + rte_eal_vfio_get_vf_token; }; INTERNAL {
Signature · 1 line
2.27.0
Re: [dpdk-dev] [PATCH v17 2/2] eal: support for VFIO-PCI VF token
Quoted 7 lines · Haiyue Wang
The kernel module vfio-pci introduces the VF token to enable SR-IOV support since 5.7.
The VF token can be set by a vfio-pci based PF driver and must be known by the vfio-pci based VF driver in order to gain access to the device.
Quoted 8 lines
Since the vfio-pci module uses the VF token as internal data to provide the collaboration between SR-IOV PF and VFs, so DPDK can use the same VF token for all PF devices by specifying the related EAL option.
Signed-off-by Haiyue Wang <[email protected]>Acked-by Anatoly Burakov <[email protected]>Acked-by Andrew Rybchenko <[email protected]>Tested-by Harman Kalra <[email protected]>
Series applied, thanks Haiyue.
Signature · 1 line
David Marchand