bus/pci: restricted bus scanning to allowed devices
PATCH 1/1 bus/pci: restricted bus scanning to allowed devices
rte_bus_scan API scans all the available PCI devices irrespective of white or black listing parameters then further devices are probed based on white or black listing parameters. So unnecessary CPU cycles are wasted during rte_pci_scan.
For Octeontx2 platform with core frequency 2.4 Ghz, rte_bus_scan consumes around 26ms to scan around 90 PCI devices but all may not be used by the application. So for the application which uses 2 NICs, rte_bus_scan consumes few microseconds and rest time is saved with this patch.
Patch restricts devices to be scanned as per below mentioned conditions:
- All devices will be scanned if no parameters are passed.
- Only white listed devices will be scanned if white list is available.
- All devices, except black listed, will be scanned if black list is available.
--- drivers/bus/pci/bsd/pci.c | 18 +++++++++++++++++- drivers/bus/pci/linux/pci.c | 11 +++++++++++ drivers/bus/pci/pci_common.c | 4 ++-- drivers/bus/pci/private.h | 20 ++++++++++++++++++++ 4 files changed, 50 insertions(+), 3 deletions(-)
drivers/bus/pci/bsd/pci.c +17 -1
@@ -338,6 +338,9 @@ rte_pci_scan(void) .match_buf_len = sizeof(matches), .matches = &matches[0], }; +struct rte_pci_device dummy_dev; + +memset (&dummy_dev,0 ,sizeof (struct rte_pci_device ));/* for debug purposes, PCI can be disabled */ if (!rte_eal_has_pci ())
@@ -357,9 +360,22 @@ rte_pci_scan(void) goto error; } -for (i =0 ; i < conf_io.num_matches; i++) +for (i =0 ; i < conf_io.num_matches; i++) { +/* Create dummy pci device to get devargs */ + dummy_dev.addr.domain = matches[i].pc_sel.pc_domain; + dummy_dev.addr.bus = matches[i].pc_sel.pc_bus; + dummy_dev.addr.devid = matches[i].pc_sel.pc_dev; + dummy_dev.addr.function = matches[i].pc_sel.pc_func; + dummy_dev.device.devargs = +pci_devargs_lookup (&dummy_dev); + +/* Check that device should be ignored or not */ +if (pci_ignore_device (&dummy_dev)) +continue ; +if (pci_scan_one (fd, &matches[i]) <0 ) goto error; + } dev_count += conf_io.num_matches; }while (conf_io.status ==PCI_GETCONF_MORE_DEVS );
drivers/bus/pci/linux/pci.c +11 -0
@@ -458,6 +458,9 @@ rte_pci_scan(void)DIR *dir;char dirname[PATH_MAX ];struct rte_pci_addr addr; +struct rte_pci_device dummy_dev; + +memset (&dummy_dev,0 ,sizeof (struct rte_pci_device ));/* for debug purposes, PCI can be disabled */ if (!rte_eal_has_pci ())
@@ -482,6 +485,14 @@ rte_pci_scan(void)if (parse_pci_addr_format (e->d_name,sizeof (e->d_name), &addr) !=0 )continue ; +/* Create dummy pci device to get devargs */ + dummy_dev.addr = addr; + dummy_dev.device.devargs =pci_devargs_lookup (&dummy_dev); + +/* Check that device should be ignored or not */ +if (pci_ignore_device (&dummy_dev)) +continue ; +snprintf (dirname,sizeof (dirname),"%s/%s" ,rte_pci_get_sysfs_path (), e->d_name);
drivers/bus/pci/pci_common.c +2 -2
@@ -42,7 +42,7 @@ const char *rte_pci_get_sysfs_path(void)return path; } -static struct rte_devargs *pci_devargs_lookup (struct rte_pci_device *dev) +struct rte_devargs *pci_devargs_lookup (struct rte_pci_device *dev) {struct rte_devargs *devargs;struct rte_pci_addr addr;
@@ -589,7 +589,7 @@ pci_dma_unmap(struct rte_device *dev, void *addr, uint64_t iova, size_t len)return -1 ; } -static bool +bool pci_ignore_device (const struct rte_pci_device *dev) {struct rte_devargs *devargs = dev->device.devargs;
drivers/bus/pci/private.h +20 -0
@@ -42,6 +42,26 @@ int rte_pci_scan(void);void pci_name_set (struct rte_pci_device *dev); +/** +* Get the devargs of a PCI device. +* +* @param pci_dev +* PCI device to be validated +* @return +* devargs on succes, NULL otherwise +*/ +struct rte_devargs *pci_devargs_lookup (struct rte_pci_device *pci_dev); + +/** +* Validate whether a pci device should be ignored or not. +* +* @param pci_dev +* PCI device to be validated +* @return +* 1 if device is to be ignored, 0 otherwise +*/ +bool pci_ignore_device (const struct rte_pci_device *pci_dev); +/** * Add a PCI device to the PCI Bus (append to PCI Device list). This function * also updates the bus references of the PCI Device (and the generic device
Signature · 1 line
2.17.1
drivers/bus/pci/bsd/pci.c rte_pci_scan(void) · line 374
Quoted 11 lines
/* Create dummy pci device to get devargs */drivers/bus/pci/bsd/pci.c +10
-0+ dummy_dev.addr.domain = matches[i].pc_sel.pc_domain; + dummy_dev.addr.bus = matches[i].pc_sel.pc_bus; + dummy_dev.addr.devid = matches[i].pc_sel.pc_dev; + dummy_dev.addr.function = matches[i].pc_sel.pc_func; + dummy_dev.device.devargs = +pci_devargs_lookup (&dummy_dev); + +/* Check that device should be ignored or not */ +if (pci_ignore_device (&dummy_dev)) +continue ;
It seems that you are creating dummy_dev as an alternative to passing just the PCI bus/device/function. Wouldn't be easier to just use BDF instead. Dummy arguments on the stack can lead to more corner cases in the future if device subsystem changes.
drivers/bus/pci/private.h int rte_pci_scan(void); · line 53
Quoted 9 lines
drivers/bus/pci/private.h +9
-0+/** +* Get the devargs of a PCI device. +* +* @param pci_dev +* PCI device to be validated +* @return +* devargs on succes, NULL otherwise +*/ +struct rte_devargs *pci_devargs_lookup (struct rte_pci_device *pci_dev);
Must be marked experimental (or internal).
The pci_device should be marked const.
drivers/bus/pci/private.h int rte_pci_scan(void); · line 63
Quoted 10 lines
drivers/bus/pci/private.h +10
-0+ +/** +* Validate whether a pci device should be ignored or not. +* +* @param pci_dev +* PCI device to be validated +* @return +* 1 if device is to be ignored, 0 otherwise +*/ +bool pci_ignore_device (const struct rte_pci_device *pci_dev);
ditto
Regards
Sunil Kumar Kori
Quoted 21 lines
-----Original Message-----
From: Stephen Hemminger <[email protected]>
Sent: Monday, December 16, 2019 9:43 PM
To: Sunil Kumar Kori <[email protected]>
Cc: [email protected]
Subject: [EXT] Re: [dpdk-dev] [PATCH] bus/pci: restricted bus scanning to allowed devicesExternal Email
----------------------------------------------------------------------
Review
Quoted 2 lines
/* Create dummy pci device to get devargs */(quoted diff) +1
-0+ dummy_dev.addr.domain =matches[i].pc_sel.pc_domain;
drivers/bus/pci/bsd/pci.c rte_pci_scan(void)
Quoted 3 lines
drivers/bus/pci/bsd/pci.c +3
-0+ dummy_dev.addr.bus = matches[i].pc_sel.pc_bus; + dummy_dev.addr.devid = matches[i].pc_sel.pc_dev; + dummy_dev.addr.function =matches[i].pc_sel.pc_func;
Review
Quoted 2 lines
(quoted diff) +2
-0+ dummy_dev.device.devargs = +pci_devargs_lookup(&dummy_dev);
Quoted 9 lines
drivers/bus/pci/bsd/pci.c rte_pci_scan(void) · line 374
Quoted 4 lines
drivers/bus/pci/bsd/pci.c +4
-0+ +/* Check that device should be ignored or not */ +if (pci_ignore_device (&dummy_dev)) +continue ;It seems that you are creating
dummy_devas an alternative to passing just the PCI bus/device/function. Wouldn't be easier to just use BDF instead. Dummy arguments on the stack can lead to more corner cases in the future if device subsystem changes.
Agreed and initially I have implemented using BDF only instead of using dummy device. But using that approach, I was not able to use existing APIs to get devargs and ignore device. I had to write almost same functions to solve the purpose. So just to avoid having replica of same code, I followed this approach. Please suggest.
Quoted 14 lines
drivers/bus/pci/private.h int rte_pci_scan(void);
Quoted 11 lines
drivers/bus/pci/private.h +10
-0+/** +* Get the devargs of a PCI device. +* +* @param pci_dev +* PCI device to be validated +* @return +* devargs on succes, NULL otherwise +*/ +struct rte_devargs *pci_devargs_lookup (struct rte_pci_device +*pci_dev);Must be marked experimental (or internal).
Thepci_deviceshould be marked const.
Okay but If I go with BDF one then this change is not required anyway.
Quoted 14 lines
drivers/bus/pci/private.h int rte_pci_scan(void); · line 63
Quoted 12 lines
drivers/bus/pci/private.h +10
-0+ +/** +* Validate whether a pci device should be ignored or not. +* +* @param pci_dev +* PCI device to be validated +* @return +* 1 if device is to be ignored, 0 otherwise +*/ +bool pci_ignore_device (const struct rte_pci_device *pci_dev);ditto
ditto
Hello Stephen,
Any suggestions ?
Regards
Sunil Kumar Kori
Quoted 79 lines
-----Original Message-----
From: dev <[email protected]> On Behalf Of Sunil Kumar Kori
Sent: Tuesday, December 17, 2019 4:30 PM
To: Stephen Hemminger <[email protected]>
Cc: [email protected]
Subject: Re: [dpdk-dev] [EXT] Re: [PATCH] bus/pci: restricted bus scanning to allowed devicesRegards
Sunil Kumar KoriQuoted 21 lines
-----Original Message-----
From: Stephen Hemminger <[email protected]>
Sent: Monday, December 16, 2019 9:43 PM
To: Sunil Kumar Kori <[email protected]>
Cc: [email protected]
Subject: [EXT] Re: [dpdk-dev] [PATCH] bus/pci: restricted bus scanning to allowed devicesExternal Email
----------------------------------------------------------------------
Review
Quoted 2 lines
/* Create dummy pci device to get devargs */(quoted diff) +1
-0+ dummy_dev.addr.domain =matches[i].pc_sel.pc_domain;
drivers/bus/pci/bsd/pci.c rte_pci_scan(void)
Quoted 3 lines
drivers/bus/pci/bsd/pci.c +3
-0+ dummy_dev.addr.bus = matches[i].pc_sel.pc_bus; + dummy_dev.addr.devid = matches[i].pc_sel.pc_dev; + dummy_dev.addr.function =matches[i].pc_sel.pc_func;
Review
Quoted 2 lines
(quoted diff) +2
-0+ dummy_dev.device.devargs = +pci_devargs_lookup(&dummy_dev);Quoted 9 lines
drivers/bus/pci/bsd/pci.c rte_pci_scan(void) · line 374
Quoted 4 lines
drivers/bus/pci/bsd/pci.c +4
-0+ +/* Check that device should be ignored or not */ +if (pci_ignore_device (&dummy_dev)) +continue ;It seems that you are creating
dummy_devas an alternative to passing just the PCI bus/device/function. Wouldn't be easier to just use BDF instead. Dummy arguments on the stack can lead to more corner cases in the future if device subsystem changes.Agreed and initially I have implemented using BDF only instead of using dummy device.
But using that approach, I was not able to use existing APIs to get devargs and ignore device.
I had to write almost same functions to solve the purpose. So just to avoid having replica of same code, I followed this approach. Please suggest.Quoted 14 lines
drivers/bus/pci/private.h int rte_pci_scan(void);
Quoted 11 lines
drivers/bus/pci/private.h +10
-0+/** +* Get the devargs of a PCI device. +* +* @param pci_dev +* PCI device to be validated +* @return +* devargs on succes, NULL otherwise +*/ +struct rte_devargs *pci_devargs_lookup (struct rte_pci_device +*pci_dev);Must be marked experimental (or internal).
Thepci_deviceshould be marked const.Okay but If I go with BDF one then this change is not required anyway.
Quoted 14 lines
drivers/bus/pci/private.h int rte_pci_scan(void); · line 63
Quoted 12 lines
drivers/bus/pci/private.h +10
-0+ +/** +* Validate whether a pci device should be ignored or not. +* +* @param pci_dev +* PCI device to be validated +* @return +* 1 if device is to be ignored, 0 otherwise +*/ +bool pci_ignore_device (const struct rte_pci_device *pci_dev);ditto
ditto
A message that is not in the archive (sent off-list or not yet received): MN2PR18MB327936807460D9F2AE4894F3B40F0@MN2PR18MB3279.namprd18.prod.outlook.com
Hello All,
Is there any thought on this ? Otherwise it can be merged.
Regards
Sunil Kumar Kori
Quoted 110 lines
-----Original Message-----
From: Sunil Kumar Kori
Sent: Thursday, January 23, 2020 2:13 PM
To: Stephen Hemminger <[email protected]>; Jerin Jacob Kollanukkaran <[email protected]>
Subject: FW: [dpdk-dev] [EXT] Re: [PATCH] bus/pci: restricted bus scanning to allowed devicesHello Stephen,
Can you please look into this patch or provide your thought in this ? So that it can be merged within 20.02 release.
Regards
Sunil Kumar Kori-----Original Message-----
From: Sunil Kumar Kori <[email protected]>
Sent: Tuesday, January 21, 2020 2:09 PM
To: Sunil Kumar Kori <[email protected]>; Stephen Hemminger <[email protected]>
Cc: [email protected]
Subject: RE: [dpdk-dev] [EXT] Re: [PATCH] bus/pci: restricted bus scanning to allowed devicesHello Stephen,
Any suggestions ?Regards
Sunil Kumar KoriQuoted 79 lines
-----Original Message-----
From: dev <[email protected]> On Behalf Of Sunil Kumar Kori
Sent: Tuesday, December 17, 2019 4:30 PM
To: Stephen Hemminger <[email protected]>
Cc: [email protected]
Subject: Re: [dpdk-dev] [EXT] Re: [PATCH] bus/pci: restricted bus scanning to allowed devicesRegards
Sunil Kumar KoriQuoted 21 lines
-----Original Message-----
From: Stephen Hemminger <[email protected]>
Sent: Monday, December 16, 2019 9:43 PM
To: Sunil Kumar Kori <[email protected]>
Cc: [email protected]
Subject: [EXT] Re: [dpdk-dev] [PATCH] bus/pci: restricted bus scanning to allowed devicesExternal Email
----------------------------------------------------------------------
Review
Quoted 2 lines
/* Create dummy pci device to get devargs */(quoted diff) +1
-0+ dummy_dev.addr.domain =matches[i].pc_sel.pc_domain;
drivers/bus/pci/bsd/pci.c rte_pci_scan(void)
Quoted 3 lines
drivers/bus/pci/bsd/pci.c +3
-0+ dummy_dev.addr.bus = matches[i].pc_sel.pc_bus; + dummy_dev.addr.devid = matches[i].pc_sel.pc_dev; + dummy_dev.addr.function =matches[i].pc_sel.pc_func;
Review
Quoted 2 lines
(quoted diff) +2
-0+ dummy_dev.device.devargs = +pci_devargs_lookup(&dummy_dev);Quoted 9 lines
drivers/bus/pci/bsd/pci.c rte_pci_scan(void) · line 374
Quoted 4 lines
drivers/bus/pci/bsd/pci.c +4
-0+ +/* Check that device should be ignored or not */ +if (pci_ignore_device (&dummy_dev)) +continue ;It seems that you are creating
dummy_devas an alternative to passing just the PCI bus/device/function. Wouldn't be easier to just use BDF instead. Dummy arguments on the stack can lead to more corner cases in the future if device subsystem changes.Agreed and initially I have implemented using BDF only instead of using dummy device.
But using that approach, I was not able to use existing APIs to get devargs and ignore device.
I had to write almost same functions to solve the purpose. So just to avoid having replica of same code, I followed this approach. Please suggest.Quoted 14 lines
drivers/bus/pci/private.h int rte_pci_scan(void);
Quoted 11 lines
drivers/bus/pci/private.h +10
-0+/** +* Get the devargs of a PCI device. +* +* @param pci_dev +* PCI device to be validated +* @return +* devargs on succes, NULL otherwise +*/ +struct rte_devargs *pci_devargs_lookup (struct rte_pci_device +*pci_dev);Must be marked experimental (or internal).
Thepci_deviceshould be marked const.Okay but If I go with BDF one then this change is not required anyway.
Quoted 14 lines
drivers/bus/pci/private.h int rte_pci_scan(void); · line 63
Quoted 12 lines
drivers/bus/pci/private.h +10
-0+ +/** +* Validate whether a pci device should be ignored or not. +* +* @param pci_dev +* PCI device to be validated +* @return +* 1 if device is to be ignored, 0 otherwise +*/ +bool pci_ignore_device (const struct rte_pci_device *pci_dev);ditto
ditto
Hello Stephen,
Please provide ack on below change if there is no concern so that it can be accepted on 20.05.
Regards
Sunil Kumar Kori
Quoted 126 lines
-----Original Message-----
From: Sunil Kumar Kori
Sent: Thursday, February 27, 2020 2:00 PM
To: Stephen Hemminger <[email protected]>; Jerin Jacob Kollanukkaran <[email protected]>; [email protected]
Subject: RE: [dpdk-dev] [EXT] Re: [PATCH] bus/pci: restricted bus scanning to allowed devicesHello All,
Is there any thought on this ? Otherwise it can be merged.
Regards
Sunil Kumar KoriQuoted 111 lines
-----Original Message-----
From: Sunil Kumar Kori
Sent: Thursday, January 23, 2020 2:13 PM
To: Stephen Hemminger <[email protected]>; Jerin Jacob Kollanukkaran <[email protected]>
Subject: FW: [dpdk-dev] [EXT] Re: [PATCH] bus/pci: restricted bus scanning to allowed devicesHello Stephen,
Can you please look into this patch or provide your thought in this ? So that it can be merged within 20.02 release.
Regards
Sunil Kumar Kori-----Original Message-----
From: Sunil Kumar Kori <[email protected]>
Sent: Tuesday, January 21, 2020 2:09 PM
To: Sunil Kumar Kori <[email protected]>; Stephen Hemminger <[email protected]>
Cc: [email protected]
Subject: RE: [dpdk-dev] [EXT] Re: [PATCH] bus/pci: restricted bus scanning to allowed devicesHello Stephen,
Any suggestions ?Regards
Sunil Kumar KoriQuoted 80 lines
-----Original Message-----
From: dev <[email protected]> On Behalf Of Sunil Kumar Kori
Sent: Tuesday, December 17, 2019 4:30 PM
To: Stephen Hemminger <[email protected]>
Cc: [email protected]
Subject: Re: [dpdk-dev] [EXT] Re: [PATCH] bus/pci: restricted bus scanning to allowed devicesRegards
Sunil Kumar KoriQuoted 22 lines
-----Original Message-----
From: Stephen Hemminger <[email protected]>
Sent: Monday, December 16, 2019 9:43 PM
To: Sunil Kumar Kori <[email protected]>
Cc: [email protected]
Subject: [EXT] Re: [dpdk-dev] [PATCH] bus/pci: restricted bus scanning to allowed devicesExternal Email
---------------------------------------------------------------------
-Review
Quoted 2 lines
/* Create dummy pci device to get devargs */(quoted diff) +1
-0+ dummy_dev.addr.domain =matches[i].pc_sel.pc_domain;
drivers/bus/pci/bsd/pci.c rte_pci_scan(void)
Quoted 3 lines
drivers/bus/pci/bsd/pci.c +3
-0+ dummy_dev.addr.bus = matches[i].pc_sel.pc_bus; + dummy_dev.addr.devid = matches[i].pc_sel.pc_dev; + dummy_dev.addr.function =matches[i].pc_sel.pc_func;
Review
Quoted 2 lines
(quoted diff) +2
-0+ dummy_dev.device.devargs = +pci_devargs_lookup(&dummy_dev);Quoted 9 lines
drivers/bus/pci/bsd/pci.c rte_pci_scan(void) · line 374
Quoted 4 lines
drivers/bus/pci/bsd/pci.c +4
-0+ +/* Check that device should be ignored or not */ +if (pci_ignore_device (&dummy_dev)) +continue ;It seems that you are creating
dummy_devas an alternative to passing just the PCI bus/device/function. Wouldn't be easier to just use BDF instead. Dummy arguments on the stack can lead to more corner cases in the future if device subsystem changes.Agreed and initially I have implemented using BDF only instead of using dummy device.
But using that approach, I was not able to use existing APIs to get devargs and ignore device.
I had to write almost same functions to solve the purpose. So just to avoid having replica of same code, I followed this approach. Please suggest.Quoted 14 lines
drivers/bus/pci/private.h int rte_pci_scan(void);
Quoted 11 lines
drivers/bus/pci/private.h +10
-0+/** +* Get the devargs of a PCI device. +* +* @param pci_dev +* PCI device to be validated +* @return +* devargs on succes, NULL otherwise +*/ +struct rte_devargs *pci_devargs_lookup (struct rte_pci_device +*pci_dev);Must be marked experimental (or internal).
Thepci_deviceshould be marked const.Okay but If I go with BDF one then this change is not required anyway.
Quoted 14 lines
drivers/bus/pci/private.h int rte_pci_scan(void); · line 63
Quoted 12 lines
drivers/bus/pci/private.h +10
-0+ +/** +* Validate whether a pci device should be ignored or not. +* +* @param pci_dev +* PCI device to be validated +* @return +* 1 if device is to be ignored, 0 otherwise +*/ +bool pci_ignore_device (const struct rte_pci_device *pci_dev);ditto
ditto
Hello,
It looks like there is no comment/objection on following patch and it can be merged. I would request to @David Marchand, please take care of this towards the merging process for 20.05.
Regards
Sunil Kumar Kori
Quoted 147 lines
-----Original Message-----
From: Sunil Kumar Kori
Sent: Monday, March 9, 2020 11:37 AM
To: 'Stephen Hemminger' <[email protected]>; Jerin Jacob Kollanukkaran <[email protected]>; '[email protected]' <[email protected]>
Subject: RE: [dpdk-dev] [EXT] Re: [PATCH] bus/pci: restricted bus scanning to allowed devicesHello Stephen,
Please provide ack on below change if there is no concern so that it can be accepted on 20.05.
Regards
Sunil Kumar KoriQuoted 69 lines
-----Original Message-----
From: Sunil Kumar Kori
Sent: Thursday, February 27, 2020 2:00 PM
To: Stephen Hemminger <[email protected]>; Jerin Jacob Kollanukkaran <[email protected]>; [email protected]
Subject: RE: [dpdk-dev] [EXT] Re: [PATCH] bus/pci: restricted bus scanning to allowed devicesHello All,
Is there any thought on this ? Otherwise it can be merged.
Regards
Sunil Kumar KoriQuoted 54 lines
-----Original Message-----
From: Sunil Kumar Kori
Sent: Thursday, January 23, 2020 2:13 PM
To: Stephen Hemminger <[email protected]>; Jerin Jacob Kollanukkaran <[email protected]>
Subject: FW: [dpdk-dev] [EXT] Re: [PATCH] bus/pci: restricted bus scanning to allowed devicesHello Stephen,
Can you please look into this patch or provide your thought in this ? So that it can be merged within 20.02 release.
Regards
Sunil Kumar Kori-----Original Message-----
From: Sunil Kumar Kori <[email protected]>
Sent: Tuesday, January 21, 2020 2:09 PM
To: Sunil Kumar Kori <[email protected]>; Stephen Hemminger <[email protected]>
Cc: [email protected]
Subject: RE: [dpdk-dev] [EXT] Re: [PATCH] bus/pci: restricted bus scanning to allowed devicesHello Stephen,
Any suggestions ?Regards
Sunil Kumar KoriQuoted 23 lines
-----Original Message-----
From: dev <[email protected]> On Behalf Of Sunil Kumar Kori
Sent: Tuesday, December 17, 2019 4:30 PM
To: Stephen Hemminger <[email protected]>
Cc: [email protected]
Subject: Re: [dpdk-dev] [EXT] Re: [PATCH] bus/pci: restricted bus scanning to allowed devicesRegards
Sunil Kumar KoriQuoted 10 lines
-----Original Message-----
From: Stephen Hemminger <[email protected]>
Sent: Monday, December 16, 2019 9:43 PM
To: Sunil Kumar Kori <[email protected]>
Cc: [email protected]
Subject: [EXT] Re: [dpdk-dev] [PATCH] bus/pci: restricted bus scanning to allowed devicesExternal Email
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
--------------------------------------------------------------------
(quoted diff) +0
-2- -Review
Quoted 2 lines
/* Create dummy pci device to get devargs */(quoted diff) +1
-0+ dummy_dev.addr.domain =matches[i].pc_sel.pc_domain;
Review
Quoted 1 line
(quoted diff) +1
-0+ dummy_dev.addr.bus =matches[i].pc_sel.pc_bus;
Quoted 1 line
Quoted 1 line
Quoted 1 line
Quoted 1 line
Review
Quoted 1 line
(quoted diff) +1
-0+ dummy_dev.addr.devid =matches[i].pc_sel.pc_dev;
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Review
Quoted 1 line
(quoted diff) +1
-0+ dummy_dev.addr.function =matches[i].pc_sel.pc_func;
Review
Quoted 2 lines
(quoted diff) +2
-0+ dummy_dev.device.devargs = +pci_devargs_lookup(&dummy_dev);Review
Quoted 2 lines
(quoted diff) +2
-0+ + /* Check that device should be ignored or not*/
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Quoted 7 lines
Review
Quoted 2 lines
(quoted diff) +2
-0+ if (pci_ignore_device(&dummy_dev)) + continue;It seems that you are creating
dummy_devas an alternative to passing just the PCI bus/device/function. Wouldn't be easier to just use BDF instead. Dummy arguments on the stack can lead to more corner cases in the future if device subsystem changes.Quoted 6 lines
Quoted 6 lines
Quoted 6 lines
Agreed and initially I have implemented using BDF only instead of using dummy device.
But using that approach, I was not able to use existing APIs to get devargs and ignore device.
I had to write almost same functions to solve the purpose. So just to avoid having replica of same code, I followed this approach. Pleasesuggest.
Quoted 30 lines
Quoted 30 lines
Quoted 30 lines
Quoted 14 lines
Review
Quoted 11 lines
(quoted diff) +10
-0+/** + * Get the devargs of a PCI device. + * + * @param pci_dev + * PCI device to be validated + * @return + * devargs on succes, NULL otherwise + */ +struct rte_devargs *pci_devargs_lookup(struct rte_pci_device +*pci_dev);Must be marked experimental (or internal).
Thepci_deviceshould be marked const.Okay but If I go with BDF one then this change is not required anyway.
Quoted 14 lines
Review
Quoted 12 lines
(quoted diff) +10
-0+ +/** + * Validate whether a pci device should be ignored or not. + * + * @param pci_dev + * PCI device to be validated + * @return + * 1 if device is to be ignored, 0 otherwise + */ +bool pci_ignore_device(const struct rte_pci_device *pci_dev);ditto
ditto
Quoted 1 line · Sunil Kumar Kori
It looks like there is no comment/objection on following patch and it can be merged.
The title does not reflect what this patch is about. I understand this as an optimisation for the pci bus scanning.
I agree with Stephen comment.
If you change pci_ignore_device and pci_devargs_lookup to take a rte_pci_addr as input, then pci_ignore_device can call pci_devargs_lookup itself.
And pci_devargs_lookup does not need to be exported in the private header.
Untested (just tried compilation),
https://github.com/david-marchand/dpdk/commit/e7860231ecdce91f9f70027d4090a7057b8fd5f7
Signature · 1 line
David Marchand
Regards
Sunil Kumar Kori
Quoted 33 lines
-----Original Message-----
From: David Marchand <[email protected]>
Sent: Monday, April 6, 2020 6:52 PM
To: Sunil Kumar Kori <[email protected]>
Cc: Stephen Hemminger <[email protected]>; Jerin Jacob Kollanukkaran <[email protected]>; [email protected]
Subject: Re: [dpdk-dev] [EXT] Re: [PATCH] bus/pci: restricted bus scanning to allowed devicesQuoted 1 line · Sunil Kumar Kori
It looks like there is no comment/objection on following patch and it can be
merged.
The title does not reflect what this patch is about. I understand this as an optimisation for the pci bus scanning.
I agree with Stephen comment.
If you change
pci_ignore_deviceandpci_devargs_lookupto take arte_pci_addras input, thenpci_ignore_devicecan callpci_devargs_lookupitself.
Andpci_devargs_lookupdoes not need to be exported in the private header.Untested (just tried compilation),
https://urldefense.proofpoint.com/v2/url?u=https-3A__github.com_david-
2Dmarchand_dpdk_commit_e7860231ecdce91f9f70027d4090a7057b8fd5f7& d=DwIFaQ&c=nKjWec2b6R0mOyPaz7xtfQ&r=dXeXaAMkP5COgn1zxHMyaF1_d 9IIuq6vHQO6NrIPjaE&m=H7IQgd6gnpQBcbVgN952C_oIcMj5Z34kRhYngZr8Pyc &s=eq-hCdZ-C4xZ0Y7rQEkYwJS04r5sh7G2I9CXHTuBmws&e=
Quoted 2 lines
--
David Marchand
Okay. I will share updated version of this based on review comments.
PATCH v2 1/1 bus/pci: optimise scanning with whitelist/blacklist
In reply to Sunil Kumar Korirte_bus_scan API scans all the available PCI devices irrespective of white or black listing parameters then further devices are probed based on white or black listing parameters. So unnecessary CPU cycles are wasted during rte_pci_scan.
For Octeontx2 platform with core frequency 2.4 Ghz, rte_bus_scan consumes around 26ms to scan around 90 PCI devices but all may not be used by the application. So for the application which uses 2 NICs, rte_bus_scan consumes few microseconds and rest time is saved with this patch.
Patch restricts devices to be scanned as per below mentioned conditions:
- All devices will be scanned if no parameters are passed.
- Only white listed devices will be scanned if white list is available.
- All devices, except black listed, will be scanned if black list is available.
---
v2:
- Added function to validate ignorance of device based on PCI address.
- Marked device validation function as experimental.
drivers/bus/pci/bsd/pci.c | 15 ++++++++++- drivers/bus/pci/linux/pci.c | 3 +++ drivers/bus/pci/pci_common.c | 34 +++++++++++++++++++++++++ drivers/bus/pci/private.h | 12 +++++++++ drivers/bus/pci/rte_bus_pci_version.map | 5 ++++ 5 files changed, 68 insertions(+), 1 deletion(-)
drivers/bus/pci/bsd/pci.c +14 -1
@@ -338,6 +338,9 @@ rte_pci_scan(void) .match_buf_len = sizeof(matches), .matches = &matches[0], }; +struct rte_pci_addr pci_addr; + +memset (&pci_addr,0 ,sizeof (struct rte_pci_addr ));/* for debug purposes, PCI can be disabled */ if (!rte_eal_has_pci ())
@@ -357,9 +360,19 @@ rte_pci_scan(void) goto error; } -for (i =0 ; i < conf_io.num_matches; i++) +for (i =0 ; i < conf_io.num_matches; i++) { + pci_addr.domain = matches[i].pc_sel.pc_domain; + pci_addr.bus = matches[i].pc_sel.pc_bus; + pci_addr.devid = matches[i].pc_sel.pc_dev; + pci_addr.function = matches[i].pc_sel.pc_func; + +/* Check that device should be ignored or not */ +if (pci_addr_ignore_device (&pci_addr)) +continue ; +if (pci_scan_one (fd, &matches[i]) <0 ) goto error; + } dev_count += conf_io.num_matches; }while (conf_io.status ==PCI_GETCONF_MORE_DEVS );
drivers/bus/pci/linux/pci.c +3 -0
@@ -487,6 +487,9 @@ rte_pci_scan(void)if (parse_pci_addr_format (e->d_name,sizeof (e->d_name), &addr) !=0 )continue ; +if (pci_addr_ignore_device (&addr)) +continue ; +snprintf (dirname,sizeof (dirname),"%s/%s" ,rte_pci_get_sysfs_path (), e->d_name);
drivers/bus/pci/pci_common.c +34 -0
@@ -589,6 +589,40 @@ pci_dma_unmap(struct rte_device *dev, void *addr, uint64_t iova, size_t len)return -1 ; } +static struct rte_devargs * +pci_addr_devargs_lookup (const struct rte_pci_addr *pci_addr) +{ +struct rte_devargs *devargs; +struct rte_pci_addr addr; + + RTE_EAL_DEVARGS_FOREACH("pci", devargs) { + devargs->bus->parse (devargs->name, &addr); +if (!rte_pci_addr_cmp (pci_addr, &addr)) +return devargs; + } +return NULL ; +} + +bool +pci_addr_ignore_device (const struct rte_pci_addr *pci_addr) +{ +struct rte_devargs *devargs =pci_addr_devargs_lookup (pci_addr); + +switch (rte_pci_bus.bus.conf.scan_mode) { +case RTE_BUS_SCAN_WHITELIST : +if (devargs && devargs->policy ==RTE_DEV_WHITELISTED ) +return false; +break ; +case RTE_BUS_SCAN_UNDEFINED : +case RTE_BUS_SCAN_BLACKLIST : +if (devargs ==NULL || + devargs->policy !=RTE_DEV_BLACKLISTED ) +return false; +break ; + } +return true; +} +static bool pci_ignore_device (const struct rte_pci_device *dev) {
drivers/bus/pci/private.h +12 -0
@@ -42,6 +42,18 @@ int rte_pci_scan(void);void pci_name_set (struct rte_pci_device *dev); +/** +* Validate whether a device with given pci address should be ignored or not. +* +* @param pci_addr +* PCI address of device to be validated +* @return +* 1: if device is to be ignored, +* 0: if device is to be scanned, +*/ +__rte_experimental +bool pci_addr_ignore_device(const struct rte_pci_addr *pci_addr); +/** * Add a PCI device to the PCI Bus (append to PCI Device list). This function * also updates the bus references of the PCI Device (and the generic device
drivers/bus/pci/rte_bus_pci_version.map +5 -0
@@ -16,3 +16,8 @@ DPDK_20.0 { local: *; }; + +EXPERIMENTAL { + # added in 20.05 + pci_addr_ignore_device; +};
Signature · 1 line
2.17.1
Re: [dpdk-dev] [PATCH v2 1/1] bus/pci: optimise scanning with whitelist/blacklist
Hello David and Stephen,
I have uploaded second version of patch as per your suggestion. Please have a look and ack if it looks okay.
Regards
Sunil Kumar Kori
Quoted 168 lines
-----Original Message-----
From: Sunil Kumar Kori <[email protected]>
Sent: Tuesday, April 7, 2020 2:59 PM
To: [email protected]; [email protected]; Jerin Jacob Kollanukkaran <[email protected]>
Cc: [email protected]; Sunil Kumar Kori <[email protected]>
Subject: [PATCH v2 1/1] bus/pci: optimise scanning with whitelist/blacklist
rte_bus_scanAPI scans all the available PCI devices irrespective of white or black listing parameters then further devices are probed based on white or black listing parameters. So unnecessary CPU cycles are wasted duringrte_pci_scan.For Octeontx2 platform with core frequency 2.4 Ghz,
rte_bus_scanconsumes around 26ms to scan around 90 PCI devices but all may not be used by the application. So for the application which uses 2 NICs,rte_bus_scanconsumes few microseconds and rest time is saved with this patch.Patch restricts devices to be scanned as per below mentioned conditions:
- All devices will be scanned if no parameters are passed.
- Only white listed devices will be scanned if white list is available.
- All devices, except black listed, will be scanned if black list is available.Signed-off-by Sunil Kumar Kori <[email protected]>---
v2:
- Added function to validate ignorance of device based on PCI address.
- Marked device validation function as experimental.drivers/bus/pci/bsd/pci.c | 15 ++++++++++- drivers/bus/pci/linux/pci.c | 3 +++ drivers/bus/pci/pci_common.c | 34 +++++++++++++++++++++++++ drivers/bus/pci/private.h | 12 +++++++++ drivers/bus/pci/rte_bus_pci_version.map | 5 ++++ 5 files changed, 68 insertions(+), 1 deletion(-)drivers/bus/pci/bsd/pci.c index renamed from drivers/bus/pci/bsd/pci.c +0
-0
ebbfeb13a..074f4556e100644drivers/bus/pci/bsd/pci.c +14
-1@@ -338,6 +338,9 @@ rte_pci_scan(void) .match_buf_len = sizeof(matches), .matches = &matches[0], }; +struct rte_pci_addr pci_addr; + +memset (&pci_addr,0 ,sizeof (struct rte_pci_addr ));/* for debug purposes, PCI can be disabled */ if (!rte_eal_has_pci ())@@ -357,9 +360,19 @@ rte_pci_scan(void) goto error; } -for (i =0 ; i < conf_io.num_matches; i++) +for (i =0 ; i < conf_io.num_matches; i++) { + pci_addr.domain = matches[i].pc_sel.pc_domain; + pci_addr.bus = matches[i].pc_sel.pc_bus; + pci_addr.devid = matches[i].pc_sel.pc_dev; + pci_addr.function = matches[i].pc_sel.pc_func; + +/* Check that device should be ignored or not */ +if (pci_addr_ignore_device (&pci_addr)) +continue ; +if (pci_scan_one (fd, &matches[i]) <0 ) goto error; + } dev_count += conf_io.num_matches; } while(conf_io.status == PCI_GETCONF_MORE_DEVS); diff --git
a/drivers/bus/pci/linux/pci.cb/drivers/bus/pci/linux/pci.cindex71b0a3053..92bdad826100644drivers/bus/pci/linux/pci.c +3
-0@@ -487,6 +487,9 @@ rte_pci_scan(void)if (parse_pci_addr_format (e->d_name,sizeof (e->d_name), &addr) !=0 )continue ; +if (pci_addr_ignore_device (&addr)) +continue ; +snprintf (dirname,sizeof (dirname),"%s/%s" ,rte_pci_get_sysfs_path (), e->d_name);drivers/bus/pci/pci_common.c +32
-0@@ -589,6 +589,40 @@ pci_dma_unmap(struct rte_device *dev, void *addr, uint64_t iova, size_t len) return -1; } +static struct rte_devargs * +pci_addr_devargs_lookup (const struct rte_pci_addr *pci_addr) { +struct rte_devargs *devargs; +struct rte_pci_addr addr; + + RTE_EAL_DEVARGS_FOREACH("pci", devargs) { + devargs->bus->parse (devargs->name, &addr); +if (!rte_pci_addr_cmp (pci_addr, &addr)) +return devargs; + } +return NULL ; +} + +bool +pci_addr_ignore_device (const struct rte_pci_addr *pci_addr) { +struct rte_devargs *devargs =pci_addr_devargs_lookup (pci_addr); + +switch (rte_pci_bus.bus.conf.scan_mode) { +case RTE_BUS_SCAN_WHITELIST : +if (devargs && devargs->policy ==RTE_DEV_WHITELISTED ) +return false; +break ; +case RTE_BUS_SCAN_UNDEFINED : +case RTE_BUS_SCAN_BLACKLIST : +if (devargs ==NULL || + devargs->policy !=RTE_DEV_BLACKLISTED ) +return false; +break ; + } +return true; +} +static bool
pci_ignore_device(const struct rte_pci_device *dev){ diff --gita/drivers/bus/pci/private.hb/drivers/bus/pci/private.hindexa205d4d9f..25075b806100644drivers/bus/pci/private.h +12
-0@@ -42,6 +42,18 @@ int rte_pci_scan(void); void pci_name_set(struct rte_pci_device *dev); +/** +* Validate whether a device with given pci address should be ignored or not. +* +* @param pci_addr +* PCI address of device to be validated +* @return +* 1: if device is to be ignored, +* 0: if device is to be scanned, +*/ +__rte_experimental +bool pci_addr_ignore_device(const struct rte_pci_addr *pci_addr); +/** * Add a PCI device to the PCI Bus (append to PCI Device list). This function * also updates the bus references of the PCI Device (and the generic device drivers/bus/pci/rte_bus_pci_version.map +5
-1@@ -16,3 +16,8 @@ DPDK_20.0 { local: *; }; + +EXPERIMENTAL { + # added in 20.05 + pci_addr_ignore_device; +}; --2.17.1
Re: [dpdk-dev] [PATCH v2 1/1] bus/pci: optimise scanning with whitelist/blacklist
In reply to Sunil Kumar KoriQuoted 17 lines · Sunil Kumar Kori
rte_bus_scanAPI scans all the available PCI devices irrespective of white or black listing parameters then further devices are probed based on white or black listing parameters. So unnecessary CPU cycles are wasted duringrte_pci_scan.For Octeontx2 platform with core frequency 2.4 Ghz,
rte_bus_scanconsumes around 26ms to scan around 90 PCI devices but all may not be used by the application. So for the application which uses 2 NICs,rte_bus_scanconsumes few microseconds and rest time is saved with this patch.Patch restricts devices to be scanned as per below mentioned conditions:
- All devices will be scanned if no parameters are passed.
- Only white listed devices will be scanned if white list is available.
- All devices, except black listed, will be scanned if black list is available.
Quoted 4 lines
Signed-off-by Sunil Kumar Kori <[email protected]>---
v2:
- Added function to validate ignorance of device based on PCI address.
First you objected to Stephen comment, and later announced there was no objection.
Now, it seems you ignored what I replied without any explanation. So tell me, what was wrong with
https://github.com/david-marchand/dpdk/commit/e7860231ecdce91f9f70027d4090a7057b8fd5f7
?
Quoted 1 line
- Marked device validation function as experimental.
Useless, this symbol is internal and not exported.
Signature · 1 line
David Marchand
Re: [dpdk-dev] [EXT] Re: [PATCH v2 1/1] bus/pci: optimise scanning with whitelist/blacklist
Hello David,
Answers inline.
Regards
Sunil Kumar Kori
Quoted 32 lines
-----Original Message-----
From: David Marchand <[email protected]>
Sent: Friday, April 17, 2020 2:14 PM
To: Sunil Kumar Kori <[email protected]>
Cc: Stephen Hemminger <[email protected]>; Jerin Jacob Kollanukkaran <[email protected]>; dev <[email protected]>
Subject: [EXT] Re: [PATCH v2 1/1] bus/pci: optimise scanning with whitelist/blacklistExternal Email
----------------------------------------------------------------------
Quoted 10 lines · Sunil Kumar Kori
rte_bus_scanAPI scans all the available PCI devices irrespective of white or black listing parameters then further devices are probed based on white or black listing parameters. So unnecessary CPU cycles are wasted duringrte_pci_scan.For Octeontx2 platform with core frequency 2.4 Ghz,
rte_bus_scanconsumes around 26ms to scan around 90 PCI devices but all may not be used by the application. So for the application which uses 2 NICs,rte_bus_scanconsumes few microseconds and rest time is saved with thispatch.
Quoted 7 lines
Patch restricts devices to be scanned as per below mentioned conditions:
- All devices will be scanned if no parameters are passed.
- Only white listed devices will be scanned if white list is available.
- All devices, except black listed, will be scanned if black list is available.
Quoted 7 lines
Quoted 4 lines
Signed-off-by Sunil Kumar Kori <[email protected]>---
v2:
- Added function to validate ignorance of device based on PCI address.First you objected to Stephen comment, and later announced there was no objection.
Please refer: https://patches.dpdk.org/patch/63924/. In first reply to Stephen's comment, Itself I said that I agree with that approach and why I have not taken that path is also mentioned, and requested for suggestions. But there were no further inputs after asking multiple times then I thought no one has any concern and then I asked to get it merged for 20.05.
Quoted 8 lines
Now, it seems you ignored what I replied without any explanation. So tell me, what was wrong with
https://urldefense.proofpoint.com/v2/url?u=https-3A__github.com_david-
2Dmarchand_dpdk_commit_e7860231ecdce91f9f70027d4090a7057b8fd5f7& d=DwIFaQ&c=nKjWec2b6R0mOyPaz7xtfQ&r=dXeXaAMkP5COgn1zxHMyaF1_d 9IIuq6vHQO6NrIPjaE&m=3nE0hIIwz2cXBpYrewLujeRWz5WPE7LB9j_HvOtBd68 &s=OjPCDnof_PNgATyzPIbjG8EtSYa5fE4EwbLD0oaIw5w&e=
No, Neither I have ignored your code changes nor denied. Both submitted patches uses similar approaches having one difference only that is you modified existing functions and I have written the new without touching the existing one. I have already explained in v1 that why I have not taken that path what you have implemented. Also I thought, its not good to change pci_ignore_device and pci_devargs_lookup because in future if more parameters (part of rte_pci_device structure) are considered to ignore a device then again we have to change this function to support it.
It may be a rare case but it was one thought process.
Quoted 10 lines
?
Quoted 1 line
- Marked device validation function as experimental.
Useless, this symbol is internal and not exported.
--
David Marchand
Re: [dpdk-dev] [EXT] Re: [PATCH v2 1/1] bus/pci: optimise scanning with whitelist/blacklist
Quoted 10 lines · Sunil Kumar Kori
Quoted 7 lines
Now, it seems you ignored what I replied without any explanation. So tell me, what was wrong with
https://urldefense.proofpoint.com/v2/url?u=https-3A__github.com_david-
2Dmarchand_dpdk_commit_e7860231ecdce91f9f70027d4090a7057b8fd5f7& d=DwIFaQ&c=nKjWec2b6R0mOyPaz7xtfQ&r=dXeXaAMkP5COgn1zxHMyaF1_d 9IIuq6vHQO6NrIPjaE&m=3nE0hIIwz2cXBpYrewLujeRWz5WPE7LB9j_HvOtBd68 &s=OjPCDnof_PNgATyzPIbjG8EtSYa5fE4EwbLD0oaIw5w&e=No, Neither I have ignored your code changes nor denied. Both submitted patches uses similar approaches having one difference only that is you modified existing functions and I have written the new without touching the existing one. I have already explained in v1 that why I have not taken that path what you have implemented. Also I thought, its not good to change
pci_ignore_deviceandpci_devargs_lookupbecause in future if more parameters (part ofrte_pci_devicestructure) are considered to ignore a device then again we have to change this function to support it.
It may be a rare case but it was one thought process.
Your current patch is a no go anyway.
The __rte_experimental tagging makes no sense.
Signature · 1 line
David Marchand
Re: [dpdk-dev] [EXT] Re: [PATCH v2 1/1] bus/pci: optimise scanning with whitelist/blacklist
Quoted 27 lines
-----Original Message-----
From: David Marchand <[email protected]>
Sent: Friday, April 17, 2020 6:56 PM
To: Sunil Kumar Kori <[email protected]>
Cc: Stephen Hemminger <[email protected]>; Jerin Jacob Kollanukkaran <[email protected]>; dev <[email protected]>
Subject: Re: [EXT] Re: [PATCH v2 1/1] bus/pci: optimise scanning with whitelist/blacklistQuoted 4 lines · Sunil Kumar Kori
Quoted 4 lines
Now, it seems you ignored what I replied without any explanation. So tell me, what was wrong with
https://urldefense.proofpoint.com/v2/url?u=https-3A__github.com_david
-2Dmarchand_dpdk_commit_e7860231ecdce91f9f70027d4090a7057b8fd5f7&
Quoted 2 lines
d=DwIFaQ&c=nKjWec2b6R0mOyPaz7xtfQ&r=dXeXaAMkP5COgn1zxHMyaF1_
d
Quoted 2 lines
9IIuq6vHQO6NrIPjaE&m=3nE0hIIwz2cXBpYrewLujeRWz5WPE7LB9j_HvOtBd6
8
Quoted 2 lines
Quoted 1 line
&s=OjPCDnof_PNgATyzPIbjG8EtSYa5fE4EwbLD0oaIw5w&e=
No, Neither I have ignored your code changes nor denied. Both submitted
patches uses similar approaches having one difference only that is you modified existing functions and I have written the new without touching the existing one. I have already explained in v1 that why I have not taken that path what you have implemented.
Quoted 8 lines
Quoted 1 line
Also I thought, its not good to change
pci_ignore_deviceand
pci_devargs_lookupbecause in future if more parameters (part ofrte_pci_devicestructure) are considered to ignore a device then again we have to change this function to support it.Quoted 1 line
It may be a rare case but it was one thought process.
Your current patch is a no go anyway.
The__rte_experimentaltagging makes no sense.
That’s not a problem. My only intention is to resolve a problem what we have observed and corresponding fix caters a practical use case.
If you don’t like the way it is implemented then we can go with your suggested way. Please let me know, will you be sending that github patch or should I send ?
Quoted 4 lines
--
David Marchand
Re: [dpdk-dev] [EXT] Re: [PATCH v2 1/1] bus/pci: optimise scanning with whitelist/blacklist
Quoted 7 lines · Sunil Kumar Kori
Quoted 2 lines
Your current patch is a no go anyway.
The__rte_experimentaltagging makes no sense.That’s not a problem. My only intention is to resolve a problem what we have observed and corresponding fix caters a practical use case.
If you don’t like the way it is implemented then we can go with your suggested way. Please let me know, will you be sending that github patch or should I send ?
And again.
The experimental tag is *useless*.private.h is not exported to users, the symbol you want to add is internal to the pci bus library.
I will stop at this.
Signature · 1 line
David Marchand
Re: [dpdk-dev] [EXT] Re: [PATCH v2 1/1] bus/pci: optimise scanning with whitelist/blacklist
Sent from Workspace ONE Boxer
Quoted 10 lines · David Marchand
Quoted 7 lines · Sunil Kumar Kori
Quoted 2 lines
Your current patch is a no go anyway.
The__rte_experimentaltagging makes no sense.That’s not a problem. My only intention is to resolve a problem what we have observed and corresponding fix caters a practical use case.
If you don’t like the way it is implemented then we can go with your suggested way. Please let me know, will you be sending that github patch or should I send ?
Quoted 7 lines
And again.
The experimental tag is *useless*.private.his not exported to users, the symbol you want to add is internal to the pci bus library.I will stop at this.
Got it. If it's only about exporting the symbol to the user then I will send next version of my patch after removing __rte_experimental and will make symbol to internal.
Quoted 4 lines
--
David Marchand
Re: [dpdk-dev] [EXT] Re: [PATCH v2 1/1] bus/pci: optimise scanning with whitelist/blacklist
Quoted 30 lines
-----Original Message-----
From: dev <[email protected]> On Behalf Of Sunil Kumar Kori
Sent: Friday, April 17, 2020 9:30 PM
To: David Marchand <[email protected]>
Cc: Jerin Jacob Kollanukkaran <[email protected]>; dev <[email protected]>; Stephen Hemminger <[email protected]>
Subject: Re: [dpdk-dev] [EXT] Re: [PATCH v2 1/1] bus/pci: optimise scanning with whitelist/blacklistSent from Workspace ONE Boxer
Quoted 2 lines · David Marchand
On Fri, Apr 17, 2020 at 5:12 PM Sunil Kumar Kori <[email protected]>
Quoted 6 lines · wrote:
Quoted 6 lines
Quoted 2 lines
Your current patch is a no go anyway.
The__rte_experimentaltagging makes no sense.That’s not a problem. My only intention is to resolve a problem what we have observed and corresponding fix caters a practical use case. If you don’t like the way it is implemented then we can go with your
suggested way.
Quoted 1 line
Quoted 1 line
Please let me know, will you be sending that github patch or should I send
?
Quoted 6 lines
And again.
The experimental tag is *useless*.private.his not exported to users, the symbol you want to add is internal to the pci bus library.
Quoted 5 lines
Quoted 2 lines
I will stop at this.
Got it. If it's only about exporting the symbol to the user then I will send next version of my patch after removing
__rte_experimentaland will make symbol to internal.
Uploaded v3.
Quoted 4 lines
Quoted 4 lines
--
David Marchand
PATCH v3 1/1 bus/pci: optimise scanning with whitelist/blacklist
In reply to Sunil Kumar Korirte_bus_scan API scans all the available PCI devices irrespective of white or black listing parameters then further devices are probed based on white or black listing parameters. So unnecessary CPU cycles are wasted during rte_pci_scan.
For Octeontx2 platform with core frequency 2.4 Ghz, rte_bus_scan consumes around 26ms to scan around 90 PCI devices but all may not be used by the application. So for the application which uses 2 NICs, rte_bus_scan consumes few microseconds and rest time is saved with this patch.
Patch restricts devices to be scanned as per below mentioned conditions:
- All devices will be scanned if no parameters are passed.
- Only white listed devices will be scanned if white list is available.
- All devices, except black listed, will be scanned if black list is available.
---
v3:
- remove __rte_experimental from private function.
- remove entry from map file too.
v2:
- Added function to validate ignorance of device based on PCI address.
- Marked device validation function as experimental.
drivers/bus/pci/bsd/pci.c | 13 ++++++++++++- drivers/bus/pci/linux/pci.c | 3 +++ drivers/bus/pci/pci_common.c | 34 ++++++++++++++++++++++++++++++++++ drivers/bus/pci/private.h | 11 +++++++++++ 4 files changed, 60 insertions(+), 1 deletion(-)
drivers/bus/pci/bsd/pci.c +12 -1
@@ -338,6 +338,7 @@ rte_pci_scan(void) .match_buf_len = sizeof(matches), .matches = &matches[0], }; +struct rte_pci_addr pci_addr;/* for debug purposes, PCI can be disabled */ if (!rte_eal_has_pci ())
@@ -357,9 +358,19 @@ rte_pci_scan(void) goto error; } -for (i =0 ; i < conf_io.num_matches; i++) +for (i =0 ; i < conf_io.num_matches; i++) { + pci_addr.domain = matches[i].pc_sel.pc_domain; + pci_addr.bus = matches[i].pc_sel.pc_bus; + pci_addr.devid = matches[i].pc_sel.pc_dev; + pci_addr.function = matches[i].pc_sel.pc_func; + +/* Check that device should be ignored or not */ +if (pci_addr_ignore_device (&pci_addr)) +continue ; +if (pci_scan_one (fd, &matches[i]) <0 ) goto error; + } dev_count += conf_io.num_matches; }while (conf_io.status ==PCI_GETCONF_MORE_DEVS );
drivers/bus/pci/linux/pci.c +3 -0
@@ -487,6 +487,9 @@ rte_pci_scan(void)if (parse_pci_addr_format (e->d_name,sizeof (e->d_name), &addr) !=0 )continue ; +if (pci_addr_ignore_device (&addr)) +continue ; +snprintf (dirname,sizeof (dirname),"%s/%s" ,rte_pci_get_sysfs_path (), e->d_name);
drivers/bus/pci/pci_common.c +34 -0
@@ -589,6 +589,40 @@ pci_dma_unmap(struct rte_device *dev, void *addr, uint64_t iova, size_t len)return -1 ; } +static struct rte_devargs * +pci_addr_devargs_lookup (const struct rte_pci_addr *pci_addr) +{ +struct rte_devargs *devargs; +struct rte_pci_addr addr; + + RTE_EAL_DEVARGS_FOREACH("pci", devargs) { + devargs->bus->parse (devargs->name, &addr); +if (!rte_pci_addr_cmp (pci_addr, &addr)) +return devargs; + } +return NULL ; +} + +bool +pci_addr_ignore_device (const struct rte_pci_addr *pci_addr) +{ +struct rte_devargs *devargs =pci_addr_devargs_lookup (pci_addr); + +switch (rte_pci_bus.bus.conf.scan_mode) { +case RTE_BUS_SCAN_WHITELIST : +if (devargs && devargs->policy ==RTE_DEV_WHITELISTED ) +return false; +break ; +case RTE_BUS_SCAN_UNDEFINED : +case RTE_BUS_SCAN_BLACKLIST : +if (devargs ==NULL || + devargs->policy !=RTE_DEV_BLACKLISTED ) +return false; +break ; + } +return true; +} +static bool pci_ignore_device (const struct rte_pci_device *dev) {
drivers/bus/pci/private.h +11 -0
@@ -42,6 +42,17 @@ int rte_pci_scan(void);void pci_name_set (struct rte_pci_device *dev); +/** +* Validate whether a device with given pci address should be ignored or not. +* +* @param pci_addr +* PCI address of device to be validated +* @return +* 1: if device is to be ignored, +* 0: if device is to be scanned, +*/ +bool pci_addr_ignore_device (const struct rte_pci_addr *pci_addr); +/** * Add a PCI device to the PCI Bus (append to PCI Device list). This function * also updates the bus references of the PCI Device (and the generic device
Signature · 1 line
2.17.1
Re: [dpdk-dev] [PATCH v3 1/1] bus/pci: optimise scanning with whitelist/blacklist
Quoted 10 lines · Sunil Kumar Kori
rte_bus_scanAPI scans all the available PCI devices irrespective of white or black listing parameters then further devices are probed based on white or black listing parameters. So unnecessary CPU cycles are wasted duringrte_pci_scan.For Octeontx2 platform with core frequency 2.4 Ghz,
rte_bus_scanconsumes around 26ms to scan around 90 PCI devices but all may not be used by the application. So for the application which uses 2 NICs,rte_bus_scanconsumes few microseconds and rest time is saved with this patch.
Hi Sunil,
The PCI bus was written at first with the understanding that all PCI devices were scanned and made available on the bus -- the probe will filter afterward.
Device hotplug and iteration were written with this in mind. Changing this principle might have unintended consequences in other EAL parts. I'm not fundamentally against it, but it is not how buses are currently designed in DPDK.
To me, a one-time 26ms gain is not enough justification to change this principle. How problematic is this for you? Do you encounter specific issues due to this delay?
Thanks,
Signature · 1 line
Gaëtan
Re: [dpdk-dev] [EXT] Re: [PATCH v3 1/1] bus/pci: optimise scanning with whitelist/blacklist
Quoted 27 lines
-----Original Message-----
From: Gaëtan Rivet <[email protected]>
Sent: Tuesday, April 21, 2020 8:48 PM
To: Sunil Kumar Kori <[email protected]>
Cc: [email protected]; [email protected]; Jerin Jacob Kollanukkaran <[email protected]>; [email protected]
Subject: [EXT] Re: [dpdk-dev] [PATCH v3 1/1] bus/pci: optimise scanning with whitelist/blacklistExternal Email
----------------------------------------------------------------------
Quoted 9 lines · Sunil Kumar Kori
rte_bus_scanAPI scans all the available PCI devices irrespective of white or black listing parameters then further devices are probed based on white or black listing parameters. So unnecessary CPU cycles are wasted duringrte_pci_scan.For Octeontx2 platform with core frequency 2.4 Ghz,
rte_bus_scanconsumes around 26ms to scan around 90 PCI devices but all may not be used by the application. So for the application which uses 2 NICs,rte_bus_scanconsumes few microseconds and rest time is saved with thispatch.
Quoted 1 line
Hi Sunil,
Quoted 8 lines
The PCI bus was written at first with the understanding that all PCI devices were scanned and made available on the bus -- the probe will filter afterward.
Device hotplug and iteration were written with this in mind. Changing this principle might have unintended consequences in other EAL parts. I'm not fundamentally against it, but it is not how buses are currently designed in DPDK.
I am also not sure about this. I would request you provide suggestion to ensure that there won't be any negative consequences if any. So that I can handle those too.
Quoted 5 lines
To me, a one-time 26ms gain is not enough justification to change this principle. How problematic is this for you? Do you encounter specific issues due to this delay?
Thanks,
Recently we observed this requirement to cater a use of having lowest bootup time for DPDK application. One of the use-case for this to reduce the downtime as part of DPDK SW upgrade in the field. i.e after the SW update, time to close the application and restart it again for packet processing. Having this solution application will be up soon and lesser traffic impact will be there in a deployed system.
Quoted 2 lines
--
Gaëtan
Re: [dpdk-dev] [EXT] Re: [PATCH v3 1/1] bus/pci: optimise scanning with whitelist/blacklist
Quoted 30 lines · Sunil Kumar Kori
Quoted 30 lines
-----Original Message-----
From: Gaëtan Rivet <[email protected]>
Sent: Tuesday, April 21, 2020 8:48 PM
To: Sunil Kumar Kori <[email protected]>
Cc: [email protected]; [email protected]; Jerin Jacob Kollanukkaran <[email protected]>; [email protected]
Subject: [EXT] Re: [dpdk-dev] [PATCH v3 1/1] bus/pci: optimise scanning with whitelist/blacklistExternal Email
----------------------------------------------------------------------
Quoted 9 lines · Sunil Kumar Kori
rte_bus_scanAPI scans all the available PCI devices irrespective of white or black listing parameters then further devices are probed based on white or black listing parameters. So unnecessary CPU cycles are wasted duringrte_pci_scan.For Octeontx2 platform with core frequency 2.4 Ghz,
rte_bus_scanconsumes around 26ms to scan around 90 PCI devices but all may not be used by the application. So for the application which uses 2 NICs,rte_bus_scanconsumes few microseconds and rest time is saved with thispatch.
Quoted 1 line
Hi Sunil,
The PCI bus was written at first with the understanding that all PCI devices were scanned and made available on the bus -- the probe will filter afterward.
Quoted 8 lines
Quoted 5 lines
Device hotplug and iteration were written with this in mind. Changing this principle might have unintended consequences in other EAL parts. I'm not fundamentally against it, but it is not how buses are currently designed in DPDK.
I am also not sure about this. I would request you provide suggestion to ensure that there won't be any negative consequences if any. So that I can handle those too.
I would like also to hear from other stakeholders for the PCI bus.
Generally, as long as the blacklist mode is the default, behavior should not change, but devil is in the details.
I would have some comments on the patch itself if everyone agrees with this direction.
If the principle of the patch is accepted, it would be great for you to test hotplug and device listing with testpmd:
hotplug:
* You can spawn VMs with virtual e1000 ports on PCI using QEMU for this, and within testpmd port attach <pci-id> -- of course, the port(s) should not be attached when starting testpmd. You might have to either blacklist them, or you could hotplug them in QEMU using the monitor. I don't recall the QEMU commands to do that, sorry.
device list:
* show device info all in testpmd. I thought I had added a command to test the device iterator, taking an arbitrary device string, but the patch has been dropped it seems.
If you have no segfault and no surprise, it is a good start.
Quoted 10 lines
Quoted 5 lines
To me, a one-time 26ms gain is not enough justification to change this principle. How problematic is this for you? Do you encounter specific issues due to this delay?
Thanks,
Recently we observed this requirement to cater a use of having lowest bootup time for DPDK application. One of the use-case for this to reduce the downtime as part of DPDK SW upgrade in the field. i.e after the SW update, time to close the application and restart it again for packet processing. Having this solution application will be up soon and lesser traffic impact will be there in a deployed system.
DPDK startup was not written with low latency in mind. You will find here and there minute improvements, but I think it is a pipedream to reduce service disruption on binary upgrade.
People in the field would be better served with HA, not relying on a critical apps restarting as fast as possible.
Cheers,
Signature · 1 line
Gaëtan
Re: [dpdk-dev] [EXT] Re: [PATCH v3 1/1] bus/pci: optimise scanning with whitelist/blacklist
Quoted 70 lines
-----Original Message-----
From: Gaëtan Rivet <[email protected]>
Sent: Wednesday, April 22, 2020 3:09 PM
To: Sunil Kumar Kori <[email protected]>
Cc: [email protected]; [email protected]; Jerin Jacob Kollanukkaran <[email protected]>; [email protected]
Subject: Re: [EXT] Re: [dpdk-dev] [PATCH v3 1/1] bus/pci: optimise scanning with whitelist/blacklistQuoted 23 lines · Sunil Kumar Kori
Quoted 23 lines
-----Original Message-----
From: Gaëtan Rivet <[email protected]>
Sent: Tuesday, April 21, 2020 8:48 PM
To: Sunil Kumar Kori <[email protected]>
Cc: [email protected]; [email protected]; Jerin Jacob Kollanukkaran <[email protected]>; [email protected]
Subject: [EXT] Re: [dpdk-dev] [PATCH v3 1/1] bus/pci: optimise scanning with whitelist/blacklistExternal Email
---------------------------------------------------------------------
Quoted 10 lines · Sunil Kumar Kori
rte_bus_scanAPI scans all the available PCI devices irrespective of white or black listing parameters then further devices are probed based on white or black listing parameters. So unnecessary CPU cycles are wasted duringrte_pci_scan.For Octeontx2 platform with core frequency 2.4 Ghz,
rte_bus_scanconsumes around 26ms to scan around 90 PCI devices but all may not be used by the application. So for the application which uses 2 NICs,rte_bus_scanconsumes few microseconds and rest time is saved with thisQuoted 7 lines
Quoted 7 lines
patch.
Quoted 1 line
Hi Sunil,
The PCI bus was written at first with the understanding that all PCI devices were scanned and made available on the bus -- the probe will filter
afterward.
Quoted 8 lines
Quoted 6 lines
Device hotplug and iteration were written with this in mind. Changing this principle might have unintended consequences in other EAL parts. I'm not fundamentally against it, but it is not how buses are currently designed in DPDK.
I am also not sure about this. I would request you provide suggestion to ensure that there won't be any negative consequences if any. So that I
can handle those too.
Quoted 1 line
I would like also to hear from other stakeholders for the PCI bus.
Generally, as long as the blacklist mode is the default, behavior should not change, but devil is in the details.
I would have some comments on the patch itself if everyone agrees with this direction.
If the principle of the patch is accepted, it would be great for you to test hotplug and device listing with testpmd:
hotplug:
* You can spawn VMs with virtual e1000 ports on PCI using QEMU for this, and within testpmdport attach <pci-id>-- of course, the port(s) should not be attached when starting testpmd. You might have to either blacklist them, or you could hotplug them in QEMU using the monitor. I don't recall the QEMU commands to do that, sorry.
Quoted 7 lines
device list:
*show device info allin testpmd. I thought I had added a command to test the device iterator, taking an arbitrary device string, but the patch has been dropped it seems.If you have no segfault and no surprise, it is a good start.
Okay but before verification I would appreciate to have more comments and closure on principle from other PCI stakeholders. If there is no objection on principle then I will invest energy in testing.
I will wait for inputs by next week and if there are no inputs then
assuming it fundamentally correct, I will start verification of above test cases.
Also If anyone has already validated above mentioned test cases then Suggestions, about the impact of this patch on PCI bus design, will be very helpful to understand the real issues with this.
Quoted 12 lines
Quoted 7 lines
Quoted 5 lines
To me, a one-time 26ms gain is not enough justification to change this principle. How problematic is this for you? Do you encounter specific issues due to this delay?
Thanks,
Recently we observed this requirement to cater a use of having lowest
bootup time for DPDK application.
Quoted 2 lines
One of the use-case for this to reduce the downtime as part of DPDK SW upgrade in the field. i.e after the SW update, time to close the application
and restart it again for packet processing.
Quoted 1 line
Having this solution application will be up soon and lesser traffic impact will
Quoted 9 lines
be there in a deployed system.
DPDK startup was not written with low latency in mind. You will find here and there minute improvements, but I think it is a pipedream to reduce service disruption on binary upgrade.
People in the field would be better served with HA, not relying on a critical apps restarting as fast as possible.
Recently we had a requirement to have bootup time <= 500ms and find it as one of the candidate to be improved. So thought of to upstream it. Also having mechanism to improve bootup time is good. I think, there is no harm in this.
Quoted 3 lines
Cheers,
--
Gaëtan
Re: [dpdk-dev] [PATCH v3 1/1] bus/pci: optimise scanning with whitelist/blacklist
In reply to Sunil Kumar KoriHello Sunil,
As it seems that this patch does not overly alarm people using the PCI bus, I have a few comments on this version. Sending those a little earlier will allow you to change as needed before proceeding with your tests.
Quoted 45 lines · Sunil Kumar Kori
rte_bus_scanAPI scans all the available PCI devices irrespective of white or black listing parameters then further devices are probed based on white or black listing parameters. So unnecessary CPU cycles are wasted duringrte_pci_scan.For Octeontx2 platform with core frequency 2.4 Ghz,
rte_bus_scanconsumes around 26ms to scan around 90 PCI devices but all may not be used by the application. So for the application which uses 2 NICs,rte_bus_scanconsumes few microseconds and rest time is saved with this patch.Patch restricts devices to be scanned as per below mentioned conditions:
- All devices will be scanned if no parameters are passed.
- Only white listed devices will be scanned if white list is available.
- All devices, except black listed, will be scanned if black list is available.Signed-off-by Sunil Kumar Kori <[email protected]>---
v3:
- remove__rte_experimentalfrom private function.
- remove entry from map file too.
v2:
- Added function to validate ignorance of device based on PCI address.
- Marked device validation function as experimental.drivers/bus/pci/bsd/pci.c | 13 ++++++++++++- drivers/bus/pci/linux/pci.c | 3 +++ drivers/bus/pci/pci_common.c | 34 ++++++++++++++++++++++++++++++++++ drivers/bus/pci/private.h | 11 +++++++++++ 4 files changed, 60 insertions(+), 1 deletion(-)drivers/bus/pci/bsd/pci.c +1
-0@@ -338,6 +338,7 @@ rte_pci_scan(void) .match_buf_len = sizeof(matches), .matches = &matches[0], }; +struct rte_pci_addr pci_addr;/* for debug purposes, PCI can be disabled */ if (!rte_eal_has_pci ())@@ -357,9 +358,19 @@ rte_pci_scan(void) goto error;
drivers/bus/pci/bsd/pci.c rte_pci_scan(void) · line 367
Quoted 10 lines
drivers/bus/pci/bsd/pci.c +7
-1} -for (i =0 ; i < conf_io.num_matches; i++) +for (i =0 ; i < conf_io.num_matches; i++) { + pci_addr.domain = matches[i].pc_sel.pc_domain; + pci_addr.bus = matches[i].pc_sel.pc_bus; + pci_addr.devid = matches[i].pc_sel.pc_dev; + pci_addr.function = matches[i].pc_sel.pc_func; + +/* Check that device should be ignored or not */
This comment is unnecessary, the function name should be sufficient to describe the check done.
drivers/bus/pci/bsd/pci.c rte_pci_scan(void) · line 370
Quoted 3 lines
drivers/bus/pci/bsd/pci.c +3
-0+if (pci_addr_ignore_device (&pci_addr)) +continue ; +
As this function is almost a copy of pci_ignore_device(), with a twist on the addr, I think the name pci_ignore_device_addr() would be more helpful.
I think in general however, that exposed symbols, even internals, should be prefixed with rte_. It was (almost) ok for pci_ignore_device() to forego the namespace as it is a static function that will be mangled on export, but that won't be the case for your function.
Please add rte_ prefix.
Quoted 26 lines
drivers/bus/pci/bsd/pci.c +1
-0if (pci_scan_one (fd, &matches[i]) <0 ) goto error; + } dev_count += conf_io.num_matches; }while (conf_io.status ==PCI_GETCONF_MORE_DEVS );drivers/bus/pci/linux/pci.c +3
-0@@ -487,6 +487,9 @@ rte_pci_scan(void)if (parse_pci_addr_format (e->d_name,sizeof (e->d_name), &addr) !=0 )continue ; +if (pci_addr_ignore_device (&addr)) +continue ; +snprintf (dirname,sizeof (dirname),"%s/%s" ,rte_pci_get_sysfs_path (), e->d_name);drivers/bus/pci/pci_common.c +0
-0@@ -589,6 +589,40 @@ pci_dma_unmap(struct rte_device *dev, void *addr, uint64_t iova, size_t len)return -1 ;
drivers/bus/pci/pci_common.c pci_dma_unmap(struct rte_device *dev, void *addr, uint64_t iova, size_t len) · line 599
Quoted 10 lines
drivers/bus/pci/pci_common.c +8
-0} +static struct rte_devargs * +pci_addr_devargs_lookup (const struct rte_pci_addr *pci_addr) +{ +struct rte_devargs *devargs; +struct rte_pci_addr addr; + + RTE_EAL_DEVARGS_FOREACH("pci", devargs) { + devargs->bus->parse (devargs->name, &addr);
Why not use rte_pci_addr_parse directly there? The bus->parse() API, while stable, is one-level of indirection removed from what's done, it's simpler for the reader to see the intent by using the proper function.
Return value should be checked. If the devargs name is not parseable, there are other issues at hand (memory corruption), we should skip the device as well or crash, but not proceed with comparison.
Quoted 19 lines
drivers/bus/pci/pci_common.c +19
-0+if (!rte_pci_addr_cmp (pci_addr, &addr)) +return devargs; + } +return NULL ; +} + +bool +pci_addr_ignore_device (const struct rte_pci_addr *pci_addr) +{ +struct rte_devargs *devargs =pci_addr_devargs_lookup (pci_addr); + +switch (rte_pci_bus.bus.conf.scan_mode) { +case RTE_BUS_SCAN_WHITELIST : +if (devargs && devargs->policy ==RTE_DEV_WHITELISTED ) +return false; +break ; +case RTE_BUS_SCAN_UNDEFINED : +case RTE_BUS_SCAN_BLACKLIST : + if (devargs == NULL ||
drivers/bus/pci/pci_common.c pci_dma_unmap(struct rte_device *dev, void *addr, uint64_t iova, size_t len) · line 628
Quoted 10 lines
drivers/bus/pci/pci_common.c +7
-0+ devargs->policy != RTE_DEV_BLACKLISTED) +return false; +break ; + } +return true; +} +static bool pci_ignore_device (const struct rte_pci_device *dev) {
The logic seems ok to me.
However, the logic is the same as the one in rte_pci_probe(). During probe, any device on the bus would have already been vetted during scan. It should be ok to probe all existing rte_pci_device.
The same argument can be made for rte_pci_get_iommu_class() then, no need to use pci_ignore_device(). It is done after the scan() so it should be ok.
And if pci_ignore_device() can be completely removed, then you should rename your function from rte_pci_ignore_device_addr() to rte_pci_ignore_device() altogether.
Quoted 24 lines
drivers/bus/pci/private.h +11
-0@@ -42,6 +42,17 @@ int rte_pci_scan(void);void pci_name_set (struct rte_pci_device *dev); +/** +* Validate whether a device with given pci address should be ignored or not. +* +* @param pci_addr +* PCI address of device to be validated +* @return +* 1: if device is to be ignored, +* 0: if device is to be scanned, +*/ +bool pci_addr_ignore_device (const struct rte_pci_addr *pci_addr); +/** * Add a PCI device to the PCI Bus (append to PCI Device list). This function * also updates the bus references of the PCI Device (and the generic device Signature · 1 line
2.17.1
Best regards,
Signature · 1 line
Gaëtan
Re: [dpdk-dev] [EXT] Re: [PATCH v3 1/1] bus/pci: optimise scanning with whitelist/blacklist
Quoted 69 lines
-----Original Message-----
From: Gaëtan Rivet <[email protected]>
Sent: Tuesday, April 28, 2020 12:14 AM
To: Sunil Kumar Kori <[email protected]>
Cc: [email protected]; [email protected]; Jerin Jacob Kollanukkaran <[email protected]>; [email protected]
Subject: [EXT] Re: [dpdk-dev] [PATCH v3 1/1] bus/pci: optimise scanning with whitelist/blacklistExternal Email
----------------------------------------------------------------------
Hello Sunil,As it seems that this patch does not overly alarm people using the PCI bus, I have a few comments on this version. Sending those a little earlier will allow you to change as needed before proceeding with your tests.
Quoted 6 lines · Sunil Kumar Kori
rte_bus_scanAPI scans all the available PCI devices irrespective of white or black listing parameters then further devices are probed based on white or black listing parameters. So unnecessary CPU cycles are wasted duringrte_pci_scan.For Octeontx2 platform with core frequency 2.4 Ghz,
rte_bus_scanconsumes
Quoted 19 lines
around 26ms to scan around 90 PCI devices but all may not be used by the application. So for the application which uses 2 NICs,
rte_bus_scanconsumes few microseconds and rest time is saved with this patch.Patch restricts devices to be scanned as per below mentioned conditions:
- All devices will be scanned if no parameters are passed.
- Only white listed devices will be scanned if white list is available.
- All devices, except black listed, will be scanned if black list is available.Signed-off-by Sunil Kumar Kori <[email protected]>---
v3:
- remove__rte_experimentalfrom private function.
- remove entry from map file too.
v2:
- Added function to validate ignorance of device based on PCI address.
- Marked device validation function as experimental.Quoted 3 lines
drivers/bus/pci/bsd/pci.c | 13 ++++++++++++- drivers/bus/pci/linux/pci.c | 3 +++ drivers/bus/pci/pci_common.c | 34++++++++++++++++++++++++++++++++++
Quoted 19 lines
drivers/bus/pci/private.h | 11 +++++++++++ 4 files changed, 60 insertions(+), 1 deletion(-)drivers/bus/pci/bsd/pci.c +1
-0@@ -338,6 +338,7 @@ rte_pci_scan(void) .match_buf_len = sizeof(matches), .matches = &matches[0], }; +struct rte_pci_addr pci_addr;/* for debug purposes, PCI can be disabled */ if (!rte_eal_has_pci ())@@ -357,9 +358,19 @@ rte_pci_scan(void) goto error; }
Quoted 12 lines
drivers/bus/pci/bsd/pci.c rte_pci_scan(void) · line 367
Quoted 8 lines
drivers/bus/pci/bsd/pci.c +7
-1-for (i =0 ; i < conf_io.num_matches; i++) +for (i =0 ; i < conf_io.num_matches; i++) { + pci_addr.domain = matches[i].pc_sel.pc_domain; + pci_addr.bus = matches[i].pc_sel.pc_bus; + pci_addr.devid = matches[i].pc_sel.pc_dev; + pci_addr.function = matches[i].pc_sel.pc_func; + +/* Check that device should be ignored or not */ This comment is unnecessary, the function name should be sufficient to describe the check done.
Ack
Quoted 8 lines
drivers/bus/pci/bsd/pci.c rte_pci_scan(void) · line 370
Quoted 3 lines
drivers/bus/pci/bsd/pci.c +3
-0+if (pci_addr_ignore_device (&pci_addr)) +continue ; +As this function is almost a copy of
pci_ignore_device(), with a twist on the addr, I think the namepci_ignore_device_addr()would be more helpful.
Quoted 8 lines
I think in general however, that exposed symbols, even internals, should be prefixed with
rte_. It was (almost) ok forpci_ignore_device()to forego the namespace as it is a static function that will be mangled on export, but that won't be the case for your function.Please add
rte_prefix.
Ack
Quoted 43 lines
drivers/bus/pci/linux/pci.c rte_pci_scan(void) · line 487
Quoted 12 lines
drivers/bus/pci/bsd/pci.c +1
-0if (pci_scan_one (fd, &matches[i]) <0 ) goto error; + } dev_count += conf_io.num_matches; }while (conf_io.status ==PCI_GETCONF_MORE_DEVS );drivers/bus/pci/linux/pci.c +0
-0@@ -487,6 +487,9 @@ rte_pci_scan(void) if (parse_pci_addr_format(e->d_name, sizeof(e->d_name),&addr) != 0)
drivers/bus/pci/pci_common.c pci_dma_unmap(struct rte_device *dev, void
Quoted 13 lines
drivers/bus/pci/linux/pci.c +3
-0continue ; +if (pci_addr_ignore_device (&addr)) +continue ; +snprintf (dirname,sizeof (dirname),"%s/%s" ,rte_pci_get_sysfs_path (), e->d_name);drivers/bus/pci/pci_common.c +0
-0@@ -589,6 +589,40 @@ pci_dma_unmap(struct rte_device *dev, void*addr,
uint64_tiova,size_tlen)drivers/bus/pci/pci_common.c pci_dma_unmap(struct rte_device *dev, void *addr, uint64_t iova, size_t len) · line 599
Quoted 11 lines
drivers/bus/pci/pci_common.c +8
-0return -1 ; } +static struct rte_devargs * +pci_addr_devargs_lookup (const struct rte_pci_addr *pci_addr) +{ +struct rte_devargs *devargs; +struct rte_pci_addr addr; + + RTE_EAL_DEVARGS_FOREACH("pci", devargs) { + devargs->bus->parse (devargs->name, &addr);Why not use
rte_pci_addr_parsedirectly there? Thebus->parse()API, while stable, is one-level of indirection removed from what's done, it's simpler for the reader to see the intent by using the proper function.
Quoted 4 lines
Return value should be checked. If the devargs name is not parseable, there are other issues at hand (memory corruption), we should skip the device as well or crash, but not proceed with comparison.
Ack
Quoted 36 lines
Quoted 19 lines
drivers/bus/pci/pci_common.c +19
-0+if (!rte_pci_addr_cmp (pci_addr, &addr)) +return devargs; + } +return NULL ; +} + +bool +pci_addr_ignore_device (const struct rte_pci_addr *pci_addr) +{ +struct rte_devargs *devargs =pci_addr_devargs_lookup (pci_addr); + +switch (rte_pci_bus.bus.conf.scan_mode) { +case RTE_BUS_SCAN_WHITELIST : +if (devargs && devargs->policy ==RTE_DEV_WHITELISTED ) +return false; +break ; +case RTE_BUS_SCAN_UNDEFINED : +case RTE_BUS_SCAN_BLACKLIST : + if (devargs == NULL ||drivers/bus/pci/pci_common.c pci_dma_unmap(struct rte_device *dev, void *addr, uint64_t iova, size_t len) · line 628
Quoted 10 lines
drivers/bus/pci/pci_common.c +7
-0+ devargs->policy != RTE_DEV_BLACKLISTED) +return false; +break ; + } +return true; +} +static bool pci_ignore_device (const struct rte_pci_device *dev) {The logic seems ok to me.
However, the logic is the same as the one in
rte_pci_probe(). During probe, any device on the bus would have already been vetted during scan. It should be ok to probe all existingrte_pci_device.
Quoted 8 lines
The same argument can be made for
rte_pci_get_iommu_class()then, no need to usepci_ignore_device(). It is done after thescan()so it should be ok.And if
pci_ignore_device()can be completely removed, then you should rename your function fromrte_pci_ignore_device_addr()torte_pci_ignore_device()altogether.
Ack
Quoted 29 lines
drivers/bus/pci/private.h int rte_pci_scan(void); · line 46
Quoted 10 lines
drivers/bus/pci/private.h +2
-0@@ -42,6 +42,17 @@ int rte_pci_scan(void);void pci_name_set (struct rte_pci_device *dev); +/** +* Validate whether a device with given pci address should be ignored or not.
Quoted 13 lines
drivers/bus/pci/private.h +9
-1+* +* @param pci_addr +* PCI address of device to be validated +* @return +* 1: if device is to be ignored, +* 0: if device is to be scanned, +*/ +bool pci_addr_ignore_device (const struct rte_pci_addr *pci_addr); +/** * Add a PCI device to the PCI Bus (append to PCI Device list). This function * also updates the bus references of the PCI Device (and the generic device -- Quoted 1 line
2.17.1
Best regards,
--
Gaëtan
PATCH v4 1/1 bus/pci: optimise scanning with whitelist/blacklist
In reply to Sunil Kumar Korirte_bus_scan API scans all the available PCI devices irrespective of white or black listing parameters then further devices are probed based on white or black listing parameters. So unnecessary CPU cycles are wasted during rte_pci_scan.
For Octeontx2 platform with core frequency 2.4 Ghz, rte_bus_scan consumes around 26ms to scan around 90 PCI devices but all may not be used by the application. So for the application which uses 2 NICs, rte_bus_scan consumes few microseconds and rest time is saved with this patch.
Patch restricts devices to be scanned as per below mentioned conditions:
- All devices will be scanned if no parameters are passed.
- Only white listed devices will be scanned if white list is available.
- All devices, except black listed, will be scanned if black list is available.
---
v4:
- Review comments incorporated (Gaeten and David).
- Rebased on top of tree.
v3:
- Remove __rte_experimental from private function.
- Remove entry from map file too.
v2:
- Added function to validate ignorance of device based on PCI address.
- Marked device validation function as experimental.
drivers/bus/pci/bsd/pci.c | 12 +++++++++++- drivers/bus/pci/linux/pci.c | 3 +++ drivers/bus/pci/pci_common.c | 33 ++++++++++++--------------------- drivers/bus/pci/private.h | 11 +++++++++++ 4 files changed, 37 insertions(+), 22 deletions(-)
drivers/bus/pci/bsd/pci.c +11 -1
@@ -338,6 +338,7 @@ rte_pci_scan(void) .match_buf_len = sizeof(matches), .matches = &matches[0], }; +struct rte_pci_addr pci_addr;/* for debug purposes, PCI can be disabled */ if (!rte_eal_has_pci ())
@@ -357,9 +358,18 @@ rte_pci_scan(void) goto error; } -for (i =0 ; i < conf_io.num_matches; i++) +for (i =0 ; i < conf_io.num_matches; i++) { + pci_addr.domain = matches[i].pc_sel.pc_domain; + pci_addr.bus = matches[i].pc_sel.pc_bus; + pci_addr.devid = matches[i].pc_sel.pc_dev; + pci_addr.function = matches[i].pc_sel.pc_func; + +if (rte_pci_ignore_device_addr (&pci_addr)) +continue ; +if (pci_scan_one (fd, &matches[i]) <0 ) goto error; + } dev_count += conf_io.num_matches; }while (conf_io.status ==PCI_GETCONF_MORE_DEVS );
drivers/bus/pci/linux/pci.c +3 -0
@@ -487,6 +487,9 @@ rte_pci_scan(void)if (parse_pci_addr_format (e->d_name,sizeof (e->d_name), &addr) !=0 )continue ; +if (rte_pci_ignore_device_addr (&addr)) +continue ; +snprintf (dirname,sizeof (dirname),"%s/%s" ,rte_pci_get_sysfs_path (), e->d_name);
drivers/bus/pci/pci_common.c +12 -21
@@ -42,14 +42,17 @@ const char *rte_pci_get_sysfs_path(void)return path; } -static struct rte_devargs *pci_devargs_lookup (struct rte_pci_device *dev) +static struct rte_devargs * +pci_devargs_lookup (const struct rte_pci_addr *pci_addr) {struct rte_devargs *devargs;struct rte_pci_addr addr; RTE_EAL_DEVARGS_FOREACH("pci", devargs) { - devargs->bus->parse (devargs->name, &addr); -if (!rte_pci_addr_cmp (&dev->addr, &addr)) +if (rte_pci_addr_parse (devargs->name, &addr) <0 ) +continue ; + +if (!rte_pci_addr_cmp (pci_addr, &addr))return devargs; }return NULL ;
@@ -63,7 +66,7 @@ pci_name_set(struct rte_pci_device *dev)/* Each device has its internal, canonical name set. */ rte_pci_device_name (&dev->addr, dev->name,sizeof (dev->name)); - devargs =pci_devargs_lookup (dev); + devargs =pci_devargs_lookup (&dev->addr); dev->device.devargs = devargs;/* In blacklist mode, if the device is not blacklisted, no * rte_devargs exists for it.
@@ -293,23 +296,12 @@ rte_pci_probe(void) {struct rte_pci_device *dev =NULL ;size_t probed =0 , failed =0 ; -struct rte_devargs *devargs; -int probe_all =0 ;int ret =0 ; -if (rte_pci_bus.bus.conf.scan_mode !=RTE_BUS_SCAN_WHITELIST ) - probe_all =1 ; -FOREACH_DEVICE_ON_PCIBUS (dev) { probed++; - devargs = dev->device.devargs; -/* probe all or only whitelisted devices */ -if (probe_all) - ret =pci_probe_all_drivers (dev); -else if (devargs !=NULL && - devargs->policy ==RTE_DEV_WHITELISTED ) - ret =pci_probe_all_drivers (dev); + ret =pci_probe_all_drivers (dev);if (ret <0 ) {if (ret != -EEXIST ) { RTE_LOG(ERR, EAL, "Requested device "
@@ -589,10 +581,10 @@ pci_dma_unmap(struct rte_device *dev, void *addr, uint64_t iova, size_t len)return -1 ; } -static bool -pci_ignore_device (const struct rte_pci_device *dev) +bool +rte_pci_ignore_device_addr (const struct rte_pci_addr *pci_addr) { -struct rte_devargs *devargs = dev->device.devargs; +struct rte_devargs *devargs =pci_devargs_lookup (pci_addr);switch (rte_pci_bus.bus.conf.scan_mode) {case RTE_BUS_SCAN_WHITELIST :
@@ -627,8 +619,7 @@ rte_pci_get_iommu_class(void)if (iommu_no_va ==-1 ) iommu_no_va =pci_device_iommu_support_va (dev) ?0 :1 ; -if (pci_ignore_device (dev)) -continue ; +if (dev->kdrv ==RTE_KDRV_UNKNOWN || dev->kdrv ==RTE_KDRV_NONE )continue ;
drivers/bus/pci/private.h +11 -0
@@ -42,6 +42,17 @@ int rte_pci_scan(void);void pci_name_set (struct rte_pci_device *dev); +/** +* Validate whether a device with given pci address should be ignored or not. +* +* @param pci_addr +* PCI address of device to be validated +* @return +* 1: if device is to be ignored, +* 0: if device is to be scanned, +*/ +bool rte_pci_ignore_device_addr (const struct rte_pci_addr *pci_addr); +/** * Add a PCI device to the PCI Bus (append to PCI Device list). This function * also updates the bus references of the PCI Device (and the generic device
Signature · 1 line
2.17.1
Re: [dpdk-dev] [PATCH v4 1/1] bus/pci: optimise scanning with whitelist/blacklist
Hello Gaeten,
I have validated hotplug functionality using multi process hotplug application. Devices are being attached and detached at runtime properly.
Regards
Sunil Kumar Kori
Quoted 42 lines
-----Original Message-----
From: Sunil Kumar Kori <[email protected]>
Sent: Friday, May 1, 2020 5:09 PM
To: [email protected]; [email protected]; Jerin Jacob Kollanukkaran <[email protected]>; [email protected]
Cc: [email protected]; Sunil Kumar Kori <[email protected]>
Subject: [PATCH v4 1/1] bus/pci: optimise scanning with whitelist/blacklist
rte_bus_scanAPI scans all the available PCI devices irrespective of white or black listing parameters then further devices are probed based on white or black listing parameters. So unnecessary CPU cycles are wasted duringrte_pci_scan.For Octeontx2 platform with core frequency 2.4 Ghz,
rte_bus_scanconsumes around 26ms to scan around 90 PCI devices but all may not be used by the application. So for the application which uses 2 NICs,rte_bus_scanconsumes few microseconds and rest time is saved with this patch.Patch restricts devices to be scanned as per below mentioned conditions:
- All devices will be scanned if no parameters are passed.
- Only white listed devices will be scanned if white list is available.
- All devices, except black listed, will be scanned if black list is available.Signed-off-by Sunil Kumar Kori <[email protected]>---
v4:
- Review comments incorporated (Gaeten and David).
- Rebased on top of tree.
v3:
- Remove__rte_experimentalfrom private function.
- Remove entry from map file too.
v2:
- Added function to validate ignorance of device based on PCI address.
- Marked device validation function as experimental.drivers/bus/pci/bsd/pci.c | 12 +++++++++++- drivers/bus/pci/linux/pci.c | 3 +++ drivers/bus/pci/pci_common.c | 33 ++++++++++++--------------------- drivers/bus/pci/private.h | 11 +++++++++++ 4 files changed, 37 insertions(+), 22 deletions(-)
[snip]
Re: [dpdk-dev] [PATCH v4 1/1] bus/pci: optimise scanning with whitelist/blacklist
In reply to Sunil Kumar KoriHello Sunil,
It's pretty close, thanks. Unfortunately, I just have a few nits remaining.
Quoted 92 lines · Sunil Kumar Kori
rte_bus_scanAPI scans all the available PCI devices irrespective of white or black listing parameters then further devices are probed based on white or black listing parameters. So unnecessary CPU cycles are wasted duringrte_pci_scan.For Octeontx2 platform with core frequency 2.4 Ghz,
rte_bus_scanconsumes around 26ms to scan around 90 PCI devices but all may not be used by the application. So for the application which uses 2 NICs,rte_bus_scanconsumes few microseconds and rest time is saved with this patch.Patch restricts devices to be scanned as per below mentioned conditions:
- All devices will be scanned if no parameters are passed.
- Only white listed devices will be scanned if white list is available.
- All devices, except black listed, will be scanned if black list is available.Signed-off-by Sunil Kumar Kori <[email protected]>---
v4:
- Review comments incorporated (Gaeten and David).
- Rebased on top of tree.
v3:
- Remove__rte_experimentalfrom private function.
- Remove entry from map file too.
v2:
- Added function to validate ignorance of device based on PCI address.
- Marked device validation function as experimental.drivers/bus/pci/bsd/pci.c | 12 +++++++++++- drivers/bus/pci/linux/pci.c | 3 +++ drivers/bus/pci/pci_common.c | 33 ++++++++++++--------------------- drivers/bus/pci/private.h | 11 +++++++++++ 4 files changed, 37 insertions(+), 22 deletions(-)drivers/bus/pci/bsd/pci.c +11
-1@@ -338,6 +338,7 @@ rte_pci_scan(void) .match_buf_len = sizeof(matches), .matches = &matches[0], }; +struct rte_pci_addr pci_addr;/* for debug purposes, PCI can be disabled */ if (!rte_eal_has_pci ())@@ -357,9 +358,18 @@ rte_pci_scan(void) goto error; } -for (i =0 ; i < conf_io.num_matches; i++) +for (i =0 ; i < conf_io.num_matches; i++) { + pci_addr.domain = matches[i].pc_sel.pc_domain; + pci_addr.bus = matches[i].pc_sel.pc_bus; + pci_addr.devid = matches[i].pc_sel.pc_dev; + pci_addr.function = matches[i].pc_sel.pc_func; + +if (rte_pci_ignore_device_addr (&pci_addr)) +continue ; +if (pci_scan_one (fd, &matches[i]) <0 ) goto error; + } dev_count += conf_io.num_matches; }while (conf_io.status ==PCI_GETCONF_MORE_DEVS );drivers/bus/pci/linux/pci.c +3
-0@@ -487,6 +487,9 @@ rte_pci_scan(void)if (parse_pci_addr_format (e->d_name,sizeof (e->d_name), &addr) !=0 )continue ; +if (rte_pci_ignore_device_addr (&addr)) +continue ; +snprintf (dirname,sizeof (dirname),"%s/%s" ,rte_pci_get_sysfs_path (), e->d_name);drivers/bus/pci/pci_common.c +2
-1@@ -42,14 +42,17 @@ const char *rte_pci_get_sysfs_path(void)return path; } -static struct rte_devargs *pci_devargs_lookup (struct rte_pci_device *dev) +static struct rte_devargs * +pci_devargs_lookup (const struct rte_pci_addr *pci_addr) {
drivers/bus/pci/pci_common.c const char *rte_pci_get_sysfs_path(void) · line 55
Quoted 10 lines
drivers/bus/pci/pci_common.c +4
-2struct rte_devargs *devargs;struct rte_pci_addr addr; RTE_EAL_DEVARGS_FOREACH("pci", devargs) { - devargs->bus->parse (devargs->name, &addr); -if (!rte_pci_addr_cmp (&dev->addr, &addr)) +if (rte_pci_addr_parse (devargs->name, &addr) <0 ) +continue ; + +if (!rte_pci_addr_cmp (pci_addr, &addr))
I'm really sorry, I was overzealous on your v4. I thought using devargs->bus->parse() was from your patch, but it was already there.
It's even more shameful as it was me who wrote this.
Anyway, it's much better looking this way, but it should be a separate patch.
Quoted 38 lines
drivers/bus/pci/pci_common.c +2
-13return devargs; }return NULL ;@@ -63,7 +66,7 @@ pci_name_set(struct rte_pci_device *dev)/* Each device has its internal, canonical name set. */ rte_pci_device_name (&dev->addr, dev->name,sizeof (dev->name)); - devargs =pci_devargs_lookup (dev); + devargs =pci_devargs_lookup (&dev->addr); dev->device.devargs = devargs;/* In blacklist mode, if the device is not blacklisted, no * rte_devargs exists for it. @@ -293,23 +296,12 @@ rte_pci_probe(void) {struct rte_pci_device *dev =NULL ;size_t probed =0 , failed =0 ; -struct rte_devargs *devargs; -int probe_all =0 ;int ret =0 ; -if (rte_pci_bus.bus.conf.scan_mode !=RTE_BUS_SCAN_WHITELIST ) - probe_all =1 ; -FOREACH_DEVICE_ON_PCIBUS (dev) { probed++; - devargs = dev->device.devargs; -/* probe all or only whitelisted devices */ -if (probe_all) - ret =pci_probe_all_drivers (dev); -else if (devargs !=NULL && - devargs->policy ==RTE_DEV_WHITELISTED ) - ret =pci_probe_all_drivers (dev); + ret =pci_probe_all_drivers (dev);if (ret <0 ) {if (ret != -EEXIST ) { RTE_LOG(ERR, EAL, "Requested device "
drivers/bus/pci/pci_common.c pci_dma_unmap(struct rte_device *dev, void *addr, uint64_t iova, size_t len) · line 585
Quoted 7 lines
drivers/bus/pci/pci_common.c +2
-2@@ -589,10 +581,10 @@ pci_dma_unmap(struct rte_device *dev, void *addr, uint64_t iova, size_t len)return -1 ; } -static bool -pci_ignore_device (const struct rte_pci_device *dev) +bool +rte_pci_ignore_device_addr (const struct rte_pci_addr *pci_addr)
I'd prefer naming this function rte_pci_ignore_device(), given that pci_ignore_device() disappears.
Quoted 41 lines
drivers/bus/pci/pci_common.c +2
-3{ -struct rte_devargs *devargs = dev->device.devargs; +struct rte_devargs *devargs =pci_devargs_lookup (pci_addr);switch (rte_pci_bus.bus.conf.scan_mode) {case RTE_BUS_SCAN_WHITELIST :@@ -627,8 +619,7 @@ rte_pci_get_iommu_class(void)if (iommu_no_va ==-1 ) iommu_no_va =pci_device_iommu_support_va (dev) ?0 :1 ; -if (pci_ignore_device (dev)) -continue ; +if (dev->kdrv ==RTE_KDRV_UNKNOWN || dev->kdrv ==RTE_KDRV_NONE )continue ;drivers/bus/pci/private.h +11
-0@@ -42,6 +42,17 @@ int rte_pci_scan(void);void pci_name_set (struct rte_pci_device *dev); +/** +* Validate whether a device with given pci address should be ignored or not. +* +* @param pci_addr +* PCI address of device to be validated +* @return +* 1: if device is to be ignored, +* 0: if device is to be scanned, +*/ +bool rte_pci_ignore_device_addr (const struct rte_pci_addr *pci_addr); +/** * Add a PCI device to the PCI Bus (append to PCI Device list). This function * also updates the bus references of the PCI Device (and the generic device Signature · 1 line
2.17.1
Otherwise it looks good to me, almost finished!
Signature · 1 line
Gaëtan
Re: [dpdk-dev] [EXT] Re: [PATCH v4 1/1] bus/pci: optimise scanning with whitelist/blacklist
Quoted 126 lines
-----Original Message-----
From: Gaëtan Rivet <[email protected]>
Sent: Saturday, May 2, 2020 2:30 AM
To: Sunil Kumar Kori <[email protected]>
Cc: [email protected]; [email protected]; Jerin Jacob Kollanukkaran <[email protected]>; [email protected]
Subject: [EXT] Re: [PATCH v4 1/1] bus/pci: optimise scanning with whitelist/blacklistExternal Email
----------------------------------------------------------------------
Hello Sunil,It's pretty close, thanks. Unfortunately, I just have a few nits remaining.
Quoted 9 lines · Sunil Kumar Kori
rte_bus_scanAPI scans all the available PCI devices irrespective of white or black listing parameters then further devices are probed based on white or black listing parameters. So unnecessary CPU cycles are wasted duringrte_pci_scan.For Octeontx2 platform with core frequency 2.4 Ghz,
rte_bus_scanconsumes around 26ms to scan around 90 PCI devices but all may not be used by the application. So for the application which uses 2 NICs,rte_bus_scanconsumes few microseconds and rest time is saved with thispatch.
Quoted 62 lines
Patch restricts devices to be scanned as per below mentioned conditions:
- All devices will be scanned if no parameters are passed.
- Only white listed devices will be scanned if white list is available.
- All devices, except black listed, will be scanned if black list is available.Signed-off-by Sunil Kumar Kori <[email protected]>---
v4:
- Review comments incorporated (Gaeten and David).
- Rebased on top of tree.
v3:
- Remove__rte_experimentalfrom private function.
- Remove entry from map file too.
v2:
- Added function to validate ignorance of device based on PCI address.
- Marked device validation function as experimental.drivers/bus/pci/bsd/pci.c | 12 +++++++++++- drivers/bus/pci/linux/pci.c | 3 +++ drivers/bus/pci/pci_common.c | 33 ++++++++++++--------------------- drivers/bus/pci/private.h | 11 +++++++++++ 4 files changed, 37 insertions(+), 22 deletions(-)drivers/bus/pci/bsd/pci.c +11
-1@@ -338,6 +338,7 @@ rte_pci_scan(void) .match_buf_len = sizeof(matches), .matches = &matches[0], }; +struct rte_pci_addr pci_addr;/* for debug purposes, PCI can be disabled */ if (!rte_eal_has_pci ())@@ -357,9 +358,18 @@ rte_pci_scan(void) goto error; } -for (i =0 ; i < conf_io.num_matches; i++) +for (i =0 ; i < conf_io.num_matches; i++) { + pci_addr.domain = matches[i].pc_sel.pc_domain; + pci_addr.bus = matches[i].pc_sel.pc_bus; + pci_addr.devid = matches[i].pc_sel.pc_dev; + pci_addr.function = matches[i].pc_sel.pc_func; + +if (rte_pci_ignore_device_addr (&pci_addr)) +continue ; +if (pci_scan_one (fd, &matches[i]) <0 ) goto error; + } dev_count += conf_io.num_matches; } while(conf_io.status == PCI_GETCONF_MORE_DEVS); diff --git
a/drivers/bus/pci/linux/pci.cb/drivers/bus/pci/linux/pci.cindexca783b157..ec347eff3100644drivers/bus/pci/linux/pci.c rte_pci_scan(void) · line 487
Quoted 1 line
drivers/bus/pci/linux/pci.c +0
-0@@ -487,6 +487,9 @@ rte_pci_scan(void) if (parse_pci_addr_format(e->d_name, sizeof(e->d_name),&addr) != 0)
Quoted 21 lines
drivers/bus/pci/linux/pci.c +3
-0continue ; +if (rte_pci_ignore_device_addr (&addr)) +continue ; +snprintf (dirname,sizeof (dirname),"%s/%s" ,rte_pci_get_sysfs_path (), e->d_name);(quoted diff) +0
-0
b/drivers/bus/pci/pci_common.cindex3f5542076..d34e59536100644drivers/bus/pci/pci_common.c +2
-1@@ -42,14 +42,17 @@ const char *rte_pci_get_sysfs_path(void)return path; } -static struct rte_devargs *pci_devargs_lookup (struct rte_pci_device *dev) +static struct rte_devargs * +pci_devargs_lookup (const struct rte_pci_addr *pci_addr) {drivers/bus/pci/pci_common.c const char *rte_pci_get_sysfs_path(void) · line 56
Quoted 10 lines
drivers/bus/pci/pci_common.c +4
-2struct rte_devargs *devargs;struct rte_pci_addr addr; RTE_EAL_DEVARGS_FOREACH("pci", devargs) { - devargs->bus->parse (devargs->name, &addr); -if (!rte_pci_addr_cmp (&dev->addr, &addr)) +if (rte_pci_addr_parse (devargs->name, &addr) <0 ) +continue ; + +if (!rte_pci_addr_cmp (pci_addr, &addr))I'm really sorry, I was overzealous on your v4. I thought using
devargs->bus->parse()was from your patch, but it was already there.
Quoted 4 lines
It's even more shameful as it was me who wrote this.
Anyway, it's much better looking this way, but it should be a separate patch.
No problem. I will revert this change to its original state and later will share separate Patch to replace devargs->bus->parse.
Quoted 41 lines
drivers/bus/pci/pci_common.c pci_dma_unmap(struct rte_device *dev, void
Quoted 37 lines
drivers/bus/pci/pci_common.c +0
-0return devargs; }drivers/bus/pci/pci_common.c +2
-13return NULL ;@@ -63,7 +66,7 @@ pci_name_set(struct rte_pci_device *dev)/* Each device has its internal, canonical name set. */ rte_pci_device_name (&dev->addr, dev->name,sizeof (dev->name)); - devargs =pci_devargs_lookup (dev); + devargs =pci_devargs_lookup (&dev->addr); dev->device.devargs = devargs;/* In blacklist mode, if the device is not blacklisted, no * rte_devargs exists for it. @@ -293,23 +296,12 @@ rte_pci_probe(void) {struct rte_pci_device *dev =NULL ;size_t probed =0 , failed =0 ; -struct rte_devargs *devargs; -int probe_all =0 ;int ret =0 ; -if (rte_pci_bus.bus.conf.scan_mode !=RTE_BUS_SCAN_WHITELIST ) - probe_all =1 ; -FOREACH_DEVICE_ON_PCIBUS (dev) { probed++; - devargs = dev->device.devargs; -/* probe all or only whitelisted devices */ -if (probe_all) - ret =pci_probe_all_drivers (dev); -else if (devargs !=NULL && - devargs->policy ==RTE_DEV_WHITELISTED ) - ret =pci_probe_all_drivers (dev); + ret =pci_probe_all_drivers (dev);if (ret <0 ) {if (ret != -EEXIST ) { RTE_LOG(ERR, EAL, "Requested device "@@ -589,10 +581,10 @@ pci_dma_unmap(struct rte_device *dev, void*addr,
uint64_tiova,size_tlen)Quoted 3 lines
drivers/bus/pci/pci_common.c +0
-0return -1 ; }
Quoted 8 lines
drivers/bus/pci/pci_common.c pci_dma_unmap(struct rte_device *dev, void *addr, uint64_t iova, size_t len) · line 585
Quoted 4 lines
drivers/bus/pci/pci_common.c +2
-2-static bool -pci_ignore_device (const struct rte_pci_device *dev) +bool +rte_pci_ignore_device_addr (const struct rte_pci_addr *pci_addr)I'd prefer naming this function
rte_pci_ignore_device(), given thatpci_ignore_device()disappears.
Ack.
Quoted 47 lines
Quoted 21 lines
drivers/bus/pci/pci_common.c +2
-3{ -struct rte_devargs *devargs = dev->device.devargs; +struct rte_devargs *devargs =pci_devargs_lookup (pci_addr);switch (rte_pci_bus.bus.conf.scan_mode) {case RTE_BUS_SCAN_WHITELIST :@@ -627,8 +619,7 @@ rte_pci_get_iommu_class(void)if (iommu_no_va ==-1 ) iommu_no_va =pci_device_iommu_support_va (dev) ?0 :1 ; -if (pci_ignore_device (dev)) -continue ; +if (dev->kdrv ==RTE_KDRV_UNKNOWN || dev->kdrv ==RTE_KDRV_NONE )continue ;drivers/bus/pci/private.h +0
-0drivers/bus/pci/private.h int rte_pci_scan(void); void pci_name_set(struct · line 45
Quoted 4 lines
drivers/bus/pci/private.h +2
-0@@ -42,6 +42,17 @@ int rte_pci_scan(void); void pci_name_set(struct rte_pci_device *dev); +/** +* Validate whether a device with given pci address should be ignored or not.
Quoted 12 lines
drivers/bus/pci/private.h +9
-0+* +* @param pci_addr +* PCI address of device to be validated +* @return +* 1: if device is to be ignored, +* 0: if device is to be scanned, +*/ +bool rte_pci_ignore_device_addr (const struct rte_pci_addr *pci_addr); +/** * Add a PCI device to the PCI Bus (append to PCI Device list). This function * also updates the bus references of the PCI Device (and the generic Quoted 4 lines
device
--
2.17.1Otherwise it looks good to me, almost finished!
--
Gaëtan
PATCH v5 1/1 bus/pci: optimise scanning with whitelist/blacklist
In reply to Sunil Kumar Korirte_bus_scan API scans all the available PCI devices irrespective of white or black listing parameters then further devices are probed based on white or black listing parameters. So unnecessary CPU cycles are wasted during rte_pci_scan.
For Octeontx2 platform with core frequency 2.4 Ghz, rte_bus_scan consumes around 26ms to scan around 90 PCI devices but all may not be used by the application. So for the application which uses 2 NICs, rte_bus_scan consumes few microseconds and rest time is saved with this patch.
Patch restricts devices to be scanned as per below mentioned conditions:
- All devices will be scanned if no parameters are passed.
- Only white listed devices will be scanned if white list is available.
- All devices, except black listed, will be scanned if black list is available.
---
v5:
- revert devargs->bus->parse changes.
v4:
- Review comments incorporated (Gaeten and David).
- Rebased on top of tree.
v3:
- Remove __rte_experimental from private function.
- Remove entry from map file too.
v2:
- Added function to validate ignorance of device based on PCI address.
- Marked device validation function as experimental.
drivers/bus/pci/bsd/pci.c | 12 +++++++++++- drivers/bus/pci/linux/pci.c | 3 +++ drivers/bus/pci/pci_common.c | 29 +++++++++-------------------- drivers/bus/pci/private.h | 11 +++++++++++ 4 files changed, 34 insertions(+), 21 deletions(-)
drivers/bus/pci/bsd/pci.c +11 -1
@@ -338,6 +338,7 @@ rte_pci_scan(void) .match_buf_len = sizeof(matches), .matches = &matches[0], }; +struct rte_pci_addr pci_addr;/* for debug purposes, PCI can be disabled */ if (!rte_eal_has_pci ())
@@ -357,9 +358,18 @@ rte_pci_scan(void) goto error; } -for (i =0 ; i < conf_io.num_matches; i++) +for (i =0 ; i < conf_io.num_matches; i++) { + pci_addr.domain = matches[i].pc_sel.pc_domain; + pci_addr.bus = matches[i].pc_sel.pc_bus; + pci_addr.devid = matches[i].pc_sel.pc_dev; + pci_addr.function = matches[i].pc_sel.pc_func; + +if (rte_pci_ignore_device (&pci_addr)) +continue ; +if (pci_scan_one (fd, &matches[i]) <0 ) goto error; + } dev_count += conf_io.num_matches; }while (conf_io.status ==PCI_GETCONF_MORE_DEVS );
drivers/bus/pci/linux/pci.c +3 -0
@@ -487,6 +487,9 @@ rte_pci_scan(void)if (parse_pci_addr_format (e->d_name,sizeof (e->d_name), &addr) !=0 )continue ; +if (rte_pci_ignore_device (&addr)) +continue ; +snprintf (dirname,sizeof (dirname),"%s/%s" ,rte_pci_get_sysfs_path (), e->d_name);
drivers/bus/pci/pci_common.c +9 -20
@@ -42,14 +42,15 @@ const char *rte_pci_get_sysfs_path(void)return path; } -static struct rte_devargs *pci_devargs_lookup (struct rte_pci_device *dev) +static struct rte_devargs * +pci_devargs_lookup (const struct rte_pci_addr *pci_addr) {struct rte_devargs *devargs;struct rte_pci_addr addr; RTE_EAL_DEVARGS_FOREACH("pci", devargs) { devargs->bus->parse (devargs->name, &addr); -if (!rte_pci_addr_cmp (&dev->addr, &addr)) +if (!rte_pci_addr_cmp (pci_addr, &addr))return devargs; }return NULL ;
@@ -63,7 +64,7 @@ pci_name_set(struct rte_pci_device *dev)/* Each device has its internal, canonical name set. */ rte_pci_device_name (&dev->addr, dev->name,sizeof (dev->name)); - devargs =pci_devargs_lookup (dev); + devargs =pci_devargs_lookup (&dev->addr); dev->device.devargs = devargs;/* In blacklist mode, if the device is not blacklisted, no * rte_devargs exists for it.
@@ -293,23 +294,12 @@ rte_pci_probe(void) {struct rte_pci_device *dev =NULL ;size_t probed =0 , failed =0 ; -struct rte_devargs *devargs; -int probe_all =0 ;int ret =0 ; -if (rte_pci_bus.bus.conf.scan_mode !=RTE_BUS_SCAN_WHITELIST ) - probe_all =1 ; -FOREACH_DEVICE_ON_PCIBUS (dev) { probed++; - devargs = dev->device.devargs; -/* probe all or only whitelisted devices */ -if (probe_all) - ret =pci_probe_all_drivers (dev); -else if (devargs !=NULL && - devargs->policy ==RTE_DEV_WHITELISTED ) - ret =pci_probe_all_drivers (dev); + ret =pci_probe_all_drivers (dev);if (ret <0 ) {if (ret != -EEXIST ) { RTE_LOG(ERR, EAL, "Requested device "
@@ -589,10 +579,10 @@ pci_dma_unmap(struct rte_device *dev, void *addr, uint64_t iova, size_t len)return -1 ; } -static bool -pci_ignore_device (const struct rte_pci_device *dev) +bool +rte_pci_ignore_device (const struct rte_pci_addr *pci_addr) { -struct rte_devargs *devargs = dev->device.devargs; +struct rte_devargs *devargs =pci_devargs_lookup (pci_addr);switch (rte_pci_bus.bus.conf.scan_mode) {case RTE_BUS_SCAN_WHITELIST :
@@ -627,8 +617,7 @@ rte_pci_get_iommu_class(void)if (iommu_no_va ==-1 ) iommu_no_va =pci_device_iommu_support_va (dev) ?0 :1 ; -if (pci_ignore_device (dev)) -continue ; +if (dev->kdrv ==RTE_KDRV_UNKNOWN || dev->kdrv ==RTE_KDRV_NONE )continue ;
drivers/bus/pci/private.h +11 -0
@@ -42,6 +42,17 @@ int rte_pci_scan(void);void pci_name_set (struct rte_pci_device *dev); +/** +* Validate whether a device with given pci address should be ignored or not. +* +* @param pci_addr +* PCI address of device to be validated +* @return +* 1: if device is to be ignored, +* 0: if device is to be scanned, +*/ +bool rte_pci_ignore_device (const struct rte_pci_addr *pci_addr); +/** * Add a PCI device to the PCI Bus (append to PCI Device list). This function * also updates the bus references of the PCI Device (and the generic device
Signature · 1 line
2.17.1
Re: [dpdk-dev] [PATCH v5 1/1] bus/pci: optimise scanning with whitelist/blacklist
Quoted 10 lines · Sunil Kumar Kori
rte_bus_scanAPI scans all the available PCI devices irrespective of white or black listing parameters then further devices are probed based on white or black listing parameters. So unnecessary CPU cycles are wasted duringrte_pci_scan.For Octeontx2 platform with core frequency 2.4 Ghz,
rte_bus_scanconsumes around 26ms to scan around 90 PCI devices but all may not be used by the application. So for the application which uses 2 NICs,rte_bus_scanconsumes few microseconds and rest time is saved with this patch.
Quoted 7 lines
Patch restricts devices to be scanned as per below mentioned conditions:
- All devices will be scanned if no parameters are passed.
- Only white listed devices will be scanned if white list is available.
- All devices, except black listed, will be scanned if black list is available.Signed-off-by Sunil Kumar Kori <[email protected]>
LGTM,
Signature · 1 line
Gaëtan
Re: [dpdk-dev] [PATCH v5 1/1] bus/pci: optimise scanning with whitelist/blacklist
In reply to Sunil Kumar KoriQuoted 6 lines · Sunil Kumar Kori
drivers/bus/pci/private.h +0
-0@@ -42,6 +42,17 @@ int rte_pci_scan(void);void
drivers/bus/pci/private.h int rte_pci_scan(void); · line 52
Quoted 10 lines
drivers/bus/pci/private.h +8
-0pci_name_set(struct rte_pci_device *dev); +/** +* Validate whether a device with given pci address should be ignored or not. +* +* @param pci_addr +* PCI address of device to be validated +* @return +* 1: if device is to be ignored, +* 0: if device is to be scanned,
true/false
drivers/bus/pci/private.h int rte_pci_scan(void); · line 54
Quoted 2 lines
drivers/bus/pci/private.h +2
-0+*/ +bool rte_pci_ignore_device (const struct rte_pci_addr *pci_addr);
Gaetan, private API must not be prefixed with rte_, this is for public APIs only.
I noticed inconsistencies in the pci bus some time ago, I will fix this patch for now and send some followup patches I had in store.
drivers/bus/pci/private.h int rte_pci_scan(void); · line 58
Quoted 4 lines
drivers/bus/pci/private.h +1
-0+/** * Add a PCI device to the PCI Bus (append to PCI Device list). This function * also updates the bus references of the PCI Device (and the generic device
Sunil, no need to send a new version, I will fix this when applying.
Thanks.
Signature · 1 line
David Marchand
Re: [dpdk-dev] [EXT] Re: [PATCH v5 1/1] bus/pci: optimise scanning with whitelist/blacklist
Quoted 42 lines
-----Original Message-----
From: David Marchand <[email protected]>
Sent: Monday, May 4, 2020 7:47 PM
To: Sunil Kumar Kori <[email protected]>; Gaetan Rivet <[email protected]>
Cc: Stephen Hemminger <[email protected]>; Jerin Jacob Kollanukkaran <[email protected]>; dev <[email protected]>
Subject: [EXT] Re: [PATCH v5 1/1] bus/pci: optimise scanning with whitelist/blacklistExternal Email
----------------------------------------------------------------------
drivers/bus/pci/private.h int rte_pci_scan(void); void pci_name_set(struct · line 45
Quoted 9 lines · Sunil Kumar Kori
drivers/bus/pci/private.h +2
-0@@ -42,6 +42,17 @@ int rte_pci_scan(void); void pci_name_set(struct rte_pci_device *dev); +/** +* Validate whether a device with given pci address should be ignored or not.
drivers/bus/pci/private.h int rte_pci_scan(void); · line 52
Quoted 6 lines
drivers/bus/pci/private.h +6
-0+* +* @param pci_addr +* PCI address of device to be validated +* @return +* 1: if device is to be ignored, +* 0: if device is to be scanned, true/false
drivers/bus/pci/private.h int rte_pci_scan(void); · line 54
Quoted 2 lines
drivers/bus/pci/private.h +2
-0+*/ +bool rte_pci_ignore_device (const struct rte_pci_addr *pci_addr);Gaetan, private API must not be prefixed with
rte_, this is for public APIs only. I noticed inconsistencies in the pci bus some time ago, I will fix this patch for now and send some followup patches I had in store.
Quoted 9 lines
Quoted 5 lines
drivers/bus/pci/private.h +1
-0+/** * Add a PCI device to the PCI Bus (append to PCI Device list). This function * also updates the bus references of the PCI Device (and the generic device
Sunil, no need to send a new version, I will fix this when applying.
Ack.
Quoted 4 lines
Thanks.
--
David Marchand
Re: [dpdk-dev] [PATCH v5 1/1] bus/pci: optimise scanning with whitelist/blacklist
In reply to David MarchandQuoted 4 lines · David Marchand
On Sat, May 2, 2020 at 9:42 AM Sunil Kumar Kori <[email protected]> wrote:
I noticed inconsistencies in the pci bus some time ago, I will fix this patch for now and send some followup patches I had in store.
I only sent the minimum, there are other changes like hiding internal structures that could go later.
https://patchwork.dpdk.org/project/dpdk/list/?series=9874
Signature · 1 line
David Marchand
Re: [dpdk-dev] [PATCH v5 1/1] bus/pci: optimise scanning with whitelist/blacklist
In reply to Sunil Kumar KoriQuoted 11 lines · Sunil Kumar Kori
rte_bus_scanAPI scans all the available PCI devices irrespective of white or black listing parameters then further devices are probed based on white or black listing parameters. So unnecessary CPU cycles are wasted duringrte_pci_scan.For Octeontx2 platform with core frequency 2.4 Ghz,
rte_bus_scanconsumes around 26ms to scan around 90 PCI devices but all may not be used by the application. So for the application which uses 2 NICs,rte_bus_scanconsumes few microseconds and rest time is saved with this patch.
Quoted 7 lines
Patch restricts devices to be scanned as per below mentioned conditions:
- All devices will be scanned if no parameters are passed.
- Only white listed devices will be scanned if white list is available.
- All devices, except black listed, will be scanned if black list is available.Signed-off-by Sunil Kumar Kori <[email protected]>
drivers/bus/pci/private.h int rte_pci_scan(void); · line 46
Quoted 10 lines
drivers/bus/pci/private.h +2
-0@@ -42,6 +42,17 @@ int rte_pci_scan(void);void pci_name_set (struct rte_pci_device *dev); +/** +* Validate whether a device with given pci address should be ignored or not.
PCI address
Quoted 6 lines
drivers/bus/pci/private.h +6
-0+* +* @param pci_addr +* PCI address of device to be validated +* @return +* 1: if device is to be ignored, +* 0: if device is to be scanned,
Quoted 6 lines · Sunil Kumar Kori
drivers/bus/pci/private.h +0
-0@@ -42,6 +42,17 @@ int rte_pci_scan(void);void
drivers/bus/pci/private.h int rte_pci_scan(void); · line 52
Quoted 10 lines
drivers/bus/pci/private.h +8
-0pci_name_set(struct rte_pci_device *dev); +/** +* Validate whether a device with given pci address should be ignored or not. +* +* @param pci_addr +* PCI address of device to be validated +* @return +* 1: if device is to be ignored, +* 0: if device is to be scanned,
true/false
drivers/bus/pci/private.h int rte_pci_scan(void); · line 58
Quoted 6 lines
drivers/bus/pci/private.h +3
-0+*/ +bool rte_pci_ignore_device (const struct rte_pci_addr *pci_addr); +/** * Add a PCI device to the PCI Bus (append to PCI Device list). This function * also updates the bus references of the PCI Device (and the generic device
Applied, thanks.
Signature · 1 line
David Marchand