bus/pci: avoid depending on private value in kernel source
RFC PATCH 1/1 bus/pci: avoid depending on private value in kernel source
The value 40 used in VFIO_GET_REGION_ADDR() is a private value (VFIO_PCI_OFFSET_SHIFT) defined in Linux kernel source [1]. It is not part of VFIO API, and we should not depend on it.
[1] https://github.com/torvalds/linux/blob/6fbc7275c7a9/drivers/vfio/pci/vfio_pci_private.h#L19
--- drivers/bus/pci/linux/pci.c | 18 +++- drivers/bus/pci/linux/pci_init.h | 6 +- drivers/bus/pci/linux/pci_vfio.c | 157 ++++++++++++++++++++++++------- drivers/bus/pci/rte_bus_pci.h | 12 +++ lib/librte_eal/linux/eal/eal.c | 17 ++-- 5 files changed, 163 insertions(+), 47 deletions(-)
drivers/bus/pci/linux/pci.c +16 -2
@@ -337,6 +337,20 @@ pci_scan_one(const char *dirname, const struct rte_pci_addr *addr) }else dev->kdrv =RTE_KDRV_NONE ; +switch (dev->kdrv) { +case RTE_KDRV_VFIO : +#ifdef VFIO_PRESENT + ret =pci_vfio_fill_regions (dev); +if (ret !=0 ) { +free (dev); +return 1 ; + } +#endif +break ; +default : +break ; + } +/* device is valid, add in list (sorted) */ if (TAILQ_EMPTY (&rte_pci_bus.device_list)) {rte_pci_add_device (dev);
@@ -721,7 +735,7 @@ int rte_pci_read_config(const struct rte_pci_device *device,return pci_uio_read_config (intr_handle, buf, len, offset);#ifdef VFIO_PRESENT case RTE_KDRV_VFIO : -return pci_vfio_read_config (intr_handle, buf, len, offset); +return pci_vfio_read_config (device, buf, len, offset);#endif default : rte_pci_device_name(&device->addr, devname,
@@ -745,7 +759,7 @@ int rte_pci_write_config(const struct rte_pci_device *device,return pci_uio_write_config (intr_handle, buf, len, offset);#ifdef VFIO_PRESENT case RTE_KDRV_VFIO : -return pci_vfio_write_config (intr_handle, buf, len, offset); +return pci_vfio_write_config (device, buf, len, offset);#endif default : rte_pci_device_name(&device->addr, devname,
drivers/bus/pci/linux/pci_init.h +4 -2
@@ -64,9 +64,9 @@ int pci_uio_ioport_unmap(struct rte_pci_ioport *p);#endif /* access config space */ -int pci_vfio_read_config (const struct rte_intr_handle *intr_handle, +int pci_vfio_read_config (const struct rte_pci_device *dev,void *buf,size_t len,off_t offs); -int pci_vfio_write_config (const struct rte_intr_handle *intr_handle, +int pci_vfio_write_config (const struct rte_pci_device *dev,const void *buf,size_t len,off_t offs); int pci_vfio_ioport_map(struct rte_pci_device *dev, int bar,
@@ -83,6 +83,8 @@ int pci_vfio_unmap_resource(struct rte_pci_device *dev);int pci_vfio_is_enabled (void ); +int pci_vfio_fill_regions (struct rte_pci_device *dev); +#endif #endif /* EAL_PCI_INIT_H_ */
drivers/bus/pci/linux/pci_vfio.c +122 -35
270 lines not on this page · raw message
drivers/bus/pci/rte_bus_pci.h +12 -0
@@ -51,6 +51,16 @@ TAILQ_HEAD(rte_pci_driver_list, rte_pci_driver);struct rte_devargs ; +#define RTE_MAX_PCI_REGIONS 9 + +/* +* Regions provided by e.g. VFIO. +*/ +struct rte_pci_region { +uint64_t size; +uint64_t offset; +}; +/** * A structure describing a PCI device. */
@@ -59,6 +69,8 @@ struct rte_pci_device {struct rte_device device;/**< Inherit core device */ struct rte_pci_addr addr;/**< PCI location. */ struct rte_pci_id id;/**< PCI ID. */ +struct rte_pci_region region[RTE_MAX_PCI_REGIONS ]; +/**< PCI regions. */ struct rte_mem_resource mem_resource[PCI_MAX_RESOURCE ];/**< PCI Memory Resource */ struct rte_intr_handle intr_handle;/**< Interrupt handle */
lib/librte_eal/linux/eal/eal.c +9 -8
@@ -1068,6 +1068,15 @@ rte_eal_init(int argc, char **argv)return -1 ; } +#ifdef VFIO_PRESENT +if (rte_eal_vfio_setup () <0 ) { +rte_eal_init_alert ("Cannot init VFIO" ); + rte_errno =EAGAIN ; +rte_atomic32_clear (&run_once); +return -1 ; + } +#endif +if (rte_bus_scan ()) {rte_eal_init_alert ("Cannot scan the buses for devices" ); rte_errno =ENODEV ;
@@ -1130,14 +1139,6 @@ rte_eal_init(int argc, char **argv)return -1 ; } -#ifdef VFIO_PRESENT -if (rte_eal_vfio_setup () <0 ) { -rte_eal_init_alert ("Cannot init VFIO" ); - rte_errno =EAGAIN ; -rte_atomic32_clear (&run_once); -return -1 ; - } -#endif /* in secondary processes, memory init may allocate additional fbarrays * not present in primary processes, so to avoid any potential issues, * initialize memzones first.
Signature · 1 line
2.17.1
Hello,
Quoted 8 lines · Tiwei Bie
The value 40 used in
VFIO_GET_REGION_ADDR()is a private value (VFIO_PCI_OFFSET_SHIFT) defined in Linux kernel source [1]. It is not part of VFIO API, and we should not depend on it.[1]
https://github.com/torvalds/linux/blob/6fbc7275c7a9/drivers/vfio/pci/vfio_pci_private.h#L19
I did not follow linux kernel changes, is there something that would change this offset?
It looks like a cleanup (did not look into the details yet), do we need this now?
Signature · 1 line
David Marchand
Hi David,
Quoted 13 lines · David Marchand
Hello,
On Wed, Jul 3, 2019 at 7:47 AM Tiwei Bie <[email protected]> wrote:
The value 40 used in
VFIO_GET_REGION_ADDR()is a private value (VFIO_PCI_OFFSET_SHIFT) defined in Linux kernel source [1]. It is not part of VFIO API, and we should not depend on it.[1] https://github.com/torvalds/linux/blob/6fbc7275c7a9/drivers/vfio/pci/
vfio_pci_private.h#L19
Quoted 4 lines
I did not follow linux kernel changes, is there something that would change this offset?
It looks like a cleanup (did not look into the details yet), do we need this now?
In VFIO/mdev [1], the offset can be something different. It depends on the parent device. It's not just a cleanup. It's a preparation for the mdev support in DPDK.
[1] https://github.com/torvalds/linux/blob/master/Documentation/vfio-mediated-device.txt
Thanks,
Tiwei
Quoted 3 lines
--
David Marchand
Quoted 22 lines · Tiwei Bie
Hi David,
Quoted 9 lines · David Marchand
Hello,
On Wed, Jul 3, 2019 at 7:47 AM Tiwei Bie <[email protected]> wrote:
The value 40 used in
VFIO_GET_REGION_ADDR()is a private value (VFIO_PCI_OFFSET_SHIFT) defined in Linux kernel source [1]. It is not part of VFIO API, and we should not depend on it.[1]
https://github.com/torvalds/linux/blob/6fbc7275c7a9/drivers/vfio/pci/
Quoted 5 lines
vfio_pci_private.h#L19I did not follow linux kernel changes, is there something that would
change
Quoted 2 lines
this offset?
It looks like a cleanup (did not look into the details yet), do we needthis
Quoted 10 lines
Quoted 1 line
now?
In VFIO/mdev [1], the offset can be something different. It depends on the parent device. It's not just a cleanup. It's a preparation for the mdev support in DPDK.
[1]
https://github.com/torvalds/linux/blob/master/Documentation/vfio-mediated-device.txt
Ok, thanks.
So we can wait for mdev to be ready before working on this.
Signature · 1 line
David Marchand
Quoted 35 lines · David Marchand
On Wed, Jul 3, 2019 at 9:35 AM Tiwei Bie <[email protected]> wrote:
Hi David, On Wed, Jul 03, 2019 at 09:02:59AM +0200, David Marchand wrote: > Hello, > > On Wed, Jul 3, 2019 at 7:47 AM Tiwei Bie <[email protected]> wrote: > > The value 40 used in VFIO_GET_REGION_ADDR() is a private value > (VFIO_PCI_OFFSET_SHIFT) defined in Linux kernel source [1]. It > is not part of VFIO API, and we should not depend on it. > > [1] https://github.com/torvalds/linux/blob/6fbc7275c7a9/drivers/vfio/ pci/ > vfio_pci_private.h#L19 > > > > I did not follow linux kernel changes, is there something that would change > this offset? > It looks like a cleanup (did not look into the details yet), do we need this > now? In VFIO/mdev [1], the offset can be something different. It depends on the parent device. It's not just a cleanup. It's a preparation for the mdev support in DPDK. [1] https://github.com/torvalds/linux/blob/master/Documentation/ vfio-mediated-device.txt
Quoted 2 lines
Ok, thanks.
So we can wait for mdev to be ready before working on this.
What do you mean by "mdev to be ready"? RFC ready? I don't see anything blocking the discussion on this now.
PS. I already sent a RFC series of the mdev support in DPDK
to the mailing list 3 month ago.
Thanks,
Tiwei
Quoted 4 lines
--
David Marchand
Quoted 40 lines · Tiwei Bie
Quoted 8 lines · David Marchand
On Wed, Jul 3, 2019 at 9:35 AM Tiwei Bie <[email protected]> wrote:
Hi David, On Wed, Jul 03, 2019 at 09:02:59AM +0200, David Marchand wrote: > Hello, > > On Wed, Jul 3, 2019 at 7:47 AM Tiwei Bie <[email protected]>Quoted 6 lines · wrote:
Quoted 6 lines
The value 40 used in
VFIO_GET_REGION_ADDR()is a private value (VFIO_PCI_OFFSET_SHIFT) defined in Linux kernel source [1]. It is not part of VFIO API, and we should not depend on it.[1]
https://github.com/torvalds/linux/blob/6fbc7275c7a9/drivers/vfio/
Quoted 6 lines
pci/ > vfio_pci_private.h#L19 > > > > I did not follow linux kernel changes, is there something thatwould
Quoted 3 lines
change
> this offset?
> It looks like a cleanup (did not look into the details yet), do weneed
Quoted 12 lines
this
> now?In VFIO/mdev [1], the offset can be something different. It depends on the parent device. It's not just a cleanup. It's a preparation for the mdev support in DPDK.
[1] https://github.com/torvalds/linux/blob/master/Documentation/
vfio-mediated-device.txt
Quoted 9 lines
Quoted 2 lines
Ok, thanks.
So we can wait for mdev to be ready before working on this.What do you mean by "mdev to be ready"? RFC ready? I don't see anything blocking the discussion on this now.
PS. I already sent a RFC series of the mdev support in DPDK
to the mailing list 3 month ago.
If you need it and the mdev support has been posted already, why not send a n+1 patchset with this patch in it?
This patch alone looked odd to me.
I want to understand if this is a fix we need now or something that can wait.
We have a really large backlog.
Signature · 1 line
David Marchand
Quoted 48 lines · David Marchand
On Wed, Jul 3, 2019 at 9:58 AM Tiwei Bie <[email protected]> wrote:
On Wed, Jul 03, 2019 at 09:36:26AM +0200, David Marchand wrote:
> On Wed, Jul 3, 2019 at 9:35 AM Tiwei Bie <[email protected]> wrote:
>
> Hi David,
>
> On Wed, Jul 03, 2019 at 09:02:59AM +0200, David Marchand wrote:
> > Hello,
> >
> > On Wed, Jul 3, 2019 at 7:47 AM Tiwei Bie <[email protected]>
wrote:
> >
> > The value 40 used inVFIO_GET_REGION_ADDR()is a private value
> > (VFIO_PCI_OFFSET_SHIFT) defined in Linux kernel source [1]. It
> > is not part of VFIO API, and we should not depend on it.
> >
> > [1] https://github.com/torvalds/linux/blob/6fbc7275c7a9/drivers
/vfio/
> pci/
> > vfio_pci_private.h#L19
> >
> >
> >
> > I did not follow linux kernel changes, is there something that
would
> change
> > this offset?
> > It looks like a cleanup (did not look into the details yet), do we
need
> this
> > now?
>
> In VFIO/mdev [1], the offset can be something different. It depends
> on the parent device. It's not just a cleanup. It's a preparation
> for the mdev support in DPDK.
>
> [1] https://github.com/torvalds/linux/blob/master/Documentation/
> vfio-mediated-device.txt
>
>
>
> Ok, thanks.
> So we can wait for mdev to be ready before working on this.What do you mean by "mdev to be ready"? RFC ready? I don't see anything blocking the discussion on this now.
Quoted 8 lines
PS. I already sent a RFC series of the mdev support in DPDK
to the mailing list 3 month ago.If you need it and the mdev support has been posted already, why not send a n+1 patchset with this patch in it?
This patch alone looked odd to me.
That series was using the old API which assumes the shift is 40 which may not work in some cases. And this patch is to fix the API. I think this patch is actually trying to fix a relatively independent issue -- i.e. switching to using the proper VFIO API to get the region offsets instead of depending on kernel code's internal value.
Quoted 2 lines
I want to understand if this is a fix we need now or something that can wait. We have a really large backlog.
I guess that's why I didn't get a lot of responses in that RFC series. :)
Thanks,
Tiwei
Quoted 4 lines
--
David Marchand
Quoted 67 lines · Tiwei Bie
Quoted 4 lines · David Marchand
On Wed, Jul 3, 2019 at 9:58 AM Tiwei Bie <[email protected]> wrote:
On Wed, Jul 03, 2019 at 09:36:26AM +0200, David Marchand wrote: > On Wed, Jul 3, 2019 at 9:35 AM Tiwei Bie <[email protected]>Quoted 7 lines · wrote:
Quoted 7 lines
Hi David, On Wed, Jul 03, 2019 at 09:02:59AM +0200, David Marchand wrote: > Hello, > > On Wed, Jul 3, 2019 at 7:47 AM Tiwei Bie <Quoted 3 lines
wrote:
> >
> > The value 40 used inVFIO_GET_REGION_ADDR()is a privatevalue
Quoted 1 line
Quoted 1 line
Quoted 1 line
(
VFIO_PCI_OFFSET_SHIFT) defined in Linux kernel source[1]. It
Quoted 3 lines
Quoted 3 lines
Quoted 3 lines
is not part of VFIO API, and we should not depend on it.
[1]
https://github.com/torvalds/linux/blob/6fbc7275c7a9/drivers
Quoted 7 lines
/vfio/ > pci/ > > vfio_pci_private.h#L19 > > > > > > > > I did not follow linux kernel changes, is there somethingthat
Quoted 4 lines
would
> change
> > this offset?
> > It looks like a cleanup (did not look into the details yet),do we
Quoted 5 lines
need
> this
> > now?
>
> In VFIO/mdev [1], the offset can be something different. Itdepends
Quoted 1 line
Quoted 1 line
on the parent device. It's not just a cleanup. It's a
preparation
Quoted 3 lines
Quoted 3 lines
for the mdev support in DPDK.
[1]
https://github.com/torvalds/linux/blob/master/Documentation/
Quoted 7 lines
Quoted 6 lines
vfio-mediated-device.txt
Ok, thanks.
So we can wait for mdev to be ready before working on this.Quoted 8 lines
What do you mean by "mdev to be ready"? RFC ready? I don't see anything blocking the discussion on this now.
PS. I already sent a RFC series of the mdev support in DPDK
to the mailing list 3 month ago.If you need it and the mdev support has been posted already, why not
send a n+1
Quoted 2 lines
patchset with this patch in it?
Quoted 10 lines
Quoted 1 line
This patch alone looked odd to me.
That series was using the old API which assumes the shift is 40 which may not work in some cases. And this patch is to fix the API. I think this patch is actually trying to fix a relatively independent issue -- i.e. switching to using the proper VFIO API to get the region offsets instead of depending on kernel code's internal value.
Fix, then there is something broken ?
You said this is for mdev support which is not currently part of the features supported by DPDK.
This patch breaks the ABI by extending rte_pci_device. You must rework it to avoid this break.
Signature · 1 line
David Marchand
Quoted 80 lines · David Marchand
On Wed, Jul 3, 2019 at 10:17 AM Tiwei Bie <[email protected]> wrote:
On Wed, Jul 03, 2019 at 10:01:44AM +0200, David Marchand wrote:
> On Wed, Jul 3, 2019 at 9:58 AM Tiwei Bie <[email protected]> wrote:
>
> On Wed, Jul 03, 2019 at 09:36:26AM +0200, David Marchand wrote:
> > On Wed, Jul 3, 2019 at 9:35 AM Tiwei Bie <[email protected]>
wrote:
> >
> > Hi David,
> >
> > On Wed, Jul 03, 2019 at 09:02:59AM +0200, David Marchand wrote:
> > > Hello,
> > >
> > > On Wed, Jul 3, 2019 at 7:47 AM Tiwei Bie <[email protected]
>
> wrote:
> > >
> > > The value 40 used inVFIO_GET_REGION_ADDR()is a private
value
> > > (VFIO_PCI_OFFSET_SHIFT) defined in Linux kernel source
[1]. It
> > > is not part of VFIO API, and we should not depend on it.
> > >
> > > [1] https://github.com/torvalds/linux/blob/6fbc7275c7a9/
drivers
> /vfio/
> > pci/
> > > vfio_pci_private.h#L19
> > >
> > >
> > >
> > > I did not follow linux kernel changes, is there something
that
> would
> > change
> > > this offset?
> > > It looks like a cleanup (did not look into the details yet),
do we
> need
> > this
> > > now?
> >
> > In VFIO/mdev [1], the offset can be something different. It
depends
> > on the parent device. It's not just a cleanup. It's a
preparation
> > for the mdev support in DPDK.
> >
> > [1] https://github.com/torvalds/linux/blob/master/Documentation
/
> > vfio-mediated-device.txt
> >
> >
> >
> > Ok, thanks.
> > So we can wait for mdev to be ready before working on this.
>
> What do you mean by "mdev to be ready"? RFC ready? I don't see
> anything blocking the discussion on this now.
>
> PS. I already sent a RFC series of the mdev support in DPDK
> to the mailing list 3 month ago.
>
>
> If you need it and the mdev support has been posted already, why not send a n+1
> patchset with this patch in it?
>
> This patch alone looked odd to me.That series was using the old API which assumes the shift is 40 which may not work in some cases. And this patch is to fix the API. I think this patch is actually trying to fix a relatively independent issue -- i.e. switching to using the proper VFIO API to get the region offsets instead of depending on kernel code's internal value.
Quoted 1 line
Fix, then there is something broken ?
I should use "fix" (with "") actually :)
Quoted 6 lines
You said this is for mdev support which is not currently part of the features supported by DPDK.
This patch breaks the ABI by extending
rte_pci_device. You must rework it to avoid this break.
I didn't expect it to be merged in this release. I just want to draw other's attention on this and kick off the discussion (it would be great if you would like to share your thoughts on this). If there is a way to avoid extending rte_pci_device, it would be definitely great. But if we have to break it, then we would want to send out the announce as early as possible.
Thanks,
Tiwei
Quoted 4 lines
--
David Marchand
Quoted 11 lines · Tiwei Bie
Quoted 2 lines · David Marchand
This patch breaks the ABI by extending
rte_pci_device. You must rework it to avoid this break.I didn't expect it to be merged in this release. I just want to draw other's attention on this and kick off the discussion (it would be great if you would like to share your thoughts on this). If there is a way to avoid extending
rte_pci_device, it would be definitely great. But if we have to break it, then we would want to send out the announce as early as possible.
What we have here is a vfio private thing, we don't need it to be exposed.
Did not think it through yet.
How about having an internal (as in, in the pci driver code) representation of the pci devices?
This internal structure would embed the rte_pci_device exposed to the others subsystems and the applications and the vfio code would just get what it wants by using offsetof?
Signature · 1 line
David Marchand
Quoted 15 lines · David Marchand
On Wed, Jul 3, 2019 at 11:01 AM Tiwei Bie <[email protected]> wrote:
On Wed, Jul 03, 2019 at 10:26:39AM +0200, David Marchand wrote:
> This patch breaks the ABI by extendingrte_pci_device. > You must rework it to avoid this break.I didn't expect it to be merged in this release. I just want to draw other's attention on this and kick off the discussion (it would be great if you would like to share your thoughts on this). If there is a way to avoid extending
rte_pci_device, it would be definitely great. But if we have to break it, then we would want to send out the announce as early as possible.
Quoted 8 lines
What we have here is a vfio private thing, we don't need it to be exposed.
Did not think it through yet.
How about having an internal (as in, in the pci driver code) representation of the pci devices?
This internal structure would embed therte_pci_deviceexposed to the others subsystems and the applications and the vfio code would just get what it wants by using offsetof?
I think it's a good idea! I'll give it a try.
Thanks!
Tiwei
Quoted 4 lines
--
David Marchand