Skip to contentSkip to conversation

pci/linux: use RTE_IOVA_VA whenever possible

PATCH In review Darek Stojaczyk 5 participants 6 messages Sep 7, 2018 – Oct 28, 2018

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 pci/linux: use RTE_IOVA_VA whenever possible

This allows DPDK to use RTE_IOVA_VA with VFIO/UIO-bound PCI devices present on the system, but not attached to any rte_pci_driver at the time of init.

So far we used RTE_IOVA_VA whenever there was at least one device attached to a driver with an RTE_PCI_DRV_IOVA_AS_VA flag, meaning that other drivers which didn't explicitly report such flag could have been forced to work in RTE_IOVA_VA as well.

This patch makes the RTE_PCI_DRV_IOVA_AS_VA explicitly a hint. If it's set, but RTE_IOVA_VA cannot be used, then EAL will print a proper warning.

---
 drivers/bus/pci/linux/pci.c | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)
drivers/bus/pci/linux/pci.c +5 -6
index 04648ac93..961e24024 100644
@@ -534,7 +534,7 @@ pci_one_device_bound_uio(void)
  * Any one of the device has iova as va
  */
 static inline int
-pci_one_device_has_iova_va(void)
+pci_one_device_want_iova_va(void)
 {
 	struct rte_pci_device *dev = NULL;
 	struct rte_pci_driver *drv = NULL;
@@ -635,7 +635,7 @@ rte_pci_get_iommu_class(void)
 {
 	bool is_bound;
 	bool is_vfio_noiommu_enabled = true;
-	bool has_iova_va;
+	bool want_iova_va;
 	bool is_bound_uio;
 	bool iommu_no_va;
 
@@ -643,7 +643,7 @@ rte_pci_get_iommu_class(void)
 	if (!is_bound)
 		return RTE_IOVA_DC;
 
-	has_iova_va = pci_one_device_has_iova_va();
+	want_iova_va = pci_one_device_want_iova_va();
 	is_bound_uio = pci_one_device_bound_uio();
 	iommu_no_va = !pci_devices_iommu_support_va();
 #ifdef VFIO_PRESENT
@@ -651,11 +651,10 @@ rte_pci_get_iommu_class(void)
 					true : false;
 #endif
 
-	if (has_iova_va && !is_bound_uio && !is_vfio_noiommu_enabled &&
-			!iommu_no_va)
+	if (!is_bound_uio && !is_vfio_noiommu_enabled && !iommu_no_va)
 		return RTE_IOVA_VA;
 
-	if (has_iova_va) {
+	if (want_iova_va) {
 		RTE_LOG(WARNING, EAL, "Some devices want iova as va but pa will be used because.. ");
 		if (is_vfio_noiommu_enabled)
 			RTE_LOG(WARNING, EAL, "vfio-noiommu mode configured\n");
Signature · 1 line
2.17.1
#1 lore raw

Someone to review this patch please?

07/09/2018 16:53, Darek Stojaczyk:

Quoted 64 lines

This allows DPDK to use RTE_IOVA_VA with VFIO/UIO-bound PCI devices present on the system, but not attached to any rte_pci_driver at the time of init.

So far we used RTE_IOVA_VA whenever there was at least one device attached to a driver with an RTE_PCI_DRV_IOVA_AS_VA flag, meaning that other drivers which didn't explicitly report such flag could have been forced to work in RTE_IOVA_VA as well.

This patch makes the RTE_PCI_DRV_IOVA_AS_VA explicitly a hint. If it's set, but RTE_IOVA_VA cannot be used, then EAL will print a proper warning.

---
 drivers/bus/pci/linux/pci.c | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)
drivers/bus/pci/linux/pci.c +5 -6
index 04648ac93..961e24024 100644
@@ -534,7 +534,7 @@ pci_one_device_bound_uio(void)
  * Any one of the device has iova as va
  */
 static inline int
-pci_one_device_has_iova_va(void)
+pci_one_device_want_iova_va(void)
 {
 	struct rte_pci_device *dev = NULL;
 	struct rte_pci_driver *drv = NULL;
@@ -635,7 +635,7 @@ rte_pci_get_iommu_class(void)
 {
 	bool is_bound;
 	bool is_vfio_noiommu_enabled = true;
-	bool has_iova_va;
+	bool want_iova_va;
 	bool is_bound_uio;
 	bool iommu_no_va;
 
@@ -643,7 +643,7 @@ rte_pci_get_iommu_class(void)
 	if (!is_bound)
 		return RTE_IOVA_DC;
 
-	has_iova_va = pci_one_device_has_iova_va();
+	want_iova_va = pci_one_device_want_iova_va();
 	is_bound_uio = pci_one_device_bound_uio();
 	iommu_no_va = !pci_devices_iommu_support_va();
 #ifdef VFIO_PRESENT
@@ -651,11 +651,10 @@ rte_pci_get_iommu_class(void)
 					true : false;
 #endif
 
-	if (has_iova_va && !is_bound_uio && !is_vfio_noiommu_enabled &&
-			!iommu_no_va)
+	if (!is_bound_uio && !is_vfio_noiommu_enabled && !iommu_no_va)
 		return RTE_IOVA_VA;
 
-	if (has_iova_va) {
+	if (want_iova_va) {
 		RTE_LOG(WARNING, EAL, "Some devices want iova as va but pa will be used because.. ");
 		if (is_vfio_noiommu_enabled)
 			RTE_LOG(WARNING, EAL, "vfio-noiommu mode configured\n");
#2 lore raw
In reply to Darek Stojaczyk
Quoted 9 lines · Darek Stojaczyk

This allows DPDK to use RTE_IOVA_VA with VFIO/UIO-bound PCI devices present on the system, but not attached to any rte_pci_driver at the time of init.

So far we used RTE_IOVA_VA whenever there was at least one device attached to a driver with an RTE_PCI_DRV_IOVA_AS_VA flag, meaning that other drivers which didn't explicitly report such flag could have been forced to work in RTE_IOVA_VA as well.

This is the opposite. Just one device not being able to use IOVA VA makes all to use IOVA PA.

Quoted 42 lines

This patch makes the RTE_PCI_DRV_IOVA_AS_VA explicitly a hint. If it's set, but RTE_IOVA_VA cannot be used, then EAL will print a proper warning.

---
 drivers/bus/pci/linux/pci.c | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)
drivers/bus/pci/linux/pci.c +3 -3
index 04648ac93..961e24024 100644
@@ -534,7 +534,7 @@ pci_one_device_bound_uio(void)
  * Any one of the device has iova as va
  */
 static inline int
-pci_one_device_has_iova_va(void)
+pci_one_device_want_iova_va(void)
 {
        struct rte_pci_device *dev = NULL;
        struct rte_pci_driver *drv = NULL;
@@ -635,7 +635,7 @@ rte_pci_get_iommu_class(void)
 {
        bool is_bound;
        bool is_vfio_noiommu_enabled = true;
-       bool has_iova_va;
+       bool want_iova_va;
        bool is_bound_uio;
        bool iommu_no_va;
 
@@ -643,7 +643,7 @@ rte_pci_get_iommu_class(void)
        if (!is_bound)
                return RTE_IOVA_DC;
 
-       has_iova_va = pci_one_device_has_iova_va();
+       want_iova_va = pci_one_device_want_iova_va();
        is_bound_uio = pci_one_device_bound_uio();
        iommu_no_va = !pci_devices_iommu_support_va();
 #ifdef VFIO_PRESENT

drivers/bus/pci/linux/pci.c rte_pci_get_iommu_class(void) · line 656

Quoted 8 lines
drivers/bus/pci/linux/pci.c +1 -2
@@ -651,11 +651,10 @@ rte_pci_get_iommu_class(void)
                                        true : false;
 #endif
 
-       if (has_iova_va && !is_bound_uio && !is_vfio_noiommu_enabled &&
-                       !iommu_no_va)
+       if (!is_bound_uio && !is_vfio_noiommu_enabled && !iommu_no_va)
                return RTE_IOVA_VA;
 

This is wrong. A device not able to work with IOVA VA will fail.

Quoted 12 lines
drivers/bus/pci/linux/pci.c +1 -1
-       if (has_iova_va) {
+       if (want_iova_va) {
                RTE_LOG(WARNING, EAL, "Some devices want iova as va but pa
 will be used because.. ");
                if (is_vfio_noiommu_enabled)

RTE_LOG(WARNING, EAL, "vfio-noiommu mode
configured\n");
--
2.17.1

#3 lore raw
Quoted 62 lines · Alejandro Lucero
Quoted 9 lines · Darek Stojaczyk

This allows DPDK to use RTE_IOVA_VA with VFIO/UIO-bound PCI devices present on the system, but not attached to any rte_pci_driver at the time of init.

So far we used RTE_IOVA_VA whenever there was at least one device attached to a driver with an RTE_PCI_DRV_IOVA_AS_VA flag, meaning that other drivers which didn't explicitly report such flag could have been forced to work in RTE_IOVA_VA as well.

This is the opposite. Just one device not being able to use IOVA VA makes all to use IOVA PA.

Quoted 45 lines

This patch makes the RTE_PCI_DRV_IOVA_AS_VA explicitly a hint. If it's set, but RTE_IOVA_VA cannot be used, then EAL will print a proper warning.

---
  drivers/bus/pci/linux/pci.c | 11 +++++------
  1 file changed, 5 insertions(+), 6 deletions(-)
drivers/bus/pci/linux/pci.c +3 -3
index 04648ac93..961e24024 100644
@@ -534,7 +534,7 @@ pci_one_device_bound_uio(void)
   * Any one of the device has iova as va
   */
  static inline int
-pci_one_device_has_iova_va(void)
+pci_one_device_want_iova_va(void)
  {
         struct rte_pci_device *dev = NULL;
         struct rte_pci_driver *drv = NULL;
@@ -635,7 +635,7 @@ rte_pci_get_iommu_class(void)
  {
         bool is_bound;
         bool is_vfio_noiommu_enabled = true;
-       bool has_iova_va;
+       bool want_iova_va;
         bool is_bound_uio;
         bool iommu_no_va;
 
@@ -643,7 +643,7 @@ rte_pci_get_iommu_class(void)
         if (!is_bound)
                 return RTE_IOVA_DC;
 
-       has_iova_va = pci_one_device_has_iova_va();
+       want_iova_va = pci_one_device_want_iova_va();
         is_bound_uio = pci_one_device_bound_uio();
         iommu_no_va = !pci_devices_iommu_support_va();
  #ifdef VFIO_PRESENT
@@ -651,11 +651,10 @@ rte_pci_get_iommu_class(void)
                                         true : false;
  #endif
 
Quoted 22 lines

drivers/bus/pci/linux/pci.c rte_pci_get_iommu_class(void) · line 656

Quoted 5 lines
drivers/bus/pci/linux/pci.c +1 -2
-       if (has_iova_va && !is_bound_uio && !is_vfio_noiommu_enabled &&
-                       !iommu_no_va)
+       if (!is_bound_uio && !is_vfio_noiommu_enabled && !iommu_no_va)
                 return RTE_IOVA_VA;
 

This is wrong. A device not able to work with IOVA VA will fail.

Quoted 12 lines
drivers/bus/pci/linux/pci.c +1 -1
-       if (has_iova_va) {
+       if (want_iova_va) {
                 RTE_LOG(WARNING, EAL, "Some devices want iova as va but pa
 will be used because.. ");
                 if (is_vfio_noiommu_enabled)

RTE_LOG(WARNING, EAL, "vfio-noiommu mode
configured\n");
--
2.17.1

For these cases, i think the explicit IOVA mode on command line should work better. If the device has not reported IOVA as VA capability, it is to be assumed unsupported.

Signature · 2 lines
Thanks,
Anatoly
#4 lore raw

-----Original Message-----

Quoted 99 lines

Date: Thu, 11 Oct 2018 11:26:05 +0100
From: "Burakov, Anatoly" <[email protected]>
To: Alejandro Lucero <[email protected]>,
[email protected]
CC: dev <[email protected]>, Santosh Shukla <[email protected]>,
Hemant Agrawal <[email protected]>, Jerin Jacob <[email protected]>, Maxime Coquelin <[email protected]>, [email protected]
Subject: Re: [dpdk-dev] [PATCH] pci/linux: use RTE_IOVA_VA whenever possible
User-Agent: Mozilla/5.0 (Windows NT 10.0; WOW64; rv:52.0) Gecko/20100101
Thunderbird/52.9.1

Quoted 62 lines · Alejandro Lucero
Quoted 9 lines · Darek Stojaczyk

This allows DPDK to use RTE_IOVA_VA with VFIO/UIO-bound PCI devices present on the system, but not attached to any rte_pci_driver at the time of init.

So far we used RTE_IOVA_VA whenever there was at least one device attached to a driver with an RTE_PCI_DRV_IOVA_AS_VA flag, meaning that other drivers which didn't explicitly report such flag could have been forced to work in RTE_IOVA_VA as well.

This is the opposite. Just one device not being able to use IOVA VA makes all to use IOVA PA.

Quoted 45 lines

This patch makes the RTE_PCI_DRV_IOVA_AS_VA explicitly a hint. If it's set, but RTE_IOVA_VA cannot be used, then EAL will print a proper warning.

---
  drivers/bus/pci/linux/pci.c | 11 +++++------
  1 file changed, 5 insertions(+), 6 deletions(-)
drivers/bus/pci/linux/pci.c +3 -3
index 04648ac93..961e24024 100644
@@ -534,7 +534,7 @@ pci_one_device_bound_uio(void)
   * Any one of the device has iova as va
   */
  static inline int
-pci_one_device_has_iova_va(void)
+pci_one_device_want_iova_va(void)
  {
         struct rte_pci_device *dev = NULL;
         struct rte_pci_driver *drv = NULL;
@@ -635,7 +635,7 @@ rte_pci_get_iommu_class(void)
  {
         bool is_bound;
         bool is_vfio_noiommu_enabled = true;
-       bool has_iova_va;
+       bool want_iova_va;
         bool is_bound_uio;
         bool iommu_no_va;
 
@@ -643,7 +643,7 @@ rte_pci_get_iommu_class(void)
         if (!is_bound)
                 return RTE_IOVA_DC;
 
-       has_iova_va = pci_one_device_has_iova_va();
+       want_iova_va = pci_one_device_want_iova_va();
         is_bound_uio = pci_one_device_bound_uio();
         iommu_no_va = !pci_devices_iommu_support_va();
  #ifdef VFIO_PRESENT
@@ -651,11 +651,10 @@ rte_pci_get_iommu_class(void)
                                         true : false;
  #endif
 
Quoted 22 lines

drivers/bus/pci/linux/pci.c rte_pci_get_iommu_class(void) · line 656

Quoted 5 lines
drivers/bus/pci/linux/pci.c +1 -2
-       if (has_iova_va && !is_bound_uio && !is_vfio_noiommu_enabled &&
-                       !iommu_no_va)
+       if (!is_bound_uio && !is_vfio_noiommu_enabled && !iommu_no_va)
                 return RTE_IOVA_VA;
 

This is wrong. A device not able to work with IOVA VA will fail.

Quoted 12 lines
drivers/bus/pci/linux/pci.c +1 -1
-       if (has_iova_va) {
+       if (want_iova_va) {
                 RTE_LOG(WARNING, EAL, "Some devices want iova as va but pa
 will be used because.. ");
                 if (is_vfio_noiommu_enabled)

RTE_LOG(WARNING, EAL, "vfio-noiommu mode
configured\n");
--
2.17.1

Quoted 3 lines

For these cases, i think the explicit IOVA mode on command line should work better. If the device has not reported IOVA as VA capability, it is to be assumed unsupported.

Yes.

Quoted 4 lines

--
Thanks,
Anatoly

#5 lore raw

11/10/2018 12:47, Jerin Jacob:

Quoted 100 lines

-----Original Message-----

Quoted 99 lines

Date: Thu, 11 Oct 2018 11:26:05 +0100
From: "Burakov, Anatoly" <[email protected]>
To: Alejandro Lucero <[email protected]>,
[email protected]
CC: dev <[email protected]>, Santosh Shukla <[email protected]>,
Hemant Agrawal <[email protected]>, Jerin Jacob <[email protected]>, Maxime Coquelin <[email protected]>, [email protected]
Subject: Re: [dpdk-dev] [PATCH] pci/linux: use RTE_IOVA_VA whenever possible
User-Agent: Mozilla/5.0 (Windows NT 10.0; WOW64; rv:52.0) Gecko/20100101
Thunderbird/52.9.1

Quoted 84 lines · Alejandro Lucero
Quoted 9 lines · Darek Stojaczyk

This allows DPDK to use RTE_IOVA_VA with VFIO/UIO-bound PCI devices present on the system, but not attached to any rte_pci_driver at the time of init.

So far we used RTE_IOVA_VA whenever there was at least one device attached to a driver with an RTE_PCI_DRV_IOVA_AS_VA flag, meaning that other drivers which didn't explicitly report such flag could have been forced to work in RTE_IOVA_VA as well.

This is the opposite. Just one device not being able to use IOVA VA makes all to use IOVA PA.

Quoted 42 lines

This patch makes the RTE_PCI_DRV_IOVA_AS_VA explicitly a hint. If it's set, but RTE_IOVA_VA cannot be used, then EAL will print a proper warning.

---
  drivers/bus/pci/linux/pci.c | 11 +++++------
  1 file changed, 5 insertions(+), 6 deletions(-)
drivers/bus/pci/linux/pci.c +3 -3
index 04648ac93..961e24024 100644
@@ -534,7 +534,7 @@ pci_one_device_bound_uio(void)
   * Any one of the device has iova as va
   */
  static inline int
-pci_one_device_has_iova_va(void)
+pci_one_device_want_iova_va(void)
  {
         struct rte_pci_device *dev = NULL;
         struct rte_pci_driver *drv = NULL;
@@ -635,7 +635,7 @@ rte_pci_get_iommu_class(void)
  {
         bool is_bound;
         bool is_vfio_noiommu_enabled = true;
-       bool has_iova_va;
+       bool want_iova_va;
         bool is_bound_uio;
         bool iommu_no_va;
 
@@ -643,7 +643,7 @@ rte_pci_get_iommu_class(void)
         if (!is_bound)
                 return RTE_IOVA_DC;
 
-       has_iova_va = pci_one_device_has_iova_va();
+       want_iova_va = pci_one_device_want_iova_va();
         is_bound_uio = pci_one_device_bound_uio();
         iommu_no_va = !pci_devices_iommu_support_va();
  #ifdef VFIO_PRESENT

drivers/bus/pci/linux/pci.c rte_pci_get_iommu_class(void) · line 656

Quoted 8 lines
drivers/bus/pci/linux/pci.c +1 -2
@@ -651,11 +651,10 @@ rte_pci_get_iommu_class(void)
                                         true : false;
  #endif
 
-       if (has_iova_va && !is_bound_uio && !is_vfio_noiommu_enabled &&
-                       !iommu_no_va)
+       if (!is_bound_uio && !is_vfio_noiommu_enabled && !iommu_no_va)
                 return RTE_IOVA_VA;
 

This is wrong. A device not able to work with IOVA VA will fail.

Quoted 12 lines
drivers/bus/pci/linux/pci.c +1 -1
-       if (has_iova_va) {
+       if (want_iova_va) {
                 RTE_LOG(WARNING, EAL, "Some devices want iova as va but pa
 will be used because.. ");
                 if (is_vfio_noiommu_enabled)

RTE_LOG(WARNING, EAL, "vfio-noiommu mode
configured\n");
--
2.17.1

Quoted 5 lines
Quoted 3 lines

For these cases, i think the explicit IOVA mode on command line should work better. If the device has not reported IOVA as VA capability, it is to be assumed unsupported.

Yes.

Patch marked as rejected