Skip to contentSkip to conversation

vfio: try physical address if virtual address fails

PATCH In review Chas Williams 2 participants 4 messages Nov 24, 2017 – Nov 30, 2017

lore
Share & tools
lore

Chas Williams

author
#0 lore raw

PATCH 1/1 vfio: try physical address if virtual address fails

Some machines appear to have buggy DMAR mappings. A typical mapping error looks like:

DMAR: intel_iommu_map: iommu width (39) is not sufficient for the mapped address (7fc4fa800000)
DMAR: intel_iommu_map: iommu width (39) is not sufficient for the mapped address (7fc4fa800000)
DMAR: intel_iommu_map: iommu width (39) is not sufficient for the mapped address (7fc4fa800000)
DMAR: intel_iommu_map: iommu width (39) is not sufficient for the mapped address (7fc4fa800000)

To work around this, attempt to do a physical address mapping if the virtual address mapping fails.

Fixes e85a919286d2 ("vfio: honor IOVA mode before mapping")
Signed-off-by Chas Williams <[email protected]>
---
 lib/librte_eal/linuxapp/eal/eal_vfio.c | 8 ++++++++
 1 file changed, 8 insertions(+)
lib/librte_eal/linuxapp/eal/eal_vfio.c +8 -0
index 58f0123..6250676 100644
@@ -35,6 +35,7 @@
 #include <fcntl.h>
 #include <unistd.h>
 #include <sys/ioctl.h>
+#include <stdbool.h>
 
 #include <rte_log.h>
 #include <rte_memory.h>

@@ -702,6 +703,7 @@ vfio_type1_dma_map(int vfio_container_fd)
 	/* map all DPDK segments for DMA. use 1:1 PA to IOVA mapping */
 	for (i = 0; i < RTE_MAX_MEMSEG; i++) {
 		struct vfio_iommu_type1_dma_map dma_map;
+		int retried = false;
 
 		if (ms[i].addr == NULL)
 			break;

@@ -716,9 +718,15 @@ vfio_type1_dma_map(int vfio_container_fd)
 			dma_map.iova = ms[i].iova;
 		dma_map.flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE;
 
+retry:
 		ret = ioctl(vfio_container_fd, VFIO_IOMMU_MAP_DMA, &dma_map);
 
 		if (ret) {
+			if (!retried && rte_eal_iova_mode() == RTE_IOVA_VA) {
+				dma_map.iova = ms[i].iova;
+				retried = true;
+				goto retry;
+			}
 			RTE_LOG(ERR, EAL, "  cannot set up DMA remapping, "
 					  "error %i (%s)\n", errno,
 					  strerror(errno));
Signature · 1 line
2.9.5
#1 lore raw

Hi William:

Quoted 11 lines

-----Original Message-----
From: dev [mailto:[email protected]] On Behalf Of Chas Williams
Sent: Saturday, November 25, 2017 7:57 AM
To: [email protected]
Cc: [email protected]; Chas Williams <[email protected]>; Chas Williams <[email protected]>
Subject: [dpdk-dev] [PATCH] vfio: try physical address if virtual address fails

Some machines appear to have buggy DMAR mappings. A typical mapping error looks like:

Quoted 9 lines

DMAR: intel_iommu_map: iommu width (39) is not sufficient for the
mapped address (7fc4fa800000)
DMAR: intel_iommu_map: iommu width (39) is not sufficient for the
mapped address (7fc4fa800000)
DMAR: intel_iommu_map: iommu width (39) is not sufficient for the
mapped address (7fc4fa800000)
DMAR: intel_iommu_map: iommu width (39) is not sufficient for the
mapped address (7fc4fa800000)

I met the same issue on some intel atom platform, the root cause is IOMMU only support 39 bit virtual address.
Not sure retry with physical address will be the right fix. I saw rte_eal_iova_mode is called at other place, it still take the virtual address as the mapped result, does that break something? So far the workaround may works by using --virtbase-addr to assign a address in range explicitly (for example 0x70,0000,0000)
Regards
Qi

Quoted 52 lines

To work around this, attempt to do a physical address mapping if the virtual address mapping fails.

Fixes e85a919286d2 ("vfio: honor IOVA mode before mapping")
Signed-off-by Chas Williams <[email protected]>
---
 lib/librte_eal/linuxapp/eal/eal_vfio.c | 8 ++++++++
 1 file changed, 8 insertions(+)
lib/librte_eal/linuxapp/eal/eal_vfio.c +8 -1
index 58f0123..6250676 100644
@@ -35,6 +35,7 @@
 #include <fcntl.h>
 #include <unistd.h>
 #include <sys/ioctl.h>
+#include <stdbool.h>
 
 #include <rte_log.h>
 #include <rte_memory.h>

@@ -702,6 +703,7 @@ vfio_type1_dma_map(int vfio_container_fd)
 	/* map all DPDK segments for DMA. use 1:1 PA to IOVA mapping */
 	for (i = 0; i < RTE_MAX_MEMSEG; i++) {
 		struct vfio_iommu_type1_dma_map dma_map;
+		int retried = false;
 
 		if (ms[i].addr == NULL)
 			break;

@@ -716,9 +718,15 @@ vfio_type1_dma_map(int vfio_container_fd)
 			dma_map.iova = ms[i].iova;
 		dma_map.flags = VFIO_DMA_MAP_FLAG_READ |
 VFIO_DMA_MAP_FLAG_WRITE;
 
+retry:
 		ret = ioctl(vfio_container_fd, VFIO_IOMMU_MAP_DMA,
 &dma_map);
 
 		if (ret) {
+			if (!retried && rte_eal_iova_mode() == RTE_IOVA_VA) {
+				dma_map.iova = ms[i].iova;
 
 
+				retried = true;
+				goto retry;
+			}
 			RTE_LOG(ERR, EAL, "  cannot set up DMA remapping, "
 					  "error %i (%s)\n", errno,
 					  strerror(errno));
--
Quoted 1 line

2.9.5

Regards
Qi

author
#2 lore raw
Quoted 23 lines · Qi Z

Hi William:

Quoted 7 lines

-----Original Message-----
From: dev [mailto:[email protected]] On Behalf Of Chas Williams
Sent: Saturday, November 25, 2017 7:57 AM
To: [email protected]
Cc: [email protected]; Chas Williams <[email protected]>; Chas Williams <[email protected]>
Subject: [dpdk-dev] [PATCH] vfio: try physical address if virtual

address fails

Quoted 4 lines

Some machines appear to have buggy DMAR mappings. A typical mapping error looks like:

Quoted 9 lines

DMAR: intel_iommu_map: iommu width (39) is not sufficient for the
mapped address (7fc4fa800000)
DMAR: intel_iommu_map: iommu width (39) is not sufficient for the
mapped address (7fc4fa800000)
DMAR: intel_iommu_map: iommu width (39) is not sufficient for the
mapped address (7fc4fa800000)
DMAR: intel_iommu_map: iommu width (39) is not sufficient for the
mapped address (7fc4fa800000)

Quoted 10 lines

I met the same issue on some intel atom platform, the root cause is IOMMU only support 39 bit virtual address.
Not sure retry with physical address will be the right fix. I saw rte_eal_iova_mode is called at other place, it still take the virtual address as the mapped result, does that break something?
So far the workaround may works by using --virtbase-addr to assign a address in range explicitly (for example 0x70,0000,0000) Regards
Qi

I don't think the IOVA usage elsewhere is an issue since I limited my changes to reworking what was done in commit e85a919286d2 which appears to be what broken things for me.

It's not clear that passing --base-virtaddr is a workable solution. First, it's only a hint to mmap(). The kernel really can do whatever it wants. And then, what values do I pick? If one value fails, do I just restart and keep trying new values until it succeeds? I just want to fall back to the previous behavior. I know the physical mapping will succeed since the DMAR tables are limited to 36 bits in width.

And yes, I should update the commit to blame the IOMMU width.

Quoted 59 lines
Quoted 2 lines

To work around this, attempt to do a physical address mapping if the

virtual

Quoted 31 lines

address mapping fails.

Fixes e85a919286d2 ("vfio: honor IOVA mode before mapping")
Signed-off-by Chas Williams <[email protected]>
---
 lib/librte_eal/linuxapp/eal/eal_vfio.c | 8 ++++++++
 1 file changed, 8 insertions(+)
lib/librte_eal/linuxapp/eal/eal_vfio.c +2 -0
index 58f0123..6250676 100644
@@ -35,6 +35,7 @@
 #include <fcntl.h>
 #include <unistd.h>
 #include <sys/ioctl.h>
+#include <stdbool.h>
 
 #include <rte_log.h>
 #include <rte_memory.h>

@@ -702,6 +703,7 @@ vfio_type1_dma_map(int vfio_container_fd)
      /* map all DPDK segments for DMA. use 1:1 PA to IOVA mapping */
      for (i = 0; i < RTE_MAX_MEMSEG; i++) {
              struct vfio_iommu_type1_dma_map dma_map;
+             int retried = false;
 
              if (ms[i].addr == NULL)
                      break;

lib/librte_eal/linuxapp/eal/eal_vfio.c vfio_type1_dma_map(int vfio_container_fd) · line 727

Quoted 10 lines
lib/librte_eal/linuxapp/eal/eal_vfio.c +2 -0
@@ -716,9 +718,15 @@ vfio_type1_dma_map(int vfio_container_fd)
                      dma_map.iova = ms[i].iova;
              dma_map.flags = VFIO_DMA_MAP_FLAG_READ |
 VFIO_DMA_MAP_FLAG_WRITE;
 
+retry:
              ret = ioctl(vfio_container_fd, VFIO_IOMMU_MAP_DMA,
 &dma_map);
 
              if (ret) {
+                     if (!retried && rte_eal_iova_mode() ==

RTE_IOVA_VA) {

Quoted 11 lines
lib/librte_eal/linuxapp/eal/eal_vfio.c +4 -1
+                             dma_map.iova = ms[i].iova;
 
 
+                             retried = true;
+                             goto retry;
+                     }
                      RTE_LOG(ERR, EAL, "  cannot set up DMA remapping, "
                                        "error %i (%s)\n", errno,
                                        strerror(errno));
--

2.9.5

Regards
Qi

author
#3 lore raw

OK, I found a machine in our local collection that couldn't get by with this patch, so I will submitting a follow up that forces rte_eval_iova_mode() into PA mode.

Quoted 114 lines · Chas Williams
Quoted 23 lines · Qi Z

Hi William:

Quoted 7 lines

-----Original Message-----
From: dev [mailto:[email protected]] On Behalf Of Chas Williams
Sent: Saturday, November 25, 2017 7:57 AM
To: [email protected]
Cc: [email protected]; Chas Williams <[email protected]>; Chas Williams <[email protected]>
Subject: [dpdk-dev] [PATCH] vfio: try physical address if virtual

address fails

Quoted 4 lines

Some machines appear to have buggy DMAR mappings. A typical mapping error looks like:

Quoted 9 lines

DMAR: intel_iommu_map: iommu width (39) is not sufficient for the
mapped address (7fc4fa800000)
DMAR: intel_iommu_map: iommu width (39) is not sufficient for the
mapped address (7fc4fa800000)
DMAR: intel_iommu_map: iommu width (39) is not sufficient for the
mapped address (7fc4fa800000)
DMAR: intel_iommu_map: iommu width (39) is not sufficient for the
mapped address (7fc4fa800000)

Quoted 10 lines

I met the same issue on some intel atom platform, the root cause is IOMMU only support 39 bit virtual address.
Not sure retry with physical address will be the right fix. I saw rte_eal_iova_mode is called at other place, it still take the virtual address as the mapped result, does that break something?
So far the workaround may works by using --virtbase-addr to assign a address in range explicitly (for example 0x70,0000,0000) Regards
Qi

I don't think the IOVA usage elsewhere is an issue since I limited my changes to reworking what was done in commit e85a919286d2 which appears to be what broken things for me.

It's not clear that passing --base-virtaddr is a workable solution. First, it's only a hint to mmap(). The kernel really can do whatever it wants. And then, what values do I pick? If one value fails, do I just restart and keep trying new values until it succeeds? I just want to fall back to the previous behavior. I know the physical mapping will succeed since the DMAR tables are limited to 36 bits in width.

And yes, I should update the commit to blame the IOMMU width.

Quoted 60 lines
Quoted 2 lines

To work around this, attempt to do a physical address mapping if the

virtual

Quoted 31 lines

address mapping fails.

Fixes e85a919286d2 ("vfio: honor IOVA mode before mapping")
Signed-off-by Chas Williams <[email protected]>
---
 lib/librte_eal/linuxapp/eal/eal_vfio.c | 8 ++++++++
 1 file changed, 8 insertions(+)
lib/librte_eal/linuxapp/eal/eal_vfio.c +2 -0
index 58f0123..6250676 100644
@@ -35,6 +35,7 @@
 #include <fcntl.h>
 #include <unistd.h>
 #include <sys/ioctl.h>
+#include <stdbool.h>
 
 #include <rte_log.h>
 #include <rte_memory.h>

@@ -702,6 +703,7 @@ vfio_type1_dma_map(int vfio_container_fd)
      /* map all DPDK segments for DMA. use 1:1 PA to IOVA mapping */
      for (i = 0; i < RTE_MAX_MEMSEG; i++) {
              struct vfio_iommu_type1_dma_map dma_map;
+             int retried = false;
 
              if (ms[i].addr == NULL)
                      break;

lib/librte_eal/linuxapp/eal/eal_vfio.c vfio_type1_dma_map(int vfio_container_fd) · line 727

Quoted 10 lines
lib/librte_eal/linuxapp/eal/eal_vfio.c +2 -0
@@ -716,9 +718,15 @@ vfio_type1_dma_map(int vfio_container_fd)
                      dma_map.iova = ms[i].iova;
              dma_map.flags = VFIO_DMA_MAP_FLAG_READ |
 VFIO_DMA_MAP_FLAG_WRITE;
 
+retry:
              ret = ioctl(vfio_container_fd, VFIO_IOMMU_MAP_DMA,
 &dma_map);
 
              if (ret) {
+                     if (!retried && rte_eal_iova_mode() ==

RTE_IOVA_VA) {

lib/librte_eal/linuxapp/eal/eal_vfio.c vfio_type1_dma_map(int vfio_container_fd)

Quoted 7 lines
lib/librte_eal/linuxapp/eal/eal_vfio.c +4 -0
+                             dma_map.iova = ms[i].iova;
 
 
+                             retried = true;
+                             goto retry;
+                     }
                      RTE_LOG(ERR, EAL, "  cannot set up DMA remapping,

"

Quoted 4 lines
                                        "error %i (%s)\n", errno,
                                        strerror(errno));

--
2.9.5

Regards
Qi