Memory Hotplug for DPDK

Signed-off-by Anatoly Burakov <[email protected]>

---
 drivers/net/ena/base/ena_plat_dpdk.h | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

drivers/net/ena/base/ena_plat_dpdk.h +4 -4

index 8cba319..c1ebf00 100644
@@ -188,7 +188,8 @@ typedef uint64_t dma_addr_t;
                ENA_TOUCH(dmadev); ENA_TOUCH(handle);                   \
                snprintf(z_name, sizeof(z_name),                        \
                                "ena_alloc_%d", ena_alloc_cnt++);       \
-               mz = rte_memzone_reserve(z_name, size, SOCKET_ID_ANY, 0); \
+               mz = rte_memzone_reserve_contig(z_name,                 \
+                               size, SOCKET_ID_ANY, 0);                \
                memset(mz->addr, 0, size);                              \
                virt = mz->addr;                                        \
                phys = mz->iova;                                        \

@@ -206,7 +207,7 @@ typedef uint64_t dma_addr_t;
                ENA_TOUCH(dmadev); ENA_TOUCH(dev_node);                 \
                snprintf(z_name, sizeof(z_name),                        \
                                "ena_alloc_%d", ena_alloc_cnt++);       \
-               mz = rte_memzone_reserve(z_name, size, node, 0); \
+               mz = rte_memzone_reserve_contig(z_name, size, node, 0); \
                memset(mz->addr, 0, size);                              \
                virt = mz->addr;                                        \
                phys = mz->iova;                                        \

@@ -219,7 +220,7 @@ typedef uint64_t dma_addr_t;
                ENA_TOUCH(dmadev); ENA_TOUCH(dev_node);                 \
                snprintf(z_name, sizeof(z_name),                        \
                                "ena_alloc_%d", ena_alloc_cnt++);       \
-               mz = rte_memzone_reserve(z_name, size, node, 0); \
+               mz = rte_memzone_reserve_contig(z_name, size, node, 0); \
                memset(mz->addr, 0, size);                              \
                virt = mz->addr;                                        \
        } while (0)
--

2.7.4

Signed-off-by Anatoly Burakov <[email protected]>

---

Notes:
Doing "grep -R rte_memzone_reserve drivers/net/qede" returns the
following:

drivers/net/qede/qede_fdir.c: mz =
rte_memzone_reserve_aligned(mz_name, QEDE_MAX_FDIR_PKT_LEN,
drivers/net/qede/base/bcm_osal.c: mz =
rte_memzone_reserve_aligned_contig(mz_name, size,
drivers/net/qede/base/bcm_osal.c: mz =
rte_memzone_reserve_aligned_contig(mz_name, size, socket_id, 0,

I took a brief look at memzone in qede_fdir and it didn't look like
memzone
was used for DMA, so i left it alone. Corrections welcome.

drivers/net/qede/base/bcm_osal.c | 5 +++--

1 file changed, 3 insertions(+), 2 deletions(-)

drivers/net/qede/base/bcm_osal.c +3 -2

index fe42f32..707d553 100644
@@ -135,7 +135,7 @@ void *osal_dma_alloc_coherent(struct ecore_dev *p_dev,
 	if (core_id == (unsigned int)LCORE_ID_ANY)
 		core_id = 0;
 	socket_id = rte_lcore_to_socket_id(core_id);
-	mz = rte_memzone_reserve_aligned(mz_name, size,
+	mz = rte_memzone_reserve_aligned_contig(mz_name, size,
 					 socket_id, 0, RTE_CACHE_LINE_SIZE);
 	if (!mz) {
 		DP_ERR(p_dev, "Unable to allocate DMA memory "

@@ -174,7 +174,8 @@ void *osal_dma_alloc_coherent_aligned(struct
 ecore_dev *p_dev,
 	if (core_id == (unsigned int)LCORE_ID_ANY)
 		core_id = 0;
 	socket_id = rte_lcore_to_socket_id(core_id);
-	mz = rte_memzone_reserve_aligned(mz_name, size, socket_id, 0, align);
+	mz = rte_memzone_reserve_aligned_contig(mz_name, size, socket_id, 0,
+			align);
 	if (!mz) {
 		DP_ERR(p_dev, "Unable to allocate DMA memory "

		       "of size %zu bytes - %s\n",

Signature · 1 line

2.7.4

Signed-off-by Anatoly Burakov <[email protected]>

Reviewed-by Venkatesh Srinivas <[email protected]>

---

Notes:
Not sure if virtio needs to allocate DMA-capable memory, being a software driver and all. Corrections welcome.

  drivers/net/virtio/virtio_ethdev.c | 8 ++++----
  1 file changed, 4 insertions(+), 4 deletions(-)

This patchset introduces dynamic memory allocation for DPDK (aka memory hotplug). Based upon RFC submitted in December [1].

Dependencies (to be applied in specified order):
- IPC bugfixes patchset [2]
- IPC improvements patchset [3]
- IPC asynchronous request API patch [4]
- Function to return number of sockets [5]

2.7.4

Hi Anatoly,

I am trying to verify this patchset and have encountered few issues.

Few -Werror=maybe-uninitialized errors in eal_memalloc.c/eal_memory.c/ eal_common_memzone.c files.

lib/librte_eal/common/eal_common_memzone.c +1 -1

index a7cfdaf03..ad4413507 100644
@@ -321,7 +321,7 @@ rte_memzone_free(const struct rte_memzone *mz)
         struct rte_fbarray *arr;
         struct rte_memzone *found_mz;
         int ret = 0;
-       void *addr;
+       void *addr = NULL;
         unsigned idx;
 
         if (mz == NULL)

lib/librte_eal/linuxapp/eal/eal_memalloc.c +1 -1

index 1008faed6..32b0d5133 100644
@@ -570,7 +570,7 @@ eal_memalloc_alloc_page_bulk(struct rte_memseg **ms, int n,
         unsigned int msl_idx;
         int cur_idx, start_idx, end_idx, i, j, ret = -1;
  #ifdef RTE_EAL_NUMA_AWARE_HUGEPAGES
-       bool have_numa;
+       bool have_numa = false;
         int oldpolicy;
         struct bitmask *oldmask = numa_allocate_nodemask();
  #endif

lib/librte_eal/linuxapp/eal/eal_memory.c +2 -2

index f74291fb6..d37b4a59b 100644
@@ -1386,9 +1386,9 @@ eal_legacy_hugepage_attach(void)
         struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
         struct hugepage_file *hp = NULL;
         unsigned int num_hp = 0;
-       unsigned int i;
+       unsigned int i = 0;
         int ms_idx, msl_idx;
-       unsigned int cur_seg, max_seg;
+       unsigned int cur_seg, max_seg = 0;
         off_t size = 0;
         int fd, fd_hugepage = -1;
 

@Hemanth
Also, this patchset breaks dpaa/dpaa2 bus drivers (they rely on rte_eal_get_physmem_layout that is depricated
http://dpdk.org/dev/patchwork/patch/34002/)
So, generic arm64 linuxapp build is broken.

Regards,
Pavan.

Quoted 9 lines · Anatoly Burakov

This patchset introduces dynamic memory allocation for DPDK (aka memory hotplug). Based upon RFC submitted in December [1].

Dependencies (to be applied in specified order):
- IPC bugfixes patchset [2]
- IPC improvements patchset [3]
- IPC asynchronous request API patch [4]
- Function to return number of sockets [5]

<snip>
--

Quoted 1 line

2.7.4

Quoted 6 lines · Pavan Nikhilesh

Hi Anatoly,

I am trying to verify this patchset and have encountered few issues.

Few -Werror=maybe-uninitialized errors in eal_memalloc.c/eal_memory.c/ eal_common_memzone.c files.

Thanks for the heads up, i'll fix those in the next revision. Out of curiousity, which compiler version are you using?

Quoted 46 lines

lib/librte_eal/common/eal_common_memzone.c +1 -1

index a7cfdaf03..ad4413507 100644
@@ -321,7 +321,7 @@ rte_memzone_free(const struct rte_memzone *mz)
         struct rte_fbarray *arr;
         struct rte_memzone *found_mz;
         int ret = 0;
-       void *addr;
+       void *addr = NULL;
         unsigned idx;
 
         if (mz == NULL)

lib/librte_eal/linuxapp/eal/eal_memalloc.c +1 -1

index 1008faed6..32b0d5133 100644
@@ -570,7 +570,7 @@ eal_memalloc_alloc_page_bulk(struct rte_memseg **ms, int n,
         unsigned int msl_idx;
         int cur_idx, start_idx, end_idx, i, j, ret = -1;
  #ifdef RTE_EAL_NUMA_AWARE_HUGEPAGES
-       bool have_numa;
+       bool have_numa = false;
         int oldpolicy;
         struct bitmask *oldmask = numa_allocate_nodemask();
  #endif

lib/librte_eal/linuxapp/eal/eal_memory.c +2 -2

index f74291fb6..d37b4a59b 100644
@@ -1386,9 +1386,9 @@ eal_legacy_hugepage_attach(void)
         struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
         struct hugepage_file *hp = NULL;
         unsigned int num_hp = 0;
-       unsigned int i;
+       unsigned int i = 0;
         int ms_idx, msl_idx;
-       unsigned int cur_seg, max_seg;
+       unsigned int cur_seg, max_seg = 0;
         off_t size = 0;
         int fd, fd_hugepage = -1;
 

Quoted 5 lines

@Hemanth
Also, this patchset breaks dpaa/dpaa2 bus drivers (they rely on rte_eal_get_physmem_layout that is depricated
http://dpdk.org/dev/patchwork/patch/34002/)
So, generic arm64 linuxapp build is broken.

Should the deprecation notice have been accompanied with marking that function as __rte_deprecated?

Quoted 9 lines

Regards,
Pavan.

Quoted 3 lines · Anatoly Burakov

This patchset introduces dynamic memory allocation for DPDK (aka memory hotplug). Based upon RFC submitted in December [1].

Quoted 10 lines

Quoted 6 lines

Dependencies (to be applied in specified order):
- IPC bugfixes patchset [2]
- IPC improvements patchset [3]
- IPC asynchronous request API patch [4]
- Function to return number of sockets [5]

<snip>
--

Quoted 1 line

2.7.4

--
Thanks,
Anatoly

Quoted 10 lines · Anatoly

Quoted 6 lines · Pavan Nikhilesh

Hi Anatoly,

I am trying to verify this patchset and have encountered few issues.

Few -Werror=maybe-uninitialized errors in eal_memalloc.c/eal_memory.c/ eal_common_memzone.c files.

Thanks for the heads up, i'll fix those in the next revision. Out of curiousity, which compiler version are you using?

I'm using gcc 5.3.0.

Quoted 46 lines

Quoted 46 lines

lib/librte_eal/common/eal_common_memzone.c +1 -1

index a7cfdaf03..ad4413507 100644
@@ -321,7 +321,7 @@ rte_memzone_free(const struct rte_memzone *mz)
         struct rte_fbarray *arr;
         struct rte_memzone *found_mz;
         int ret = 0;
-       void *addr;
+       void *addr = NULL;
         unsigned idx;
 
         if (mz == NULL)

lib/librte_eal/linuxapp/eal/eal_memalloc.c +1 -1

index 1008faed6..32b0d5133 100644
@@ -570,7 +570,7 @@ eal_memalloc_alloc_page_bulk(struct rte_memseg **ms, int n,
         unsigned int msl_idx;
         int cur_idx, start_idx, end_idx, i, j, ret = -1;
  #ifdef RTE_EAL_NUMA_AWARE_HUGEPAGES
-       bool have_numa;
+       bool have_numa = false;
         int oldpolicy;
         struct bitmask *oldmask = numa_allocate_nodemask();
  #endif

lib/librte_eal/linuxapp/eal/eal_memory.c +2 -2

index f74291fb6..d37b4a59b 100644
@@ -1386,9 +1386,9 @@ eal_legacy_hugepage_attach(void)
         struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
         struct hugepage_file *hp = NULL;
         unsigned int num_hp = 0;
-       unsigned int i;
+       unsigned int i = 0;
         int ms_idx, msl_idx;
-       unsigned int cur_seg, max_seg;
+       unsigned int cur_seg, max_seg = 0;
         off_t size = 0;
         int fd, fd_hugepage = -1;
 

Quoted 8 lines

Quoted 5 lines

@Hemanth
Also, this patchset breaks dpaa/dpaa2 bus drivers (they rely on rte_eal_get_physmem_layout that is depricated
http://dpdk.org/dev/patchwork/patch/34002/)
So, generic arm64 linuxapp build is broken.

Should the deprecation notice have been accompanied with marking that function as __rte_deprecated?

Yup that's the general sequence.

Quoted 24 lines

Quoted 9 lines

Regards,
Pavan.

Quoted 3 lines · Anatoly Burakov

This patchset introduces dynamic memory allocation for DPDK (aka memory hotplug). Based upon RFC submitted in December [1].

Quoted 10 lines

Quoted 6 lines

Dependencies (to be applied in specified order):
- IPC bugfixes patchset [2]
- IPC improvements patchset [3]
- IPC asynchronous request API patch [4]
- Function to return number of sockets [5]

<snip>
--

Quoted 1 line

2.7.4

--
Thanks,
Anatoly

Hi Anatoly,

We are currently facing issues with running testpmd on thunderx platform. The issue seems to be with vfio

EAL: Detected 24 lcore(s)
EAL: Detected 1 NUMA nodes
EAL: No free hugepages reported in hugepages-2048kB
EAL: Multi-process socket /var/run/.rte_unix
EAL: Probing VFIO support...
EAL: VFIO support initialized
EAL: VFIO support not initialized

<snip>

EAL:   probe driver: 177d:a053 octeontx_fpavf
EAL: PCI device 0001:01:00.1 on NUMA socket 0
EAL:   probe driver: 177d:a034 net_thunderx
EAL:   using IOMMU type 1 (Type 1)
EAL:   cannot set up DMA remapping, error 22 (Invalid argument)
EAL:   0001:01:00.1 DMA remapping failed, error 22 (Invalid argument)
EAL: Requested device 0001:01:00.1 cannot be used
EAL: PCI device 0001:01:00.2 on NUMA socket 0
<snip>
testpmd: No probed ethernet devices
testpmd: create a new mbuf pool <mbuf_pool_socket_0>: n=251456, size=2176, socket=0
testpmd: preferred mempool ops selected: ring_mp_mc
EAL:   VFIO support not initialized
EAL:   VFIO support not initialized
EAL:   VFIO support not initialized

Done

This is because rte_service_init() calls rte_calloc() before rte_bus_probe() and vfio_dma_mem_map fails because iommu type is not set.

Call stack:
gdb) bt

#0  vfio_dma_mem_map (vaddr=281439006359552, iova=11274289152, len=536870912, do_map=1) at /root/clean/dpdk/lib/librte_eal/linuxapp/eal/eal_vfio.c:967
#1  0x00000000004fd974 in rte_vfio_dma_map (vaddr=281439006359552, iova=11274289152, len=536870912) at /root/clean/dpdk/lib/librte_eal/linuxapp/eal/eal_vfio.c:988
#2  0x00000000004fbe78 in vfio_mem_event_callback (type=RTE_MEM_EVENT_ALLOC, addr=0xfff7a0000000, len=536870912) at /root/clean/dpdk/lib/librte_eal/linuxapp/eal/eal_vfio.c:240
#3  0x00000000005070ac in eal_memalloc_notify (event=RTE_MEM_EVENT_ALLOC, start=0xfff7a0000000, len=536870912) at /root/clean/dpdk/lib/librte_eal/common/eal_common_memalloc.c:177
#4  0x0000000000515c98 in try_expand_heap_primary (heap=0xffffb7fb167c, pg_sz=536870912, elt_size=8192, socket=0, flags=0, align=128, bound=0, contig=false) at /root/clean/dpdk/lib/librte_eal/common/malloc_heap.c:247
#5  0x0000000000515e94 in try_expand_heap (heap=0xffffb7fb167c, pg_sz=536870912, elt_size=8192, socket=0, flags=0, align=128, bound=0, contig=false) at /root/clean/dpdk/lib/librte_eal/common/malloc_heap.c:327
#6  0x00000000005163a0 in alloc_more_mem_on_socket (heap=0xffffb7fb167c, size=8192, socket=0, flags=0, align=128, bound=0, contig=false) at /root/clean/dpdk/lib/librte_eal/common/malloc_heap.c:455
#7  0x0000000000516514 in heap_alloc_on_socket (type=0x85bf90 "rte_services", size=8192, socket=0, flags=0, align=128, bound=0, contig=false) at /root/clean/dpdk/lib/librte_eal/common/malloc_heap.c:491
#8  0x0000000000516664 in malloc_heap_alloc (type=0x85bf90 "rte_services", size=8192, socket_arg=-1, flags=0, align=128, bound=0, contig=false) at /root/clean/dpdk/lib/librte_eal/common/malloc_heap.c:527
#9  0x0000000000513b54 in rte_malloc_socket (type=0x85bf90 "rte_services", size=8192, align=128, socket_arg=-1) at /root/clean/dpdk/lib/librte_eal/common/rte_malloc.c:54

#10 0x0000000000513bc8 in rte_zmalloc_socket (type=0x85bf90 "rte_services", size=8192, align=128, socket=-1) at /root/clean/dpdk/lib/librte_eal/common/rte_malloc.c:72 #11 0x0000000000513c00 in rte_zmalloc (type=0x85bf90 "rte_services", size=8192, align=128) at /root/clean/dpdk/lib/librte_eal/common/rte_malloc.c:81 #12 0x0000000000513c90 in rte_calloc (type=0x85bf90 "rte_services", num=64, size=128, align=128) at /root/clean/dpdk/lib/librte_eal/common/rte_malloc.c:99 #13 0x0000000000518cec in rte_service_init () at /root/clean/dpdk/lib/librte_eal/common/rte_service.c:81
#14 0x00000000004f55f4 in rte_eal_init (argc=3, argv=0xfffffffff488) at /root/clean/dpdk/lib/librte_eal/linuxapp/eal/eal.c:959
#15 0x000000000045af5c in main (argc=3, argv=0xfffffffff488) at /root/clean/dpdk/app/test-pmd/testpmd.c:2483

Also, I have tried running with --legacy-mem but I'm stuck in pci_find_max_end_va loop because rte_fbarray_find_next_used always return
0. >

HugePages_Total:      15
HugePages_Free:       11
HugePages_Rsvd:        0
HugePages_Surp:        0
Hugepagesize:     524288 kB

Call Stack:
(gdb) bt

#0  find_next (arr=0xffffb7fb009c, start=0, used=true) at /root/clean/dpdk/lib/librte_eal/common/eal_common_fbarray.c:248
#1  0x00000000005132a8 in rte_fbarray_find_next_used (arr=0xffffb7fb009c, start=0) at /root/clean/dpdk/lib/librte_eal/common/eal_common_fbarray.c:700
#2  0x000000000052d030 in pci_find_max_end_va () at /root/clean/dpdk/drivers/bus/pci/linux/pci.c:138
#3  0x0000000000530ab8 in pci_vfio_map_resource_primary (dev=0xeae700) at /root/clean/dpdk/drivers/bus/pci/linux/pci_vfio.c:499
#4  0x0000000000530ffc in pci_vfio_map_resource (dev=0xeae700) at /root/clean/dpdk/drivers/bus/pci/linux/pci_vfio.c:601
#5  0x000000000052ce90 in rte_pci_map_device (dev=0xeae700) at /root/clean/dpdk/drivers/bus/pci/linux/pci.c:75
#6  0x0000000000531a20 in rte_pci_probe_one_driver (dr=0x997e20 <rte_nicvf_pmd>, dev=0xeae700) at /root/clean/dpdk/drivers/bus/pci/pci_common.c:164
#7  0x0000000000531c68 in pci_probe_all_drivers (dev=0xeae700) at /root/clean/dpdk/drivers/bus/pci/pci_common.c:249
#8  0x0000000000531f68 in rte_pci_probe () at /root/clean/dpdk/drivers/bus/pci/pci_common.c:359
#9  0x000000000050a140 in rte_bus_probe () at /root/clean/dpdk/lib/librte_eal/common/eal_common_bus.c:98

#10 0x00000000004f55f4 in rte_eal_init (argc=1, argv=0xfffffffff498) at /root/clean/dpdk/lib/librte_eal/linuxapp/eal/eal.c:967 #11 0x000000000045af5c in main (argc=1, argv=0xfffffffff498) at /root/clean/dpdk/app/test-pmd/testpmd.c:2483

Am I missing something here?

Quoted 80 lines · Pavan Nikhilesh

Hi Anatoly,

We are currently facing issues with running testpmd on thunderx platform. The issue seems to be with vfio

EAL: Detected 24 lcore(s)
EAL: Detected 1 NUMA nodes
EAL: No free hugepages reported in hugepages-2048kB
EAL: Multi-process socket /var/run/.rte_unix
EAL: Probing VFIO support...
EAL: VFIO support initialized
EAL:   VFIO support not initialized

<snip>

EAL:   probe driver: 177d:a053 octeontx_fpavf
EAL: PCI device 0001:01:00.1 on NUMA socket 0
EAL:   probe driver: 177d:a034 net_thunderx
EAL:   using IOMMU type 1 (Type 1)
EAL:   cannot set up DMA remapping, error 22 (Invalid argument)
EAL:   0001:01:00.1 DMA remapping failed, error 22 (Invalid argument)
EAL: Requested device 0001:01:00.1 cannot be used
EAL: PCI device 0001:01:00.2 on NUMA socket 0
<snip>
testpmd: No probed ethernet devices
testpmd: create a new mbuf pool <mbuf_pool_socket_0>: n=251456, size=2176, socket=0
testpmd: preferred mempool ops selected: ring_mp_mc
EAL:   VFIO support not initialized
EAL:   VFIO support not initialized
EAL:   VFIO support not initialized
Done

This is because rte_service_init() calls rte_calloc() before rte_bus_probe() and vfio_dma_mem_map fails because iommu type is not set.

Call stack:
gdb) bt
#0  vfio_dma_mem_map (vaddr=281439006359552, iova=11274289152, len=536870912, do_map=1) at /root/clean/dpdk/lib/librte_eal/linuxapp/eal/eal_vfio.c:967
#1  0x00000000004fd974 in rte_vfio_dma_map (vaddr=281439006359552, iova=11274289152, len=536870912) at /root/clean/dpdk/lib/librte_eal/linuxapp/eal/eal_vfio.c:988
#2  0x00000000004fbe78 in vfio_mem_event_callback (type=RTE_MEM_EVENT_ALLOC, addr=0xfff7a0000000, len=536870912) at /root/clean/dpdk/lib/librte_eal/linuxapp/eal/eal_vfio.c:240
#3  0x00000000005070ac in eal_memalloc_notify (event=RTE_MEM_EVENT_ALLOC, start=0xfff7a0000000, len=536870912) at /root/clean/dpdk/lib/librte_eal/common/eal_common_memalloc.c:177 #4  0x0000000000515c98 in try_expand_heap_primary (heap=0xffffb7fb167c, pg_sz=536870912, elt_size=8192, socket=0, flags=0, align=128, bound=0, contig=false) at /root/clean/dpdk/lib/librte_eal/common/malloc_heap.c:247 #5  0x0000000000515e94 in try_expand_heap (heap=0xffffb7fb167c, pg_sz=536870912, elt_size=8192, socket=0, flags=0, align=128, bound=0, contig=false) at /root/clean/dpdk/lib/librte_eal/common/malloc_heap.c:327 #6  0x00000000005163a0 in alloc_more_mem_on_socket (heap=0xffffb7fb167c, size=8192, socket=0, flags=0, align=128, bound=0, contig=false) at /root/clean/dpdk/lib/librte_eal/common/malloc_heap.c:455 #7  0x0000000000516514 in heap_alloc_on_socket (type=0x85bf90 "rte_services", size=8192, socket=0, flags=0, align=128, bound=0, contig=false) at /root/clean/dpdk/lib/librte_eal/common/malloc_heap.c:491 #8  0x0000000000516664 in malloc_heap_alloc (type=0x85bf90 "rte_services", size=8192, socket_arg=-1, flags=0, align=128, bound=0, contig=false) at /root/clean/dpdk/lib/librte_eal/common/malloc_heap.c:527 #9  0x0000000000513b54 in rte_malloc_socket (type=0x85bf90 "rte_services", size=8192, align=128, socket_arg=-1) at /root/clean/dpdk/lib/librte_eal/common/rte_malloc.c:54
#10 0x0000000000513bc8 in rte_zmalloc_socket (type=0x85bf90 "rte_services", size=8192, align=128, socket=-1) at /root/clean/dpdk/lib/librte_eal/common/rte_malloc.c:72
#11 0x0000000000513c00 in rte_zmalloc (type=0x85bf90 "rte_services", size=8192, align=128) at /root/clean/dpdk/lib/librte_eal/common/rte_malloc.c:81
#12 0x0000000000513c90 in rte_calloc (type=0x85bf90 "rte_services", num=64, size=128, align=128) at /root/clean/dpdk/lib/librte_eal/common/rte_malloc.c:99
#13 0x0000000000518cec in rte_service_init () at /root/clean/dpdk/lib/librte_eal/common/rte_service.c:81
#14 0x00000000004f55f4 in rte_eal_init (argc=3, argv=0xfffffffff488) at /root/clean/dpdk/lib/librte_eal/linuxapp/eal/eal.c:959
#15 0x000000000045af5c in main (argc=3, argv=0xfffffffff488) at /root/clean/dpdk/app/test-pmd/testpmd.c:2483

Also, I have tried running with --legacy-mem but I'm stuck in pci_find_max_end_va loop  because rte_fbarray_find_next_used always return
0. >
HugePages_Total:      15
HugePages_Free:       11
HugePages_Rsvd:        0
HugePages_Surp:        0
Hugepagesize:     524288 kB

Call Stack:
(gdb) bt
#0  find_next (arr=0xffffb7fb009c, start=0, used=true) at /root/clean/dpdk/lib/librte_eal/common/eal_common_fbarray.c:248
#1  0x00000000005132a8 in rte_fbarray_find_next_used (arr=0xffffb7fb009c, start=0) at /root/clean/dpdk/lib/librte_eal/common/eal_common_fbarray.c:700 #2  0x000000000052d030 in pci_find_max_end_va () at /root/clean/dpdk/drivers/bus/pci/linux/pci.c:138
#3  0x0000000000530ab8 in pci_vfio_map_resource_primary (dev=0xeae700) at /root/clean/dpdk/drivers/bus/pci/linux/pci_vfio.c:499 #4  0x0000000000530ffc in pci_vfio_map_resource (dev=0xeae700) at /root/clean/dpdk/drivers/bus/pci/linux/pci_vfio.c:601
#5  0x000000000052ce90 in rte_pci_map_device (dev=0xeae700) at /root/clean/dpdk/drivers/bus/pci/linux/pci.c:75
#6  0x0000000000531a20 in rte_pci_probe_one_driver (dr=0x997e20 <rte_nicvf_pmd>, dev=0xeae700) at /root/clean/dpdk/drivers/bus/pci/pci_common.c:164 #7  0x0000000000531c68 in pci_probe_all_drivers (dev=0xeae700) at /root/clean/dpdk/drivers/bus/pci/pci_common.c:249
#8  0x0000000000531f68 in rte_pci_probe () at /root/clean/dpdk/drivers/bus/pci/pci_common.c:359
#9  0x000000000050a140 in rte_bus_probe () at /root/clean/dpdk/lib/librte_eal/common/eal_common_bus.c:98
#10 0x00000000004f55f4 in rte_eal_init (argc=1, argv=0xfffffffff498) at /root/clean/dpdk/lib/librte_eal/linuxapp/eal/eal.c:967 #11 0x000000000045af5c in main (argc=1, argv=0xfffffffff498) at /root/clean/dpdk/app/test-pmd/testpmd.c:2483

Quoted 1 line

Am I missing something here?

I'll look into those, thanks!

Btw, i've now set up a github repo with the patchset applied:

https://github.com/anatolyburakov/dpdk

I will be pushing quick fixes there before spinning new revisions, so we can discover and fix bugs more rapidly. I'll fix compile issues reported earlier, then i'll take a look at your issues. The latter one seems like a typo, the former is probably a matter of moving things around a bit.

(also, pull requests welcome if you find it easier to fix things yourself and submit patches against my tree!)

Thanks for testing.

Quoted 87 lines · Anatoly

Quoted 80 lines · Pavan Nikhilesh

Hi Anatoly,

We are currently facing issues with running testpmd on thunderx platform. The issue seems to be with vfio

EAL: Detected 24 lcore(s)
EAL: Detected 1 NUMA nodes
EAL: No free hugepages reported in hugepages-2048kB
EAL: Multi-process socket /var/run/.rte_unix
EAL: Probing VFIO support...
EAL: VFIO support initialized
EAL:   VFIO support not initialized

<snip>

EAL:   probe driver: 177d:a053 octeontx_fpavf
EAL: PCI device 0001:01:00.1 on NUMA socket 0
EAL:   probe driver: 177d:a034 net_thunderx
EAL:   using IOMMU type 1 (Type 1)
EAL:   cannot set up DMA remapping, error 22 (Invalid argument)
EAL:   0001:01:00.1 DMA remapping failed, error 22 (Invalid argument)
EAL: Requested device 0001:01:00.1 cannot be used
EAL: PCI device 0001:01:00.2 on NUMA socket 0
<snip>
testpmd: No probed ethernet devices
testpmd: create a new mbuf pool <mbuf_pool_socket_0>: n=251456, size=2176, socket=0
testpmd: preferred mempool ops selected: ring_mp_mc
EAL:   VFIO support not initialized
EAL:   VFIO support not initialized
EAL:   VFIO support not initialized
Done

This is because rte_service_init() calls rte_calloc() before rte_bus_probe() and vfio_dma_mem_map fails because iommu type is not set.

Call stack:
gdb) bt
#0  vfio_dma_mem_map (vaddr=281439006359552, iova=11274289152, len=536870912, do_map=1) at /root/clean/dpdk/lib/librte_eal/linuxapp/eal/eal_vfio.c:967
#1  0x00000000004fd974 in rte_vfio_dma_map (vaddr=281439006359552, iova=11274289152, len=536870912) at /root/clean/dpdk/lib/librte_eal/linuxapp/eal/eal_vfio.c:988
#2  0x00000000004fbe78 in vfio_mem_event_callback (type=RTE_MEM_EVENT_ALLOC, addr=0xfff7a0000000, len=536870912) at /root/clean/dpdk/lib/librte_eal/linuxapp/eal/eal_vfio.c:240
#3  0x00000000005070ac in eal_memalloc_notify (event=RTE_MEM_EVENT_ALLOC, start=0xfff7a0000000, len=536870912) at /root/clean/dpdk/lib/librte_eal/common/eal_common_memalloc.c:177 #4  0x0000000000515c98 in try_expand_heap_primary (heap=0xffffb7fb167c, pg_sz=536870912, elt_size=8192, socket=0, flags=0, align=128, bound=0, contig=false) at /root/clean/dpdk/lib/librte_eal/common/malloc_heap.c:247 #5  0x0000000000515e94 in try_expand_heap (heap=0xffffb7fb167c, pg_sz=536870912, elt_size=8192, socket=0, flags=0, align=128, bound=0, contig=false) at /root/clean/dpdk/lib/librte_eal/common/malloc_heap.c:327 #6  0x00000000005163a0 in alloc_more_mem_on_socket (heap=0xffffb7fb167c, size=8192, socket=0, flags=0, align=128, bound=0, contig=false) at /root/clean/dpdk/lib/librte_eal/common/malloc_heap.c:455 #7  0x0000000000516514 in heap_alloc_on_socket (type=0x85bf90 "rte_services", size=8192, socket=0, flags=0, align=128, bound=0, contig=false) at /root/clean/dpdk/lib/librte_eal/common/malloc_heap.c:491 #8  0x0000000000516664 in malloc_heap_alloc (type=0x85bf90 "rte_services", size=8192, socket_arg=-1, flags=0, align=128, bound=0, contig=false) at /root/clean/dpdk/lib/librte_eal/common/malloc_heap.c:527 #9  0x0000000000513b54 in rte_malloc_socket (type=0x85bf90 "rte_services", size=8192, align=128, socket_arg=-1) at /root/clean/dpdk/lib/librte_eal/common/rte_malloc.c:54
#10 0x0000000000513bc8 in rte_zmalloc_socket (type=0x85bf90 "rte_services", size=8192, align=128, socket=-1) at /root/clean/dpdk/lib/librte_eal/common/rte_malloc.c:72
#11 0x0000000000513c00 in rte_zmalloc (type=0x85bf90 "rte_services", size=8192, align=128) at /root/clean/dpdk/lib/librte_eal/common/rte_malloc.c:81
#12 0x0000000000513c90 in rte_calloc (type=0x85bf90 "rte_services", num=64, size=128, align=128) at /root/clean/dpdk/lib/librte_eal/common/rte_malloc.c:99
#13 0x0000000000518cec in rte_service_init () at /root/clean/dpdk/lib/librte_eal/common/rte_service.c:81
#14 0x00000000004f55f4 in rte_eal_init (argc=3, argv=0xfffffffff488) at /root/clean/dpdk/lib/librte_eal/linuxapp/eal/eal.c:959
#15 0x000000000045af5c in main (argc=3, argv=0xfffffffff488) at /root/clean/dpdk/app/test-pmd/testpmd.c:2483

Also, I have tried running with --legacy-mem but I'm stuck in pci_find_max_end_va loop  because rte_fbarray_find_next_used always return
0. >
HugePages_Total:      15
HugePages_Free:       11
HugePages_Rsvd:        0
HugePages_Surp:        0
Hugepagesize:     524288 kB

Call Stack:
(gdb) bt
#0  find_next (arr=0xffffb7fb009c, start=0, used=true) at /root/clean/dpdk/lib/librte_eal/common/eal_common_fbarray.c:248
#1  0x00000000005132a8 in rte_fbarray_find_next_used (arr=0xffffb7fb009c, start=0) at /root/clean/dpdk/lib/librte_eal/common/eal_common_fbarray.c:700 #2  0x000000000052d030 in pci_find_max_end_va () at /root/clean/dpdk/drivers/bus/pci/linux/pci.c:138
#3  0x0000000000530ab8 in pci_vfio_map_resource_primary (dev=0xeae700) at /root/clean/dpdk/drivers/bus/pci/linux/pci_vfio.c:499 #4  0x0000000000530ffc in pci_vfio_map_resource (dev=0xeae700) at /root/clean/dpdk/drivers/bus/pci/linux/pci_vfio.c:601
#5  0x000000000052ce90 in rte_pci_map_device (dev=0xeae700) at /root/clean/dpdk/drivers/bus/pci/linux/pci.c:75
#6  0x0000000000531a20 in rte_pci_probe_one_driver (dr=0x997e20 <rte_nicvf_pmd>, dev=0xeae700) at /root/clean/dpdk/drivers/bus/pci/pci_common.c:164 #7  0x0000000000531c68 in pci_probe_all_drivers (dev=0xeae700) at /root/clean/dpdk/drivers/bus/pci/pci_common.c:249
#8  0x0000000000531f68 in rte_pci_probe () at /root/clean/dpdk/drivers/bus/pci/pci_common.c:359
#9  0x000000000050a140 in rte_bus_probe () at /root/clean/dpdk/lib/librte_eal/common/eal_common_bus.c:98
#10 0x00000000004f55f4 in rte_eal_init (argc=1, argv=0xfffffffff498) at /root/clean/dpdk/lib/librte_eal/linuxapp/eal/eal.c:967 #11 0x000000000045af5c in main (argc=1, argv=0xfffffffff498) at /root/clean/dpdk/app/test-pmd/testpmd.c:2483

Quoted 1 line

Am I missing something here?

I'll look into those, thanks!

Btw, i've now set up a github repo with the patchset applied:

Quoted 8 lines

https://github.com/anatolyburakov/dpdk

I will be pushing quick fixes there before spinning new revisions, so we can discover and fix bugs more rapidly. I'll fix compile issues reported earlier, then i'll take a look at your issues. The latter one seems like a typo, the former is probably a matter of moving things around a bit.

(also, pull requests welcome if you find it easier to fix things yourself and submit patches against my tree!)

Thanks for testing.

I've looked into the failures.

The VFIO one is not actually a failure. It only prints out errors because rte_malloc is called before VFIO is initialized. However, once VFIO *is* initialized, all of that memory would be added to VFIO, so these error messages are harmless. Regardless, i've added a check to see if init is finished before printing out those errors, so they won't be printed out any more.

Second one is a typo on my part that got lost in one of the rebases.

I've pushed fixes for both into the github repo.

Quoted 169 lines · Anatoly

Quoted 151 lines · Anatoly

Quoted 136 lines · Pavan Nikhilesh

Hi Anatoly,

We are currently facing issues with running testpmd on thunderx platform.
The issue seems to be with vfio

EAL: Detected 24 lcore(s)
EAL: Detected 1 NUMA nodes
EAL: No free hugepages reported in hugepages-2048kB
EAL: Multi-process socket /var/run/.rte_unix
EAL: Probing VFIO support...
EAL: VFIO support initialized
EAL:   VFIO support not initialized

<snip>

EAL:   probe driver: 177d:a053 octeontx_fpavf
EAL: PCI device 0001:01:00.1 on NUMA socket 0
EAL:   probe driver: 177d:a034 net_thunderx
EAL:   using IOMMU type 1 (Type 1)
EAL:   cannot set up DMA remapping, error 22 (Invalid argument) EAL:   0001:01:00.1 DMA remapping failed, error 22 (Invalid argument)
EAL: Requested device 0001:01:00.1 cannot be used
EAL: PCI device 0001:01:00.2 on NUMA socket 0
<snip>
testpmd: No probed ethernet devices
testpmd: create a new mbuf pool <mbuf_pool_socket_0>: n=251456, size=2176, socket=0
testpmd: preferred mempool ops selected: ring_mp_mc
EAL:   VFIO support not initialized
EAL:   VFIO support not initialized
EAL:   VFIO support not initialized
Done

This is because rte_service_init() calls rte_calloc() before rte_bus_probe() and vfio_dma_mem_map fails because iommu type is not set.

Call stack:
gdb) bt
#0  vfio_dma_mem_map (vaddr=281439006359552, iova=11274289152, len=536870912, do_map=1) at /root/clean/dpdk/lib/librte_eal/linuxapp/eal/eal_vfio.c:967 #1  0x00000000004fd974 in rte_vfio_dma_map (vaddr=281439006359552, iova=11274289152, len=536870912) at /root/clean/dpdk/lib/librte_eal/linuxapp/eal/eal_vfio.c:988 #2  0x00000000004fbe78 in vfio_mem_event_callback (type=RTE_MEM_EVENT_ALLOC, addr=0xfff7a0000000, len=536870912) at /root/clean/dpdk/lib/librte_eal/linuxapp/eal/eal_vfio.c:240 #3  0x00000000005070ac in eal_memalloc_notify (event=RTE_MEM_EVENT_ALLOC, start=0xfff7a0000000, len=536870912) at /root/clean/dpdk/lib/librte_eal/common/eal_common_memalloc.c:177 #4  0x0000000000515c98 in try_expand_heap_primary (heap=0xffffb7fb167c, pg_sz=536870912, elt_size=8192, socket=0, flags=0, align=128, bound=0, contig=false) at /root/clean/dpdk/lib/librte_eal/common/malloc_heap.c:247 #5  0x0000000000515e94 in try_expand_heap (heap=0xffffb7fb167c, pg_sz=536870912, elt_size=8192, socket=0, flags=0, align=128, bound=0, contig=false) at /root/clean/dpdk/lib/librte_eal/common/malloc_heap.c:327 #6  0x00000000005163a0 in alloc_more_mem_on_socket (heap=0xffffb7fb167c, size=8192, socket=0, flags=0, align=128, bound=0, contig=false) at /root/clean/dpdk/lib/librte_eal/common/malloc_heap.c:455 #7  0x0000000000516514 in heap_alloc_on_socket (type=0x85bf90 "rte_services", size=8192, socket=0, flags=0, align=128, bound=0, contig=false) at /root/clean/dpdk/lib/librte_eal/common/malloc_heap.c:491 #8  0x0000000000516664 in malloc_heap_alloc (type=0x85bf90 "rte_services", size=8192, socket_arg=-1, flags=0, align=128, bound=0, contig=false) at /root/clean/dpdk/lib/librte_eal/common/malloc_heap.c:527 #9  0x0000000000513b54 in rte_malloc_socket (type=0x85bf90 "rte_services", size=8192, align=128, socket_arg=-1) at /root/clean/dpdk/lib/librte_eal/common/rte_malloc.c:54 #10 0x0000000000513bc8 in rte_zmalloc_socket (type=0x85bf90 "rte_services", size=8192, align=128, socket=-1) at /root/clean/dpdk/lib/librte_eal/common/rte_malloc.c:72 #11 0x0000000000513c00 in rte_zmalloc (type=0x85bf90 "rte_services", size=8192, align=128) at /root/clean/dpdk/lib/librte_eal/common/rte_malloc.c:81 #12 0x0000000000513c90 in rte_calloc (type=0x85bf90 "rte_services", num=64, size=128, align=128) at /root/clean/dpdk/lib/librte_eal/common/rte_malloc.c:99 #13 0x0000000000518cec in rte_service_init () at /root/clean/dpdk/lib/librte_eal/common/rte_service.c:81 #14 0x00000000004f55f4 in rte_eal_init (argc=3, argv=0xfffffffff488) at /root/clean/dpdk/lib/librte_eal/linuxapp/eal/eal.c:959 #15 0x000000000045af5c in main (argc=3, argv=0xfffffffff488) at /root/clean/dpdk/app/test-pmd/testpmd.c:2483

Also, I have tried running with --legacy-mem but I'm stuck in pci_find_max_end_va loop  because rte_fbarray_find_next_used always return
0. >
HugePages_Total:      15
HugePages_Free:       11
HugePages_Rsvd:        0
HugePages_Surp:        0
Hugepagesize:     524288 kB

Call Stack:
(gdb) bt
#0  find_next (arr=0xffffb7fb009c, start=0, used=true) at /root/clean/dpdk/lib/librte_eal/common/eal_common_fbarray.c:248 #1  0x00000000005132a8 in rte_fbarray_find_next_used (arr=0xffffb7fb009c, start=0) at /root/clean/dpdk/lib/librte_eal/common/eal_common_fbarray.c:700 #2  0x000000000052d030 in pci_find_max_end_va () at /root/clean/dpdk/drivers/bus/pci/linux/pci.c:138
#3  0x0000000000530ab8 in pci_vfio_map_resource_primary (dev=0xeae700) at /root/clean/dpdk/drivers/bus/pci/linux/pci_vfio.c:499 #4  0x0000000000530ffc in pci_vfio_map_resource (dev=0xeae700) at /root/clean/dpdk/drivers/bus/pci/linux/pci_vfio.c:601 #5  0x000000000052ce90 in rte_pci_map_device (dev=0xeae700) at /root/clean/dpdk/drivers/bus/pci/linux/pci.c:75
#6  0x0000000000531a20 in rte_pci_probe_one_driver (dr=0x997e20 <rte_nicvf_pmd>, dev=0xeae700) at /root/clean/dpdk/drivers/bus/pci/pci_common.c:164
#7  0x0000000000531c68 in pci_probe_all_drivers (dev=0xeae700) at /root/clean/dpdk/drivers/bus/pci/pci_common.c:249 #8  0x0000000000531f68 in rte_pci_probe () at /root/clean/dpdk/drivers/bus/pci/pci_common.c:359
#9  0x000000000050a140 in rte_bus_probe () at /root/clean/dpdk/lib/librte_eal/common/eal_common_bus.c:98 #10 0x00000000004f55f4 in rte_eal_init (argc=1, argv=0xfffffffff498) at /root/clean/dpdk/lib/librte_eal/linuxapp/eal/eal.c:967 #11 0x000000000045af5c in main (argc=1, argv=0xfffffffff498) at /root/clean/dpdk/app/test-pmd/testpmd.c:2483

Quoted 1 line

Am I missing something here?

I'll look into those, thanks!

Btw, i've now set up a github repo with the patchset applied:

https://github.com/anatolyburakov/dpdk

I will be pushing quick fixes there before spinning new revisions, so we can discover and fix bugs more rapidly. I'll fix compile issues reported earlier, then i'll take a look at your issues. The latter one seems like a typo, the former is probably a matter of moving things around a bit.

Quoted 5 lines

(also, pull requests welcome if you find it easier to fix things yourself and submit patches against my tree!)

Thanks for testing.

I've looked into the failures.

The VFIO one is not actually a failure. It only prints out errors because rte_malloc is called before VFIO is initialized. However, once VFIO *is* initialized, all of that memory would be added to VFIO, so these error messages are harmless. Regardless, i've added a check to see if init is finished before printing out those errors, so they won't be printed out any more.

Second one is a typo on my part that got lost in one of the rebases.

Quoted 2 lines

I've pushed fixes for both into the github repo.

Although i do wonder where do the DMA remapping errors come from. The error message says "invalid argument", so that doesn't come from rte_service or anything to do with rte_malloc - this is us not providing valid arguments to VFIO. I'm not seeing these errors on my system. I'll check on others to be sure.

--
Thanks,
Anatoly

Quoted 170 lines · Anatoly

Quoted 169 lines · Anatoly

Quoted 151 lines · Anatoly

Quoted 136 lines · Pavan Nikhilesh

Hi Anatoly,

We are currently facing issues with running testpmd on thunderx platform.
The issue seems to be with vfio

EAL: Detected 24 lcore(s)
EAL: Detected 1 NUMA nodes
EAL: No free hugepages reported in hugepages-2048kB
EAL: Multi-process socket /var/run/.rte_unix
EAL: Probing VFIO support...
EAL: VFIO support initialized
EAL:   VFIO support not initialized

<snip>

EAL:   probe driver: 177d:a053 octeontx_fpavf
EAL: PCI device 0001:01:00.1 on NUMA socket 0
EAL:   probe driver: 177d:a034 net_thunderx
EAL:   using IOMMU type 1 (Type 1)
EAL:   cannot set up DMA remapping, error 22 (Invalid argument) EAL:   0001:01:00.1 DMA remapping failed, error 22 (Invalid argument)
EAL: Requested device 0001:01:00.1 cannot be used
EAL: PCI device 0001:01:00.2 on NUMA socket 0
<snip>
testpmd: No probed ethernet devices
testpmd: create a new mbuf pool <mbuf_pool_socket_0>: n=251456, size=2176, socket=0
testpmd: preferred mempool ops selected: ring_mp_mc
EAL:   VFIO support not initialized
EAL:   VFIO support not initialized
EAL:   VFIO support not initialized
Done

This is because rte_service_init() calls rte_calloc() before rte_bus_probe() and vfio_dma_mem_map fails because iommu type is not set.

Call stack:
gdb) bt
#0  vfio_dma_mem_map (vaddr=281439006359552, iova=11274289152, len=536870912, do_map=1) at /root/clean/dpdk/lib/librte_eal/linuxapp/eal/eal_vfio.c:967 #1  0x00000000004fd974 in rte_vfio_dma_map (vaddr=281439006359552, iova=11274289152, len=536870912) at /root/clean/dpdk/lib/librte_eal/linuxapp/eal/eal_vfio.c:988 #2  0x00000000004fbe78 in vfio_mem_event_callback (type=RTE_MEM_EVENT_ALLOC, addr=0xfff7a0000000, len=536870912) at /root/clean/dpdk/lib/librte_eal/linuxapp/eal/eal_vfio.c:240 #3  0x00000000005070ac in eal_memalloc_notify (event=RTE_MEM_EVENT_ALLOC, start=0xfff7a0000000, len=536870912) at /root/clean/dpdk/lib/librte_eal/common/eal_common_memalloc.c:177 #4  0x0000000000515c98 in try_expand_heap_primary (heap=0xffffb7fb167c, pg_sz=536870912, elt_size=8192, socket=0, flags=0, align=128, bound=0, contig=false) at /root/clean/dpdk/lib/librte_eal/common/malloc_heap.c:247 #5  0x0000000000515e94 in try_expand_heap (heap=0xffffb7fb167c, pg_sz=536870912, elt_size=8192, socket=0, flags=0, align=128, bound=0, contig=false) at /root/clean/dpdk/lib/librte_eal/common/malloc_heap.c:327 #6  0x00000000005163a0 in alloc_more_mem_on_socket (heap=0xffffb7fb167c, size=8192, socket=0, flags=0, align=128, bound=0, contig=false) at /root/clean/dpdk/lib/librte_eal/common/malloc_heap.c:455 #7  0x0000000000516514 in heap_alloc_on_socket (type=0x85bf90 "rte_services", size=8192, socket=0, flags=0, align=128, bound=0, contig=false) at /root/clean/dpdk/lib/librte_eal/common/malloc_heap.c:491 #8  0x0000000000516664 in malloc_heap_alloc (type=0x85bf90 "rte_services", size=8192, socket_arg=-1, flags=0, align=128, bound=0, contig=false) at /root/clean/dpdk/lib/librte_eal/common/malloc_heap.c:527 #9  0x0000000000513b54 in rte_malloc_socket (type=0x85bf90 "rte_services", size=8192, align=128, socket_arg=-1) at /root/clean/dpdk/lib/librte_eal/common/rte_malloc.c:54 #10 0x0000000000513bc8 in rte_zmalloc_socket (type=0x85bf90 "rte_services", size=8192, align=128, socket=-1) at /root/clean/dpdk/lib/librte_eal/common/rte_malloc.c:72 #11 0x0000000000513c00 in rte_zmalloc (type=0x85bf90 "rte_services", size=8192, align=128) at /root/clean/dpdk/lib/librte_eal/common/rte_malloc.c:81 #12 0x0000000000513c90 in rte_calloc (type=0x85bf90 "rte_services", num=64, size=128, align=128) at /root/clean/dpdk/lib/librte_eal/common/rte_malloc.c:99 #13 0x0000000000518cec in rte_service_init () at /root/clean/dpdk/lib/librte_eal/common/rte_service.c:81 #14 0x00000000004f55f4 in rte_eal_init (argc=3, argv=0xfffffffff488) at /root/clean/dpdk/lib/librte_eal/linuxapp/eal/eal.c:959 #15 0x000000000045af5c in main (argc=3, argv=0xfffffffff488) at /root/clean/dpdk/app/test-pmd/testpmd.c:2483

Also, I have tried running with --legacy-mem but I'm stuck in pci_find_max_end_va loop  because rte_fbarray_find_next_used always return
0. >
HugePages_Total:      15
HugePages_Free:       11
HugePages_Rsvd:        0
HugePages_Surp:        0
Hugepagesize:     524288 kB

Call Stack:
(gdb) bt
#0  find_next (arr=0xffffb7fb009c, start=0, used=true) at /root/clean/dpdk/lib/librte_eal/common/eal_common_fbarray.c:248 #1  0x00000000005132a8 in rte_fbarray_find_next_used (arr=0xffffb7fb009c, start=0) at /root/clean/dpdk/lib/librte_eal/common/eal_common_fbarray.c:700 #2  0x000000000052d030 in pci_find_max_end_va () at /root/clean/dpdk/drivers/bus/pci/linux/pci.c:138
#3  0x0000000000530ab8 in pci_vfio_map_resource_primary (dev=0xeae700) at /root/clean/dpdk/drivers/bus/pci/linux/pci_vfio.c:499 #4  0x0000000000530ffc in pci_vfio_map_resource (dev=0xeae700) at /root/clean/dpdk/drivers/bus/pci/linux/pci_vfio.c:601 #5  0x000000000052ce90 in rte_pci_map_device (dev=0xeae700) at /root/clean/dpdk/drivers/bus/pci/linux/pci.c:75
#6  0x0000000000531a20 in rte_pci_probe_one_driver (dr=0x997e20 <rte_nicvf_pmd>, dev=0xeae700) at /root/clean/dpdk/drivers/bus/pci/pci_common.c:164
#7  0x0000000000531c68 in pci_probe_all_drivers (dev=0xeae700) at /root/clean/dpdk/drivers/bus/pci/pci_common.c:249 #8  0x0000000000531f68 in rte_pci_probe () at /root/clean/dpdk/drivers/bus/pci/pci_common.c:359
#9  0x000000000050a140 in rte_bus_probe () at /root/clean/dpdk/lib/librte_eal/common/eal_common_bus.c:98 #10 0x00000000004f55f4 in rte_eal_init (argc=1, argv=0xfffffffff498) at /root/clean/dpdk/lib/librte_eal/linuxapp/eal/eal.c:967 #11 0x000000000045af5c in main (argc=1, argv=0xfffffffff498) at /root/clean/dpdk/app/test-pmd/testpmd.c:2483

Quoted 1 line

Am I missing something here?

I'll look into those, thanks!

Btw, i've now set up a github repo with the patchset applied:

https://github.com/anatolyburakov/dpdk

I will be pushing quick fixes there before spinning new revisions, so we can discover and fix bugs more rapidly. I'll fix compile issues reported earlier, then i'll take a look at your issues. The latter one seems like a typo, the former is probably a matter of moving things around a bit.

Quoted 5 lines

(also, pull requests welcome if you find it easier to fix things yourself and submit patches against my tree!)

Thanks for testing.

I've looked into the failures.

The VFIO one is not actually a failure. It only prints out errors because rte_malloc is called before VFIO is initialized. However, once VFIO *is* initialized, all of that memory would be added to VFIO, so these error messages are harmless. Regardless, i've added a check to see if init is finished before printing out those errors, so they won't be printed out any more.

Second one is a typo on my part that got lost in one of the rebases.

Quoted 8 lines

Quoted 2 lines

I've pushed fixes for both into the github repo.

Although i do wonder where do the DMA remapping errors come from. The error message says "invalid argument", so that doesn't come from rte_service or anything to do with rte_malloc - this is us not providing valid arguments to VFIO. I'm not seeing these errors on my system. I'll check on others to be sure.

I have taken a look at the github tree the issues with VFIO are gone, Although compilation issues with dpaa/dpaa2 are still present due to their dependency on rte_eal_get_physmem_layout.

Quoted 5 lines

--
Thanks,
Anatoly

Thanks,
Pavan

Quoted 180 lines · Pavan Nikhilesh

Quoted 170 lines · Anatoly

Quoted 169 lines · Anatoly

Quoted 151 lines · Anatoly

Quoted 136 lines · Pavan Nikhilesh

Hi Anatoly,

We are currently facing issues with running testpmd on thunderx platform.
The issue seems to be with vfio

EAL: Detected 24 lcore(s)
EAL: Detected 1 NUMA nodes
EAL: No free hugepages reported in hugepages-2048kB
EAL: Multi-process socket /var/run/.rte_unix
EAL: Probing VFIO support...
EAL: VFIO support initialized
EAL:   VFIO support not initialized

<snip>

EAL:   probe driver: 177d:a053 octeontx_fpavf
EAL: PCI device 0001:01:00.1 on NUMA socket 0
EAL:   probe driver: 177d:a034 net_thunderx
EAL:   using IOMMU type 1 (Type 1)
EAL:   cannot set up DMA remapping, error 22 (Invalid argument) EAL:   0001:01:00.1 DMA remapping failed, error 22 (Invalid argument)
EAL: Requested device 0001:01:00.1 cannot be used
EAL: PCI device 0001:01:00.2 on NUMA socket 0
<snip>
testpmd: No probed ethernet devices
testpmd: create a new mbuf pool <mbuf_pool_socket_0>: n=251456, size=2176, socket=0
testpmd: preferred mempool ops selected: ring_mp_mc
EAL:   VFIO support not initialized
EAL:   VFIO support not initialized
EAL:   VFIO support not initialized
Done

This is because rte_service_init() calls rte_calloc() before rte_bus_probe() and vfio_dma_mem_map fails because iommu type is not set.

Call stack:
gdb) bt
#0  vfio_dma_mem_map (vaddr=281439006359552, iova=11274289152, len=536870912, do_map=1) at /root/clean/dpdk/lib/librte_eal/linuxapp/eal/eal_vfio.c:967 #1  0x00000000004fd974 in rte_vfio_dma_map (vaddr=281439006359552, iova=11274289152, len=536870912) at /root/clean/dpdk/lib/librte_eal/linuxapp/eal/eal_vfio.c:988 #2  0x00000000004fbe78 in vfio_mem_event_callback (type=RTE_MEM_EVENT_ALLOC, addr=0xfff7a0000000, len=536870912) at /root/clean/dpdk/lib/librte_eal/linuxapp/eal/eal_vfio.c:240 #3  0x00000000005070ac in eal_memalloc_notify (event=RTE_MEM_EVENT_ALLOC, start=0xfff7a0000000, len=536870912) at /root/clean/dpdk/lib/librte_eal/common/eal_common_memalloc.c:177 #4  0x0000000000515c98 in try_expand_heap_primary (heap=0xffffb7fb167c, pg_sz=536870912, elt_size=8192, socket=0, flags=0, align=128, bound=0, contig=false) at /root/clean/dpdk/lib/librte_eal/common/malloc_heap.c:247 #5  0x0000000000515e94 in try_expand_heap (heap=0xffffb7fb167c, pg_sz=536870912, elt_size=8192, socket=0, flags=0, align=128, bound=0, contig=false) at /root/clean/dpdk/lib/librte_eal/common/malloc_heap.c:327 #6  0x00000000005163a0 in alloc_more_mem_on_socket (heap=0xffffb7fb167c, size=8192, socket=0, flags=0, align=128, bound=0, contig=false) at /root/clean/dpdk/lib/librte_eal/common/malloc_heap.c:455 #7  0x0000000000516514 in heap_alloc_on_socket (type=0x85bf90 "rte_services", size=8192, socket=0, flags=0, align=128, bound=0, contig=false) at /root/clean/dpdk/lib/librte_eal/common/malloc_heap.c:491 #8  0x0000000000516664 in malloc_heap_alloc (type=0x85bf90 "rte_services", size=8192, socket_arg=-1, flags=0, align=128, bound=0, contig=false) at /root/clean/dpdk/lib/librte_eal/common/malloc_heap.c:527 #9  0x0000000000513b54 in rte_malloc_socket (type=0x85bf90 "rte_services", size=8192, align=128, socket_arg=-1) at /root/clean/dpdk/lib/librte_eal/common/rte_malloc.c:54 #10 0x0000000000513bc8 in rte_zmalloc_socket (type=0x85bf90 "rte_services", size=8192, align=128, socket=-1) at /root/clean/dpdk/lib/librte_eal/common/rte_malloc.c:72 #11 0x0000000000513c00 in rte_zmalloc (type=0x85bf90 "rte_services", size=8192, align=128) at /root/clean/dpdk/lib/librte_eal/common/rte_malloc.c:81 #12 0x0000000000513c90 in rte_calloc (type=0x85bf90 "rte_services", num=64, size=128, align=128) at /root/clean/dpdk/lib/librte_eal/common/rte_malloc.c:99 #13 0x0000000000518cec in rte_service_init () at /root/clean/dpdk/lib/librte_eal/common/rte_service.c:81 #14 0x00000000004f55f4 in rte_eal_init (argc=3, argv=0xfffffffff488) at /root/clean/dpdk/lib/librte_eal/linuxapp/eal/eal.c:959 #15 0x000000000045af5c in main (argc=3, argv=0xfffffffff488) at /root/clean/dpdk/app/test-pmd/testpmd.c:2483

Also, I have tried running with --legacy-mem but I'm stuck in pci_find_max_end_va loop  because rte_fbarray_find_next_used always return
0. >
HugePages_Total:      15
HugePages_Free:       11
HugePages_Rsvd:        0
HugePages_Surp:        0
Hugepagesize:     524288 kB

Call Stack:
(gdb) bt
#0  find_next (arr=0xffffb7fb009c, start=0, used=true) at /root/clean/dpdk/lib/librte_eal/common/eal_common_fbarray.c:248 #1  0x00000000005132a8 in rte_fbarray_find_next_used (arr=0xffffb7fb009c, start=0) at /root/clean/dpdk/lib/librte_eal/common/eal_common_fbarray.c:700 #2  0x000000000052d030 in pci_find_max_end_va () at /root/clean/dpdk/drivers/bus/pci/linux/pci.c:138
#3  0x0000000000530ab8 in pci_vfio_map_resource_primary (dev=0xeae700) at /root/clean/dpdk/drivers/bus/pci/linux/pci_vfio.c:499 #4  0x0000000000530ffc in pci_vfio_map_resource (dev=0xeae700) at /root/clean/dpdk/drivers/bus/pci/linux/pci_vfio.c:601 #5  0x000000000052ce90 in rte_pci_map_device (dev=0xeae700) at /root/clean/dpdk/drivers/bus/pci/linux/pci.c:75
#6  0x0000000000531a20 in rte_pci_probe_one_driver (dr=0x997e20 <rte_nicvf_pmd>, dev=0xeae700) at /root/clean/dpdk/drivers/bus/pci/pci_common.c:164
#7  0x0000000000531c68 in pci_probe_all_drivers (dev=0xeae700) at /root/clean/dpdk/drivers/bus/pci/pci_common.c:249 #8  0x0000000000531f68 in rte_pci_probe () at /root/clean/dpdk/drivers/bus/pci/pci_common.c:359
#9  0x000000000050a140 in rte_bus_probe () at /root/clean/dpdk/lib/librte_eal/common/eal_common_bus.c:98 #10 0x00000000004f55f4 in rte_eal_init (argc=1, argv=0xfffffffff498) at /root/clean/dpdk/lib/librte_eal/linuxapp/eal/eal.c:967 #11 0x000000000045af5c in main (argc=1, argv=0xfffffffff498) at /root/clean/dpdk/app/test-pmd/testpmd.c:2483

Quoted 1 line

Am I missing something here?

I'll look into those, thanks!

Btw, i've now set up a github repo with the patchset applied:

https://github.com/anatolyburakov/dpdk

I will be pushing quick fixes there before spinning new revisions, so we can discover and fix bugs more rapidly. I'll fix compile issues reported earlier, then i'll take a look at your issues. The latter one seems like a typo, the former is probably a matter of moving things around a bit.

Quoted 5 lines

(also, pull requests welcome if you find it easier to fix things yourself and submit patches against my tree!)

Thanks for testing.

I've looked into the failures.

The VFIO one is not actually a failure. It only prints out errors because rte_malloc is called before VFIO is initialized. However, once VFIO *is* initialized, all of that memory would be added to VFIO, so these error messages are harmless. Regardless, i've added a check to see if init is finished before printing out those errors, so they won't be printed out any more.

Second one is a typo on my part that got lost in one of the rebases.

Quoted 8 lines

Quoted 2 lines

I've pushed fixes for both into the github repo.

Although i do wonder where do the DMA remapping errors come from. The error message says "invalid argument", so that doesn't come from rte_service or anything to do with rte_malloc - this is us not providing valid arguments to VFIO. I'm not seeing these errors on my system. I'll check on others to be sure.

Quoted 3 lines

I have taken a look at the github tree the issues with VFIO are gone, Although compilation issues with dpaa/dpaa2 are still present due to their dependency on rte_eal_get_physmem_layout.

I've fixed the dpaa compile issue and pushed it to github. I've tried to keep the semantics the same as before, but i can't compile-test (let alone test-test) them as i don't have access to a system with dpaa bus.

Also, you might want to know that dpaa bus driver references RTE_LIBRTE_DPAA_MAX_CRYPTODEV which is only found in config/common_armv8a_linuxapp but is not present in base config. Not sure if that's an issue.

On 09-Mar-18 9:15 AM, Pavan Nikhilesh wrote:

Quoted 7 lines

I have taken a look at the github tree the issues with VFIO are gone, Although
compilation issues with dpaa/dpaa2 are still present due to their dependency on
rte_eal_get_physmem_layout.

I've fixed the dpaa compile issue and pushed it to github. I've tried to keep the semantics the same as before, but i can't compile-test (let alone test-test) them as i don't have access to a system with dpaa bus.

Also, you might want to know that dpaa bus driver references RTE_LIBRTE_DPAA_MAX_CRYPTODEV which is only found in config/common_armv8a_linuxapp but is not present in base config. Not sure if that's an issue.

Hello Anatoly,

Quoted 1 line · Anatoly

On 09-Mar-18 9:15 AM, Pavan Nikhilesh wrote:

[...]

Quoted 9 lines

Quoted 7 lines

I have taken a look at the github tree the issues with VFIO are gone, Although
compilation issues with dpaa/dpaa2 are still present due to their dependency on
rte_eal_get_physmem_layout.

Quoted 3 lines

I've fixed the dpaa compile issue and pushed it to github. I've tried to keep the semantics the same as before, but i can't compile-test (let alone test-test) them as i don't have access to a system with dpaa bus.

Thanks. I will have a look at this.

Quoted 6 lines

Also, you might want to know that dpaa bus driver references RTE_LIBRTE_DPAA_MAX_CRYPTODEV which is only found in config/common_armv8a_linuxapp but is not present in base config. Not sure if that's an issue.

Hello Anatoly,

Quoted 17 lines · Shreyansh Jain

Hello Anatoly,

Quoted 1 line · Anatoly

On 09-Mar-18 9:15 AM, Pavan Nikhilesh wrote:

[...]

Quoted 9 lines

Quoted 7 lines

I have taken a look at the github tree the issues with VFIO are gone, Although
compilation issues with dpaa/dpaa2 are still present due to their dependency on
rte_eal_get_physmem_layout.

Quoted 5 lines

Quoted 3 lines

I've fixed the dpaa compile issue and pushed it to github. I've tried to keep the semantics the same as before, but i can't compile-test (let alone test-test) them as i don't have access to a system with dpaa bus.

Thanks. I will have a look at this.

Just a heads-up, DPAA2 is broken on top-of-tree (github:
784e041f6b520) as of now:

--->8---
root@ls2088ardb:~/shreyansh/07_dpdk_memory# ./arm64-dpaa2-linuxapp-gcc/app/testpmd -c 0xE -n 1 --log-level=eal,8 --log-level=mem,8 -- -i --portmask=0x3

EAL: Detected lcore 0 as core 0 on socket 0
EAL: Detected lcore 1 as core 1 on socket 0
EAL: Detected lcore 2 as core 0 on socket 0
EAL: Detected lcore 3 as core 1 on socket 0
EAL: Detected lcore 4 as core 0 on socket 0
EAL: Detected lcore 5 as core 1 on socket 0
EAL: Detected lcore 6 as core 0 on socket 0
EAL: Detected lcore 7 as core 1 on socket 0
EAL: Support maximum 16 logical core(s) by configuration.
EAL: Detected 8 lcore(s)
EAL: Detected 1 NUMA nodes
EAL: VFIO PCI modules not loaded
EAL: DPAA Bus not present. Skipping.
EAL: Container: dprc.2 has VFIO iommu group id = 4
EAL: fslmc: Bus scan completed
EAL: Module /sys/module/rte_kni not found! error 2 (No such file or directory)
EAL: Multi-process socket /var/run/.rte_unix
EAL: Probing VFIO support...
EAL:   IOMMU type 1 (Type 1) is supported
EAL:   IOMMU type 7 (sPAPR) is not supported
EAL:   IOMMU type 8 (No-IOMMU) is not supported
EAL: VFIO support initialized
EAL: Mem event callback 'vfio_mem_event_clb' registered
EAL: Ask a virtual area of 0x2e000 bytes
EAL: Virtual area found at 0xffff86cae000 (size = 0x2e000)
EAL: Setting up physically contiguous memory...
EAL: Ask a virtual area of 0x1000 bytes
EAL: Virtual area found at 0xffff8873f000 (size = 0x1000)
EAL: Memseg list allocated: 0x100000kB at socket 0
EAL: Ask a virtual area of 0x800000000 bytes
EAL: Virtual area found at 0xfff780000000 (size = 0x800000000)
EAL: Ask a virtual area of 0x1000 bytes
EAL: Virtual area found at 0xffff8873e000 (size = 0x1000)
EAL: Memseg list allocated: 0x100000kB at socket 0
EAL: Ask a virtual area of 0x800000000 bytes
EAL: Virtual area found at 0xffef40000000 (size = 0x800000000)
EAL: Ask a virtual area of 0x1000 bytes
EAL: Virtual area found at 0xffff8873d000 (size = 0x1000)
EAL: Memseg list allocated: 0x100000kB at socket 0
EAL: Ask a virtual area of 0x800000000 bytes
EAL: Virtual area found at 0xffe700000000 (size = 0x800000000)
EAL: Ask a virtual area of 0x1000 bytes
EAL: Virtual area found at 0xffff8873c000 (size = 0x1000)
EAL: Memseg list allocated: 0x100000kB at socket 0
EAL: Ask a virtual area of 0x800000000 bytes
EAL: Virtual area found at 0xffdec0000000 (size = 0x800000000)
EAL: TSC frequency is ~25000 KHz
EAL: Master lcore 1 is ready (tid=88742110;cpuset=[1])
EAL: lcore 3 is ready (tid=85cab910;cpuset=[3])
EAL: lcore 2 is ready (tid=864ab910;cpuset=[2])
EAL: eal_memalloc_alloc_page_bulk(): couldn't find suitable memseg_list

error allocating rte services array
EAL: FATAL: rte_service_init() failed

EAL: rte_service_init() failed

PANIC in main():
Cannot init EAL
1: [./arm64-dpaa2-linuxapp-gcc/app/testpmd(rte_dump_stack+0x38) [0x4f37a8]] Aborted
--->8--

Above is an initial output - still investigating. I will keep you posted.

Hello Anatoly,

This is not necessarily right chain to reply to, but reusing this email for another issue in DPAA2 so that all issues can be at a single place.

Quoted 89 lines · Shreyansh Jain

Hello Anatoly,

Quoted 17 lines · Shreyansh Jain

Hello Anatoly,

Quoted 1 line · Anatoly

On 09-Mar-18 9:15 AM, Pavan Nikhilesh wrote:

[...]

Quoted 9 lines

Quoted 7 lines

I have taken a look at the github tree the issues with VFIO are gone, Although
compilation issues with dpaa/dpaa2 are still present due to their dependency on
rte_eal_get_physmem_layout.

Quoted 5 lines

Quoted 3 lines

I've fixed the dpaa compile issue and pushed it to github. I've tried to keep the semantics the same as before, but i can't compile-test (let alone test-test) them as i don't have access to a system with dpaa bus.

Thanks. I will have a look at this.

Just a heads-up, DPAA2 is broken on top-of-tree (github:
784e041f6b520) as of now:

--->8---
root@ls2088ardb:~/shreyansh/07_dpdk_memory# ./arm64-dpaa2-linuxapp-gcc/app/testpmd -c 0xE -n 1 --log-level=eal,8 --log-level=mem,8 -- -i --portmask=0x3

EAL: Detected lcore 0 as core 0 on socket 0
EAL: Detected lcore 1 as core 1 on socket 0
EAL: Detected lcore 2 as core 0 on socket 0
EAL: Detected lcore 3 as core 1 on socket 0
EAL: Detected lcore 4 as core 0 on socket 0
EAL: Detected lcore 5 as core 1 on socket 0
EAL: Detected lcore 6 as core 0 on socket 0
EAL: Detected lcore 7 as core 1 on socket 0
EAL: Support maximum 16 logical core(s) by configuration.
EAL: Detected 8 lcore(s)
EAL: Detected 1 NUMA nodes
EAL: VFIO PCI modules not loaded
EAL: DPAA Bus not present. Skipping.
EAL: Container: dprc.2 has VFIO iommu group id = 4
EAL: fslmc: Bus scan completed
EAL: Module /sys/module/rte_kni not found! error 2 (No such file or directory)
EAL: Multi-process socket /var/run/.rte_unix
EAL: Probing VFIO support...
EAL:   IOMMU type 1 (Type 1) is supported
EAL:   IOMMU type 7 (sPAPR) is not supported
EAL:   IOMMU type 8 (No-IOMMU) is not supported
EAL: VFIO support initialized
EAL: Mem event callback 'vfio_mem_event_clb' registered
EAL: Ask a virtual area of 0x2e000 bytes
EAL: Virtual area found at 0xffff86cae000 (size = 0x2e000)
EAL: Setting up physically contiguous memory...
EAL: Ask a virtual area of 0x1000 bytes
EAL: Virtual area found at 0xffff8873f000 (size = 0x1000)
EAL: Memseg list allocated: 0x100000kB at socket 0
EAL: Ask a virtual area of 0x800000000 bytes
EAL: Virtual area found at 0xfff780000000 (size = 0x800000000)
EAL: Ask a virtual area of 0x1000 bytes
EAL: Virtual area found at 0xffff8873e000 (size = 0x1000)
EAL: Memseg list allocated: 0x100000kB at socket 0
EAL: Ask a virtual area of 0x800000000 bytes
EAL: Virtual area found at 0xffef40000000 (size = 0x800000000)
EAL: Ask a virtual area of 0x1000 bytes
EAL: Virtual area found at 0xffff8873d000 (size = 0x1000)
EAL: Memseg list allocated: 0x100000kB at socket 0
EAL: Ask a virtual area of 0x800000000 bytes
EAL: Virtual area found at 0xffe700000000 (size = 0x800000000)
EAL: Ask a virtual area of 0x1000 bytes
EAL: Virtual area found at 0xffff8873c000 (size = 0x1000)
EAL: Memseg list allocated: 0x100000kB at socket 0
EAL: Ask a virtual area of 0x800000000 bytes
EAL: Virtual area found at 0xffdec0000000 (size = 0x800000000)
EAL: TSC frequency is ~25000 KHz
EAL: Master lcore 1 is ready (tid=88742110;cpuset=[1])
EAL: lcore 3 is ready (tid=85cab910;cpuset=[3])
EAL: lcore 2 is ready (tid=864ab910;cpuset=[2])
EAL: eal_memalloc_alloc_page_bulk(): couldn't find suitable memseg_list

error allocating rte services array
EAL: FATAL: rte_service_init() failed

EAL: rte_service_init() failed

Quoted 8 lines

PANIC in main():
Cannot init EAL
1: [./arm64-dpaa2-linuxapp-gcc/app/testpmd(rte_dump_stack+0x38) [0x4f37a8]] Aborted
--->8--

Above is an initial output - still investigating. I will keep you posted.

While working on issue reported in [1], I have found another issue which I might need you help.

[1] http://dpdk.org/ml/archives/dev/2018-March/093202.html

For [1], I bypassed by changing the mempool_add_elem code for time being - it now allows non-contiguous (not explicitly demanded contiguous) allocations to go through rte_mempool_populate_iova. With that, I was able to get DPAA2 working.

Problem is:
1. When I am working with 1GB pages, I/O is working fine.
2. When using 2MB pages (1024 num), the initialization somewhere after VFIO layer fails.

All with IOVA=VA mode.

Some logs:

This is the output of the virtual memory layout demanded by DPDK:

--->8---
EAL: Ask a virtual area of 0x2e000 bytes
EAL: Virtual area found at 0xffffb6561000 (size = 0x2e000)
EAL: Setting up physically contiguous memory...
EAL: Ask a virtual area of 0x59000 bytes
EAL: Virtual area found at 0xffffb6508000 (size = 0x59000)
EAL: Memseg list allocated: 0x800kB at socket 0
EAL: Ask a virtual area of 0x400000000 bytes
EAL: Virtual area found at 0xfffbb6400000 (size = 0x400000000)
EAL: Ask a virtual area of 0x59000 bytes
EAL: Virtual area found at 0xfffbb62af000 (size = 0x59000)
EAL: Memseg list allocated: 0x800kB at socket 0
EAL: Ask a virtual area of 0x400000000 bytes
EAL: Virtual area found at 0xfff7b6200000 (size = 0x400000000)
EAL: Ask a virtual area of 0x59000 bytes
EAL: Virtual area found at 0xfff7b6056000 (size = 0x59000)
EAL: Memseg list allocated: 0x800kB at socket 0
EAL: Ask a virtual area of 0x400000000 bytes
EAL: Virtual area found at 0xfff3b6000000 (size = 0x400000000)
EAL: Ask a virtual area of 0x59000 bytes
EAL: Virtual area found at 0xfff3b5dfd000 (size = 0x59000)
EAL: Memseg list allocated: 0x800kB at socket 0
EAL: Ask a virtual area of 0x400000000 bytes
EAL: Virtual area found at 0xffefb5c00000 (size = 0x400000000) --->8---

Then, somehow VFIO mapping is able to find only a single page to map

--->8---
EAL: Device (dpci.1) abstracted from VFIO
EAL: -->Initial SHM Virtual ADDR FFFBB6400000
EAL: -----> DMA size 0x200000
EAL: Total 1 segments found.
--->8---

Then, these logs appear probably when DPAA2 code requests for memory. I am not sure why it repeats the same '...expanded by 10MB'.

--->8---
EAL: Calling mem event callback vfio_mem_event_clbEAL: request: mp_malloc_sync
EAL: Heap on socket 0 was expanded by 10MB
EAL: Calling mem event callback vfio_mem_event_clbEAL: request: mp_malloc_sync
EAL: Heap on socket 0 was expanded by 10MB
EAL: Calling mem event callback vfio_mem_event_clbEAL: request: mp_malloc_sync
EAL: Heap on socket 0 was expanded by 10MB
EAL: Calling mem event callback vfio_mem_event_clbEAL: request: mp_malloc_sync
EAL: Heap on socket 0 was expanded by 10MB
EAL: Calling mem event callback vfio_mem_event_clbEAL: request: mp_malloc_sync
EAL: Heap on socket 0 was expanded by 10MB
EAL: Calling mem event callback vfio_mem_event_clbEAL: request: mp_malloc_sync
EAL: Heap on socket 0 was expanded by 10MB
EAL: Calling mem event callback vfio_mem_event_clbEAL: request: mp_malloc_sync
EAL: Heap on socket 0 was expanded by 2MB
EAL: Calling mem event callback vfio_mem_event_clbEAL: request: mp_malloc_sync
EAL: Heap on socket 0 was expanded by 10MB
EAL: Calling mem event callback vfio_mem_event_clbEAL: request: mp_malloc_sync
EAL: Heap on socket 0 was expanded by 10MB
LPM or EM none selected, default LPM on
Initializing port 0 ...
--->8---

l3fwd is stuck at this point. What I observe is that DPAA2 driver has gone ahead to register the queues (queue_setup) with hardware and the memory has either overrun (smaller than requested size mapped) or the addresses are corrupt (that is, not dma-able). (I get SMMU faults, indicating one of these cases)

There is some change from you in the fslmc/fslmc_vfio.c file (rte_fslmc_vfio_dmamap()). Ideally, that code should have walked over all the available pages for mapping but that didn't happen and only a single virtual area got dma-mapped.

--->8---
EAL: Device (dpci.1) abstracted from VFIO
EAL: -->Initial SHM Virtual ADDR FFFBB6400000
EAL: -----> DMA size 0x200000
EAL: Total 1 segments found.
--->8---

I am looking into this but if there is some hint which come to your mind, it might help.

Regards,
Shreyansh

-----Original Message-----
From: Burakov, Anatoly [mailto:[email protected]]
Sent: Wednesday, March 21, 2018 8:18 PM
To: Shreyansh Jain <[email protected]>
Cc: [email protected]; Hemant Agrawal <[email protected]>
Subject: Re: [dpdk-dev] [PATCH v2 00/41] Memory Hotplug for DPDK

Quoted 6 lines

Quoted 1 line

While working on issue reported in [1], I have found another issue which I might need you help.

[1]

https://emea01.safelinks.protection.outlook.com/?url=http%3A%2F%2Fdpdk.o
rg%2Fml%2Farchives%2Fdev%2F2018- March%2F093202.html&data=02%7C01%7Cshreyansh.jain%40nxp.com%7C5faee716e6 fc4908bdb608d58f3ad1e5%7C686ea1d3bc2b4c6fa92cd99c5c301635%7C0%7C0%7C6365 72405182868376&sdata=WohDdktHHAuNDnss1atuixSa%2FqC7HRMSDVCtFC9Vnto%3D&re served=0

Quoted 52 lines

For [1], I bypassed by changing the mempool_add_elem code for time being - it now allows non-contiguous (not explicitly demanded contiguous) allocations to go through rte_mempool_populate_iova. With that, I was able to get DPAA2 working.

Problem is:
1. When I am working with 1GB pages, I/O is working fine.
2. When using 2MB pages (1024 num), the initialization somewhere after VFIO layer fails.

All with IOVA=VA mode.

Some logs:

This is the output of the virtual memory layout demanded by DPDK:

--->8---
EAL: Ask a virtual area of 0x2e000 bytes
EAL: Virtual area found at 0xffffb6561000 (size = 0x2e000)
EAL: Setting up physically contiguous memory...
EAL: Ask a virtual area of 0x59000 bytes
EAL: Virtual area found at 0xffffb6508000 (size = 0x59000)
EAL: Memseg list allocated: 0x800kB at socket 0
EAL: Ask a virtual area of 0x400000000 bytes
EAL: Virtual area found at 0xfffbb6400000 (size = 0x400000000)
EAL: Ask a virtual area of 0x59000 bytes
EAL: Virtual area found at 0xfffbb62af000 (size = 0x59000)
EAL: Memseg list allocated: 0x800kB at socket 0
EAL: Ask a virtual area of 0x400000000 bytes
EAL: Virtual area found at 0xfff7b6200000 (size = 0x400000000)
EAL: Ask a virtual area of 0x59000 bytes
EAL: Virtual area found at 0xfff7b6056000 (size = 0x59000)
EAL: Memseg list allocated: 0x800kB at socket 0
EAL: Ask a virtual area of 0x400000000 bytes
EAL: Virtual area found at 0xfff3b6000000 (size = 0x400000000)
EAL: Ask a virtual area of 0x59000 bytes
EAL: Virtual area found at 0xfff3b5dfd000 (size = 0x59000)
EAL: Memseg list allocated: 0x800kB at socket 0
EAL: Ask a virtual area of 0x400000000 bytes
EAL: Virtual area found at 0xffefb5c00000 (size = 0x400000000) --->8---

Then, somehow VFIO mapping is able to find only a single page to map

--->8---
EAL: Device (dpci.1) abstracted from VFIO
EAL: -->Initial SHM Virtual ADDR FFFBB6400000
EAL: -----> DMA size 0x200000
EAL: Total 1 segments found.
--->8---

Quoted 5 lines

Then, these logs appear probably when DPAA2 code requests for memory. I am not sure why it repeats the same '...expanded by 10MB'.

--->8---
EAL: Calling mem event callback vfio_mem_event_clbEAL: request:

mp_malloc_sync

Quoted 2 lines

EAL: Heap on socket 0 was expanded by 10MB
EAL: Calling mem event callback vfio_mem_event_clbEAL: request:

mp_malloc_sync

Quoted 2 lines

EAL: Heap on socket 0 was expanded by 10MB
EAL: Calling mem event callback vfio_mem_event_clbEAL: request:

mp_malloc_sync

Quoted 2 lines

EAL: Heap on socket 0 was expanded by 10MB
EAL: Calling mem event callback vfio_mem_event_clbEAL: request:

mp_malloc_sync

Quoted 2 lines

EAL: Heap on socket 0 was expanded by 10MB
EAL: Calling mem event callback vfio_mem_event_clbEAL: request:

mp_malloc_sync

Quoted 2 lines

EAL: Heap on socket 0 was expanded by 10MB
EAL: Calling mem event callback vfio_mem_event_clbEAL: request:

mp_malloc_sync

Quoted 2 lines

EAL: Heap on socket 0 was expanded by 10MB
EAL: Calling mem event callback vfio_mem_event_clbEAL: request:

mp_malloc_sync

Quoted 2 lines

EAL: Heap on socket 0 was expanded by 2MB
EAL: Calling mem event callback vfio_mem_event_clbEAL: request:

mp_malloc_sync

Quoted 2 lines

EAL: Heap on socket 0 was expanded by 10MB
EAL: Calling mem event callback vfio_mem_event_clbEAL: request:

mp_malloc_sync

Quoted 23 lines

EAL: Heap on socket 0 was expanded by 10MB
LPM or EM none selected, default LPM on
Initializing port 0 ...
--->8---

l3fwd is stuck at this point. What I observe is that DPAA2 driver has gone ahead to register the queues (queue_setup) with hardware and the memory has either overrun (smaller than requested size mapped) or the addresses are corrupt (that is, not dma-able). (I get SMMU faults, indicating one of these cases)

There is some change from you in the fslmc/fslmc_vfio.c file (rte_fslmc_vfio_dmamap()). Ideally, that code should have walked over all the available pages for mapping but that didn't happen and only a single virtual area got dma-mapped.

--->8---
EAL: Device (dpci.1) abstracted from VFIO
EAL: -->Initial SHM Virtual ADDR FFFBB6400000
EAL: -----> DMA size 0x200000
EAL: Total 1 segments found.
--->8---

Quoted 6 lines

I am looking into this but if there is some hint which come to your mind, it might help.

Regards,
Shreyansh

Hi Shreyansh,

Thanks for the feedback.

The "heap on socket 0 was expanded by 10MB" has to do with synchronization requests in primary/secondary processes. I can see you're allocating LPM tables - that's most likely what these allocations are about (it's hotplugging memory).

I think i might have an idea what is going on. I am assuming that you are starting up your DPDK application without any -m or --socket-mem flags, which means you are starting with empty heap.

During initialization, certain DPDK features (such as service cores, PMD's) allocate memory. Most likely you have essentially started up with 1 2M page, which is what you see in fslmc logs: this page gets mapped for VFIO.

Then, you allocate a bunch of LPM tables, which trigger more memory allocation, and trigger memory allocation callbacks registered through rte_mem_event_register_callback(). One of these callbacks is a VFIO callback, which is registered in eal_vfio.c:rte_vfio_enable(). However, since fslmc bus has its own VFIO implementation that is independent of what happens in EAL VFIO code, what probably happens is that the fslmc bus misses the necessary messages from the memory hotplug to map additional resources for DMA.

Try adding a rte_mem_event_register_callback() somewhere in fslmc init so that it calls necessary map function. eal_vfio.c:vfio_mem_event_callback() should provide a good template on how to approach creating such a callback. Let me know if this works!

(as a side note, how can we extend VFIO to move this stuff back into EAL and expose it as an API?)

--
Thanks,
Anatoly

Hello Anatoly,

Quoted 7 lines

-----Original Message-----
From: Burakov, Anatoly [mailto:[email protected]]
Sent: Wednesday, March 21, 2018 8:18 PM
To: Shreyansh Jain <[email protected]>
Cc: [email protected]; Hemant Agrawal <[email protected]>
Subject: Re: [dpdk-dev] [PATCH v2 00/41] Memory Hotplug for DPDK

[...]

Quoted 128 lines

Quoted 6 lines

Quoted 1 line

While working on issue reported in [1], I have found another issue which I might need you help.

[1]

https://emea01.safelinks.protection.outlook.com/?url=http%3A%2F%2Fdpdk.o
rg%2Fml%2Farchives%2Fdev%2F2018- March%2F093202.html&data=02%7C01%7Cshreyansh.jain%40nxp.com%7C5faee716e6 fc4908bdb608d58f3ad1e5%7C686ea1d3bc2b4c6fa92cd99c5c301635%7C0%7C0%7C6365 72405182868376&sdata=WohDdktHHAuNDnss1atuixSa%2FqC7HRMSDVCtFC9Vnto%3D&re served=0

Quoted 52 lines

For [1], I bypassed by changing the mempool_add_elem code for time being - it now allows non-contiguous (not explicitly demanded contiguous) allocations to go through rte_mempool_populate_iova. With that, I was able to get DPAA2 working.

Problem is:
1. When I am working with 1GB pages, I/O is working fine.
2. When using 2MB pages (1024 num), the initialization somewhere after VFIO layer fails.

All with IOVA=VA mode.

Some logs:

This is the output of the virtual memory layout demanded by DPDK:

--->8---
EAL: Ask a virtual area of 0x2e000 bytes
EAL: Virtual area found at 0xffffb6561000 (size = 0x2e000)
EAL: Setting up physically contiguous memory...
EAL: Ask a virtual area of 0x59000 bytes
EAL: Virtual area found at 0xffffb6508000 (size = 0x59000)
EAL: Memseg list allocated: 0x800kB at socket 0
EAL: Ask a virtual area of 0x400000000 bytes
EAL: Virtual area found at 0xfffbb6400000 (size = 0x400000000)
EAL: Ask a virtual area of 0x59000 bytes
EAL: Virtual area found at 0xfffbb62af000 (size = 0x59000)
EAL: Memseg list allocated: 0x800kB at socket 0
EAL: Ask a virtual area of 0x400000000 bytes
EAL: Virtual area found at 0xfff7b6200000 (size = 0x400000000)
EAL: Ask a virtual area of 0x59000 bytes
EAL: Virtual area found at 0xfff7b6056000 (size = 0x59000)
EAL: Memseg list allocated: 0x800kB at socket 0
EAL: Ask a virtual area of 0x400000000 bytes
EAL: Virtual area found at 0xfff3b6000000 (size = 0x400000000)
EAL: Ask a virtual area of 0x59000 bytes
EAL: Virtual area found at 0xfff3b5dfd000 (size = 0x59000)
EAL: Memseg list allocated: 0x800kB at socket 0
EAL: Ask a virtual area of 0x400000000 bytes
EAL: Virtual area found at 0xffefb5c00000 (size = 0x400000000) --->8---

Then, somehow VFIO mapping is able to find only a single page to map

--->8---
EAL: Device (dpci.1) abstracted from VFIO
EAL: -->Initial SHM Virtual ADDR FFFBB6400000
EAL: -----> DMA size 0x200000
EAL: Total 1 segments found.
--->8---

Quoted 5 lines

Then, these logs appear probably when DPAA2 code requests for memory. I am not sure why it repeats the same '...expanded by 10MB'.

--->8---
EAL: Calling mem event callback vfio_mem_event_clbEAL: request:

mp_malloc_sync

Quoted 2 lines

EAL: Heap on socket 0 was expanded by 10MB
EAL: Calling mem event callback vfio_mem_event_clbEAL: request:

mp_malloc_sync

Quoted 2 lines

EAL: Heap on socket 0 was expanded by 10MB
EAL: Calling mem event callback vfio_mem_event_clbEAL: request:

mp_malloc_sync

Quoted 2 lines

EAL: Heap on socket 0 was expanded by 10MB
EAL: Calling mem event callback vfio_mem_event_clbEAL: request:

mp_malloc_sync

Quoted 2 lines

EAL: Heap on socket 0 was expanded by 10MB
EAL: Calling mem event callback vfio_mem_event_clbEAL: request:

mp_malloc_sync

Quoted 2 lines

EAL: Heap on socket 0 was expanded by 10MB
EAL: Calling mem event callback vfio_mem_event_clbEAL: request:

mp_malloc_sync

Quoted 2 lines

EAL: Heap on socket 0 was expanded by 10MB
EAL: Calling mem event callback vfio_mem_event_clbEAL: request:

mp_malloc_sync

Quoted 2 lines

EAL: Heap on socket 0 was expanded by 2MB
EAL: Calling mem event callback vfio_mem_event_clbEAL: request:

mp_malloc_sync

Quoted 2 lines

EAL: Heap on socket 0 was expanded by 10MB
EAL: Calling mem event callback vfio_mem_event_clbEAL: request:

mp_malloc_sync

Quoted 23 lines

EAL: Heap on socket 0 was expanded by 10MB
LPM or EM none selected, default LPM on
Initializing port 0 ...
--->8---

l3fwd is stuck at this point. What I observe is that DPAA2 driver has gone ahead to register the queues (queue_setup) with hardware and the memory has either overrun (smaller than requested size mapped) or the addresses are corrupt (that is, not dma-able). (I get SMMU faults, indicating one of these cases)

There is some change from you in the fslmc/fslmc_vfio.c file (rte_fslmc_vfio_dmamap()). Ideally, that code should have walked over all the available pages for mapping but that didn't happen and only a single virtual area got dma-mapped.

--->8---
EAL: Device (dpci.1) abstracted from VFIO
EAL: -->Initial SHM Virtual ADDR FFFBB6400000
EAL: -----> DMA size 0x200000
EAL: Total 1 segments found.
--->8---

Quoted 6 lines

I am looking into this but if there is some hint which come to your mind, it might help.

Regards,
Shreyansh

Hi Shreyansh,

Thanks for the feedback.

Quoted 4 lines

The "heap on socket 0 was expanded by 10MB" has to do with synchronization requests in primary/secondary processes. I can see you're allocating LPM tables - that's most likely what these allocations are about (it's hotplugging memory).

I get that but why same message multiple times without any change in the expansion. Further, I don't have multiple process - in fact, I'm working with a single datapath thread. Anyways, I will look through the code for this.

Quoted 4 lines

I think i might have an idea what is going on. I am assuming that you are starting up your DPDK application without any -m or --socket-mem flags, which means you are starting with empty heap.

Yes, no specific --socket-mem passed as argument.

Quoted 5 lines

During initialization, certain DPDK features (such as service cores, PMD's) allocate memory. Most likely you have essentially started up with 1 2M page, which is what you see in fslmc logs: this page gets mapped for VFIO.

Agree.

Quoted 9 lines

Then, you allocate a bunch of LPM tables, which trigger more memory allocation, and trigger memory allocation callbacks registered through rte_mem_event_register_callback(). One of these callbacks is a VFIO callback, which is registered in eal_vfio.c:rte_vfio_enable(). However, since fslmc bus has its own VFIO implementation that is independent of what happens in EAL VFIO code, what probably happens is that the fslmc bus misses the necessary messages from the memory hotplug to map additional resources for DMA.

Makes sense

Quoted 5 lines

Try adding a rte_mem_event_register_callback() somewhere in fslmc init so that it calls necessary map function. eal_vfio.c:vfio_mem_event_callback() should provide a good template on how to approach creating such a callback. Let me know if this works!

OK. I will give this a try and update you.

Quoted 3 lines

(as a side note, how can we extend VFIO to move this stuff back into EAL and expose it as an API?)

The problem is that FSLMC VFIO driver is slightly different from generic VFIO layer in the sense that device in a VFIO container is actually another level of container. Anyways, I will have a look how much generalization is possible. Or else, I will work with the vfio_mem_event_callback() as suggested above.

Thanks for suggestions.

Quoted 4 lines

--
Thanks,
Anatoly

This patchset introduces dynamic memory allocation for DPDK (aka memory hotplug). Based upon RFC submitted in December [1].

Dependencies (to be applied in specified order):
- IPC bugfixes patchset [2]
- IPC improvements patchset [3]
- IPC asynchronous request API patch [4]
- Function to return number of sockets [5]

Deprecation notices relevant to this patchset:
- General outline of memory hotplug changes [6]
- EAL NUMA node count changes [7]

The vast majority of changes are in the EAL and malloc, the external API disruption is minimal: a new set of API's are added for contiguous memory allocation for rte_memzone, and a few API additions in rte_memory due to switch to memseg_lists as opposed to memsegs. Every other API change is internal to EAL, and all of the memory allocation/freeing is handled through rte_malloc, with no externally visible API changes.

Quick outline of all changes done as part of this patchset:

* Malloc heap adjusted to handle holes in address space
* Single memseg list replaced by multiple memseg lists
* VA space for hugepages is preallocated in advance
* Added alloc/free for pages happening as needed on rte_malloc/rte_free
* Added contiguous memory allocation API's for rte_memzone
* Integrated Pawel Wodkowski's patch for registering/unregistering memory with VFIO [8]
* Callbacks for registering memory allocations
* Multiprocess support done via DPDK IPC introduced in 18.02

The biggest difference is a "memseg" now represents a single page (as opposed to being a big contiguous block of pages). As a consequence, both memzones and malloc elements are no longer guaranteed to be physically contiguous, unless the user asks for it at reserve time. To preserve whatever functionality that was dependent on previous behavior, a legacy memory option is also provided, however it is expected (or perhaps vainly hoped) to be temporary solution.

Why multiple memseg lists instead of one? Since memseg is a single page now, the list of memsegs will get quite big, and we need to locate pages somehow when we allocate and free them. We could of course just walk the list and allocate one contiguous chunk of VA space for memsegs, but this implementation uses separate lists instead in order to speed up many operations with memseg lists.

For v1 and v2, the following limitations are present:
- FreeBSD does not even compile, let alone run
- No 32-bit support

Hi Anatoly,

Quoted 46 lines · Anatoly Burakov

This patchset introduces dynamic memory allocation for DPDK (aka memory hotplug). Based upon RFC submitted in December [1].

Dependencies (to be applied in specified order):
- IPC bugfixes patchset [2]
- IPC improvements patchset [3]
- IPC asynchronous request API patch [4]
- Function to return number of sockets [5]

Deprecation notices relevant to this patchset:
- General outline of memory hotplug changes [6]
- EAL NUMA node count changes [7]

The vast majority of changes are in the EAL and malloc, the external API disruption is minimal: a new set of API's are added for contiguous memory allocation for rte_memzone, and a few API additions in rte_memory due to switch to memseg_lists as opposed to memsegs. Every other API change is internal to EAL, and all of the memory allocation/freeing is handled through rte_malloc, with no externally visible API changes.

Quick outline of all changes done as part of this patchset:

* Malloc heap adjusted to handle holes in address space
* Single memseg list replaced by multiple memseg lists
* VA space for hugepages is preallocated in advance
* Added alloc/free for pages happening as needed on rte_malloc/rte_free
* Added contiguous memory allocation API's for rte_memzone
* Integrated Pawel Wodkowski's patch for registering/unregistering memory with VFIO [8]
* Callbacks for registering memory allocations
* Multiprocess support done via DPDK IPC introduced in 18.02

The biggest difference is a "memseg" now represents a single page (as opposed to being a big contiguous block of pages). As a consequence, both memzones and malloc elements are no longer guaranteed to be physically contiguous, unless the user asks for it at reserve time. To preserve whatever functionality that was dependent on previous behavior, a legacy memory option is also provided, however it is expected (or perhaps vainly hoped) to be temporary solution.

Why multiple memseg lists instead of one? Since memseg is a single page now, the list of memsegs will get quite big, and we need to locate pages somehow when we allocate and free them. We could of course just walk the list and allocate one contiguous chunk of VA space for memsegs, but this implementation uses separate lists instead in order to speed up many operations with memseg lists.

Quoted 3 lines

For v1 and v2, the following limitations are present:
- FreeBSD does not even compile, let alone run
- No 32-bit support

I just read on [email protected] [1] that an early merge to this series is expected. So, would this limitation be fixed before merge? Or, has it already been fixed in github repo?

[1] http://dpdk.org/ml/archives/announce/2018-March/000182.html

[...]

-
Shreyansh

-----Original Message-----
From: Burakov, Anatoly [mailto:[email protected]]
Sent: Tuesday, March 20, 2018 3:38 PM
To: Shreyansh Jain <[email protected]>
Cc: [email protected]; Thomas Monjalon <[email protected]>
Subject: Re: [dpdk-dev] [PATCH v2 00/41] Memory Hotplug for DPDK

Quoted 2 lines · Shreyansh Jain

Hi Anatoly,

Quoted 6 lines

[...]

-
Shreyansh

Hi Shreyansh,

It will be fixed before merge, yes. I would expect this code to arrive in Github in the next few days.

As we are preparing for dynamic memory allocation, we need to be able to handle holes in our malloc heap, hence we're switching to doubly linked list, and prepare infrastructure to support it.

Since our heap is now aware where are our first and last elements, there is no longer any need to have a dummy element at the end of each heap, so get rid of that as well. Instead, let insert/remove/ join/split operations handle end-of-list conditions automatically.

Signed-off-by Anatoly Burakov <[email protected]>

Quoted 9 lines · On Wed, 4 Apr 2018 00:21:15 +0100 Anatoly Burakov <[email protected]>…

As we are preparing for dynamic memory allocation, we need to be able to handle holes in our malloc heap, hence we're switching to doubly linked list, and prepare infrastructure to support it.

Since our heap is now aware where are our first and last elements, there is no longer any need to have a dummy element at the end of each heap, so get rid of that as well. Instead, let insert/remove/ join/split operations handle end-of-list conditions automatically.

Quoted 1 line

Signed-off-by Anatoly Burakov <[email protected]>

Dummy element at end of heap could be helpful for purify/valgrind style verification that code does not exceed allocation.

This adds a new set of _contig API's to rte_memzone. For now, hugepage memory is always contiguous, but we need to prepare the drivers for the switch.

Signed-off-by Anatoly Burakov <[email protected]>

Quoted 4 lines · On Wed, 4 Apr 2018 00:21:23 +0100 Anatoly Burakov <[email protected]>…

This adds a new set of _contig API's to rte_memzone. For now, hugepage memory is always contiguous, but we need to prepare the drivers for the switch.

Quoted 1 line

Signed-off-by Anatoly Burakov <[email protected]>

Why not make fragmentation an optional flag to the memzone_reserved rather than a new API. That way less drivers need to be changed.

#define RTE_MEMZONE_SPARSE 0x00100000 /* Allow zone to be non-contiguous */