Memory Hotplug for DPDK

index 54f7f4b..4660cc2 100644
@@ -3,6 +3,7 @@
  * Copyright(c) 2016 6WIND S.A.
  */
 
+#include <stdbool.h>
 #include <stdio.h>
 #include <string.h>
 #include <stdint.h>

@@ -98,6 +99,27 @@ static unsigned optimize_object_size(unsigned obj_size)
 	return new_obj_size * RTE_MEMPOOL_ALIGN;
 }
 
+static size_t
+get_min_page_size(void)
+{
+	const struct rte_mem_config *mcfg =
+			rte_eal_get_configuration()->mem_config;
+	int i;
+	size_t min_pagesz = SIZE_MAX;
+
+	for (i = 0; i < RTE_MAX_MEMSEG; i++) {
+		const struct rte_memseg *ms = &mcfg->memseg[i];
+
+		if (ms->addr == NULL)
+			continue;
+
+		if (ms->hugepage_sz < min_pagesz)
+			min_pagesz = ms->hugepage_sz;
+	}
+
+	return min_pagesz == SIZE_MAX ? (size_t) getpagesize() : min_pagesz;
+}
+
 static void
 mempool_add_elem(struct rte_mempool *mp, void *obj, rte_iova_t iova)
 {

@@ -367,16 +389,6 @@ rte_mempool_populate_iova(struct rte_mempool *mp, char *vaddr,
 	/* update mempool capabilities */
 	mp->flags |= mp_capa_flags;
 
-	/* Detect pool area has sufficient space for elements */
-	if (mp_capa_flags & MEMPOOL_F_CAPA_PHYS_CONTIG) {
-		if (len < total_elt_sz * mp->size) {
-			RTE_LOG(ERR, MEMPOOL,
-				"pool area %" PRIx64 " not enough\n",
-				(uint64_t)len);
-			return -ENOSPC;
-		}
-	}
-
 	memhdr = rte_zmalloc("MEMPOOL_MEMHDR", sizeof(*memhdr), 0);
 	if (memhdr == NULL)
 		return -ENOMEM;

@@ -549,6 +561,7 @@ rte_mempool_populate_default(struct rte_mempool *mp)
 	unsigned mz_id, n;
 	unsigned int mp_flags;
 	int ret;
+	bool force_contig, no_contig, try_contig, no_pageshift;
 
 	/* mempool must not be populated */
 	if (mp->nb_mem_chunks != 0)

@@ -563,9 +576,68 @@ rte_mempool_populate_default(struct rte_mempool *mp)
 	/* update mempool capabilities */
 	mp->flags |= mp_flags;
 
-	if (rte_eal_has_hugepages()) {
-		pg_shift = 0; /* not needed, zone is physically contiguous */
+	no_contig = mp->flags & MEMPOOL_F_NO_PHYS_CONTIG;
+	force_contig = mp->flags & MEMPOOL_F_CAPA_PHYS_CONTIG;
+
+	/*
+	 * the following section calculates page shift and page size values.
+	 *
+	 * these values impact the result of rte_mempool_xmem_size(), which
+	 * returns the amount of memory that should be allocated to store the
+	 * desired number of objects. when not zero, it allocates more memory
+	 * for the padding between objects, to ensure that an object does not
+	 * cross a page boundary. in other words, page size/shift are to be set
+	 * to zero if mempool elements won't care about page boundaries.
+	 * there are several considerations for page size and page shift here.
+	 *
+	 * if we don't need our mempools to have physically contiguous objects,
+	 * then just set page shift and page size to 0, because the user has
+	 * indicated that there's no need to care about anything.
+	 *
+	 * if we do need contiguous objects, there is also an option to reserve
+	 * the entire mempool memory as one contiguous block of memory, in
+	 * which case the page shift and alignment wouldn't matter as well.
+	 *
+	 * if we require contiguous objects, but not necessarily the entire
+	 * mempool reserved space to be contiguous, then there are two options.
+	 *
+	 * if our IO addresses are virtual, not actual physical (IOVA as VA
+	 * case), then no page shift needed - our memory allocation will give us
+	 * contiguous physical memory as far as the hardware is concerned, so
+	 * act as if we're getting contiguous memory.
+	 *
+	 * if our IO addresses are physical, we may get memory from bigger
+	 * pages, or we might get memory from smaller pages, and how much of it
+	 * we require depends on whether we want bigger or smaller pages.
+	 * However, requesting each and every memory size is too much work, so
+	 * what we'll do instead is walk through the page sizes available, pick
+	 * the smallest one and set up page shift to match that one. We will be
+	 * wasting some space this way, but it's much nicer than looping around
+	 * trying to reserve each and every page size.
+	 *
+	 * However, since size calculation will produce page-aligned sizes, it
+	 * makes sense to first try and see if we can reserve the entire memzone
+	 * in one contiguous chunk as well (otherwise we might end up wasting a
+	 * 1G page on a 10MB memzone). If we fail to get enough contiguous
+	 * memory, then we'll go and reserve space page-by-page.
+	 */
+	no_pageshift = no_contig || force_contig ||
+			rte_eal_iova_mode() == RTE_IOVA_VA;
+	try_contig = !no_contig && !no_pageshift && rte_eal_has_hugepages();
+	if (force_contig)
+		mz_flags |= RTE_MEMZONE_IOVA_CONTIG;
+
+	if (no_pageshift) {
 		pg_sz = 0;
+		pg_shift = 0;
+		align = RTE_CACHE_LINE_SIZE;
+	} else if (try_contig) {
+		pg_sz = get_min_page_size();
+		pg_shift = rte_bsf32(pg_sz);
+		/* we're trying to reserve contiguous memzone first, so try
+		 * align to cache line; if we fail to reserve a contiguous
+		 * memzone, we'll adjust alignment to equal pagesize later.
+		 */
 		align = RTE_CACHE_LINE_SIZE;
 	} else {
 		pg_sz = getpagesize();

@@ -575,8 +647,13 @@ rte_mempool_populate_default(struct rte_mempool *mp)
 
 	total_elt_sz = mp->header_size + mp->elt_size + mp->trailer_size;
 	for (mz_id = 0, n = mp->size; n > 0; mz_id++, n -= ret) {
-		size = rte_mempool_xmem_size(n, total_elt_sz, pg_shift,
-						mp->flags);
+		unsigned int flags;
+		if (try_contig || no_pageshift)
+			size = rte_mempool_xmem_size(n, total_elt_sz, 0,
+				mp->flags);
+		else
+			size = rte_mempool_xmem_size(n, total_elt_sz, pg_shift,
+				mp->flags);
 
 		ret = snprintf(mz_name, sizeof(mz_name),
 			RTE_MEMPOOL_MZ_FORMAT "_%d", mp->name, mz_id);

@@ -585,23 +662,52 @@ rte_mempool_populate_default(struct rte_mempool *mp)
 			goto fail;
 		}
 
-		mz = rte_memzone_reserve_aligned(mz_name, size,
-			mp->socket_id, mz_flags, align);
-		/* not enough memory, retry with the biggest zone we have */
-		if (mz == NULL)
+		flags = mz_flags;
+
+		/* if we're trying to reserve contiguous memory, add appropriate
+		 * memzone flag.
+		 */
+		if (try_contig)
+			flags |= RTE_MEMZONE_IOVA_CONTIG;
+
+		mz = rte_memzone_reserve_aligned(mz_name, size, mp->socket_id,
+				flags, align);
+
+		/* if we were trying to allocate contiguous memory, adjust
+		 * memzone size and page size to fit smaller page sizes, and
+		 * try again.
+		 */
+		if (mz == NULL && try_contig) {
+			try_contig = false;
+			flags &= ~RTE_MEMZONE_IOVA_CONTIG;
+			align = pg_sz;
+			size = rte_mempool_xmem_size(n, total_elt_sz,
+				pg_shift, mp->flags);
+
+			mz = rte_memzone_reserve_aligned(mz_name, size,
+				mp->socket_id, flags, align);
+		}
+		/* don't try reserving with 0 size if we were asked to reserve
+		 * IOVA-contiguous memory.
+		 */
+		if (!force_contig && mz == NULL) {
+			/* not enough memory, retry with the biggest zone we
+			 * have
+			 */
 			mz = rte_memzone_reserve_aligned(mz_name, 0,
-				mp->socket_id, mz_flags, align);
+					mp->socket_id, flags, align);
+		}
 		if (mz == NULL) {
 			ret = -rte_errno;
 			goto fail;
 		}
 
-		if (mp->flags & MEMPOOL_F_NO_PHYS_CONTIG)
+		if (no_contig)
 			iova = RTE_BAD_IOVA;
 		else
 			iova = mz->iova;
 
-		if (rte_eal_has_hugepages())
+		if (no_pageshift || try_contig)
 			ret = rte_mempool_populate_iova(mp, mz->addr,
 				iova, mz->len,
 				rte_mempool_memchunk_mz_free,

index 5b8ced4..947db1f 100644
@@ -218,6 +218,27 @@ rte_mem_lock_page(const void *virt)
 	return mlock((void *)aligned, page_size);
 }
 
+int __rte_experimental
+rte_memseg_walk(rte_memseg_walk_t func, void *arg)
+{
+	struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
+	int i, ret;
+
+	for (i = 0; i < RTE_MAX_MEMSEG; i++) {
+		const struct rte_memseg *ms = &mcfg->memseg[i];
+
+		if (ms->addr == NULL)
+			continue;
+
+		ret = func(ms, arg);
+		if (ret < 0)
+			return -1;
+		if (ret > 0)
+			return 1;
+	}
+	return 0;
+}
+
 /* init memory subsystem */
 int
 rte_eal_memory_init(void)

index 302f865..93eadaa 100644
@@ -20,6 +20,7 @@ extern "C" {
 #endif
 
 #include <rte_common.h>
+#include <rte_compat.h>
 #include <rte_config.h>
 
 __extension__

@@ -130,6 +131,30 @@ phys_addr_t rte_mem_virt2phy(const void *virt);
 rte_iova_t rte_mem_virt2iova(const void *virt);
 
 /**
+ * Memseg walk function prototype.
+ *
+ * Returning 0 will continue walk
+ * Returning 1 will stop the walk
+ * Returning -1 will stop the walk and report error
+ */
+typedef int (*rte_memseg_walk_t)(const struct rte_memseg *ms, void *arg);
+
+/**
+ * Walk list of all memsegs.
+ *
+ * @param func
+ *   Iterator function
+ * @param arg
+ *   Argument passed to iterator
+ * @return
+ *   0 if walked over the entire list
+ *   1 if stopped by the user
+ *   -1 if user function reported error
+ */
+int __rte_experimental
+rte_memseg_walk(rte_memseg_walk_t func, void *arg);
+
+/**
  * Get the layout of the available physical memory.
  *
  * It can be useful for an application to have the full physical

index 4291871..0c048dc 100644
@@ -189,17 +189,51 @@ static int vfio_map_irq_region(struct fslmc_vfio_group *group)
 	return -errno;
 }
 
-int rte_fslmc_vfio_dmamap(void)
+static int
+fslmc_vfio_map(const struct rte_memseg *ms, void *arg)
 {
-	int ret;
+	int *n_segs = arg;
 	struct fslmc_vfio_group *group;
 	struct vfio_iommu_type1_dma_map dma_map = {
 		.argsz = sizeof(struct vfio_iommu_type1_dma_map),
 		.flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE,
 	};
+	int ret;
+
+	dma_map.size = ms->len;
+	dma_map.vaddr = ms->addr_64;
+#ifdef RTE_LIBRTE_DPAA2_USE_PHYS_IOVA
+	dma_map.iova = ms->iova;
+#else
+	dma_map.iova = dma_map.vaddr;
+#endif
+
+	/* SET DMA MAP for IOMMU */
+	group = &vfio_group;
+
+	if (!group->container) {
+		DPAA2_BUS_ERR("Container is not connected ");
+		return -1;
+	}
+
+	DPAA2_BUS_DEBUG("-->Initial SHM Virtual ADDR %llX",
+			dma_map.vaddr);
+	DPAA2_BUS_DEBUG("-----> DMA size 0x%llX", dma_map.size);
+	ret = ioctl(group->container->fd, VFIO_IOMMU_MAP_DMA,
+			&dma_map);
+	if (ret) {
+		DPAA2_BUS_ERR("VFIO_IOMMU_MAP_DMA API(errno = %d)",
+				errno);
+		return -1;
+	}
+	(*n_segs)++;
+	return 0;
+}
 
-	int i;
+int rte_fslmc_vfio_dmamap(void)
+{
 	const struct rte_memseg *memseg;
+	int i = 0;
 
 	if (is_dma_done)
 		return 0;

@@ -210,51 +244,21 @@ int rte_fslmc_vfio_dmamap(void)
 		return -ENODEV;
 	}
 
-	for (i = 0; i < RTE_MAX_MEMSEG; i++) {
-		if (memseg[i].addr == NULL && memseg[i].len == 0) {
-			DPAA2_BUS_DEBUG("Total %d segments found", i);
-			break;
-		}
-
-		dma_map.size = memseg[i].len;
-		dma_map.vaddr = memseg[i].addr_64;
-#ifdef RTE_LIBRTE_DPAA2_USE_PHYS_IOVA
-		dma_map.iova = memseg[i].iova;
-#else
-		dma_map.iova = dma_map.vaddr;
-#endif
-
-		/* SET DMA MAP for IOMMU */
-		group = &vfio_group;
-
-		if (!group->container) {
-			DPAA2_BUS_ERR("Container is not connected");
-			return -1;
-		}
-
-		DPAA2_BUS_DEBUG("-->Initial SHM Virtual ADDR %llX",
-				dma_map.vaddr);
-		DPAA2_BUS_DEBUG("-----> DMA size 0x%llX", dma_map.size);
-		ret = ioctl(group->container->fd, VFIO_IOMMU_MAP_DMA,
-			    &dma_map);
-		if (ret) {
-			DPAA2_BUS_ERR("Unable to map DMA address (errno = %d)",
-				      errno);
-			return ret;
-		}
-	}
+	if (rte_memseg_walk(fslmc_vfio_map, &i) < 0)
+		return -1;
 
 	/* Verifying that at least single segment is available */
 	if (i <= 0) {
 		DPAA2_BUS_ERR("No Segments found for VFIO Mapping");
 		return -1;
 	}
+	DPAA2_BUS_DEBUG("Total %d segments found.", i);
 
 	/* TODO - This is a W.A. as VFIO currently does not add the mapping of
 	 * the interrupt region to SMMU. This should be removed once the
 	 * support is added in the Kernel.
 	 */
-	vfio_map_irq_region(group);
+	vfio_map_irq_region(&vfio_group);
 
 	is_dma_done = 1;
 

index abde641..6dda054 100644
@@ -116,22 +116,24 @@ rte_pci_unmap_device(struct rte_pci_device *dev)
 	}
 }
 
-void *
-pci_find_max_end_va(void)
+static int
+find_max_end_va(const struct rte_memseg *ms, void *arg)
 {
-	const struct rte_memseg *seg = rte_eal_get_physmem_layout();
-	const struct rte_memseg *last = seg;
-	unsigned i = 0;
+	void *end_va = RTE_PTR_ADD(ms->addr, ms->len);
+	void **max_va = arg;
 
-	for (i = 0; i < RTE_MAX_MEMSEG; i++, seg++) {
-		if (seg->addr == NULL)
-			break;
+	if (*max_va < end_va)
+		*max_va = end_va;
+	return 0;
+}
 
-		if (seg->addr > last->addr)
-			last = seg;
+void *
+pci_find_max_end_va(void)
+{
+	void *va = NULL;
 
-	}
-	return RTE_PTR_ADD(last->addr, last->len);
+	rte_memseg_walk(find_max_end_va, &va);
+	return va;
 }
 
 /* parse one line of the "resource" sysfs file (note that the 'line'

index 7d58d66..1724b65 100644
@@ -477,6 +477,19 @@ static struct rte_pci_driver mlx5_driver;
  */
 static void *uar_base;
 
+static int
+find_lower_va_bound(const struct rte_memseg *ms, void *arg)
+{
+	void **addr = arg;
+
+	if (*addr == NULL)
+		*addr = ms->addr;
+	else
+		*addr = RTE_MIN(*addr, ms->addr);
+
+	return 0;
+}
+
 /**
  * Reserve UAR address space for primary process.
  *

@@ -491,21 +504,14 @@ mlx5_uar_init_primary(struct rte_eth_dev *dev)
 {
 	struct priv *priv = dev->data->dev_private;
 	void *addr = (void *)0;
-	int i;
-	const struct rte_mem_config *mcfg;
 
 	if (uar_base) { /* UAR address space mapped. */
 		priv->uar_base = uar_base;
 		return 0;
 	}
 	/* find out lower bound of hugepage segments */
-	mcfg = rte_eal_get_configuration()->mem_config;
-	for (i = 0; i < RTE_MAX_MEMSEG && mcfg->memseg[i].addr; i++) {
-		if (addr)
-			addr = RTE_MIN(addr, mcfg->memseg[i].addr);
-		else
-			addr = mcfg->memseg[i].addr;
-	}
+	rte_memseg_walk(find_lower_va_bound, &addr);
+
 	/* keep distance to hugepages to minimize potential conflicts. */
 	addr = RTE_PTR_SUB(addr, MLX5_UAR_OFFSET + MLX5_UAR_SIZE);
 	/* anonymous mmap, no real memory consumption. */

index 4eafcb5..8e25d78 100644
@@ -429,23 +429,26 @@ eal_parse_args(int argc, char **argv)
 	return ret;
 }
 
+static int
+check_socket(const struct rte_memseg *ms, void *arg)
+{
+	int *socket_id = arg;
+
+	if (ms->socket_id == *socket_id)
+		return 1;
+
+	return 0;
+}
+
 static void
 eal_check_mem_on_local_socket(void)
 {
-	const struct rte_memseg *ms;
-	int i, socket_id;
+	int socket_id;
 
 	socket_id = rte_lcore_to_socket_id(rte_config.master_lcore);
 
-	ms = rte_eal_get_physmem_layout();
-
-	for (i = 0; i < RTE_MAX_MEMSEG; i++)
-		if (ms[i].socket_id == socket_id &&
-				ms[i].len > 0)
-			return;
-
-	RTE_LOG(WARNING, EAL, "WARNING: Master core has no "
-			"memory on local socket!\n");
+	if (rte_memseg_walk(check_socket, &socket_id) == 0)
+		RTE_LOG(WARNING, EAL, "WARNING: Master core has no memory on local socket!\n");
 }
 
 static int

89 lines not on this page · raw message

index 564b61a..79914fc 100644
@@ -67,17 +67,32 @@ check_hugepage_sz(unsigned flags, uint64_t hugepage_sz)
  * to prevent overflow. The rest of the zone is added to free list as a single
  * large free block
  */
-static void
-malloc_heap_add_memseg(struct malloc_heap *heap, struct rte_memseg *ms)
+static int
+malloc_heap_add_memseg(const struct rte_memseg *ms, void *arg __rte_unused)
 {
-	struct malloc_elem *start_elem = (struct malloc_elem *)ms->addr;
-	const size_t elem_size = ms->len - MALLOC_ELEM_OVERHEAD;
+	struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
+	struct malloc_elem *start_elem;
+	struct rte_memseg *found_ms;
+	struct malloc_heap *heap;
+	size_t elem_size;
+	int ms_idx;
+
+	heap = &mcfg->malloc_heaps[ms->socket_id];
+
+	/* ms is const, so find it */
+	ms_idx = ms - mcfg->memseg;
+	found_ms = &mcfg->memseg[ms_idx];
 
-	malloc_elem_init(start_elem, heap, ms, elem_size);
+	start_elem = (struct malloc_elem *)found_ms->addr;
+	elem_size = ms->len - MALLOC_ELEM_OVERHEAD;
+
+	malloc_elem_init(start_elem, heap, found_ms, elem_size);
 	malloc_elem_insert(start_elem);
 	malloc_elem_free_list_insert(start_elem);
 
 	heap->total_size += elem_size;
+
+	return 0;
 }
 
 /*

@@ -244,17 +259,11 @@ int
 rte_eal_malloc_heap_init(void)
 {
 	struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
-	unsigned ms_cnt;
-	struct rte_memseg *ms;
 
 	if (mcfg == NULL)
 		return -1;
 
-	for (ms = &mcfg->memseg[0], ms_cnt = 0;
-			(ms_cnt < RTE_MAX_MEMSEG) && (ms->len > 0);
-			ms_cnt++, ms++) {
-		malloc_heap_add_memseg(&mcfg->malloc_heaps[ms->socket_id], ms);
-	}
+	rte_memseg_walk(malloc_heap_add_memseg, NULL);
 
 	return 0;
 }

index 2ecd07b..77f6cb7 100644
@@ -638,23 +638,23 @@ eal_parse_args(int argc, char **argv)
 	return ret;
 }
 
+static int
+check_mem(const struct rte_memseg *ms, void *arg)
+{
+	int *socket = arg;
+
+	return ms->socket_id == *socket;
+}
+
 static void
 eal_check_mem_on_local_socket(void)
 {
-	const struct rte_memseg *ms;
-	int i, socket_id;
+	int socket_id;
 
 	socket_id = rte_lcore_to_socket_id(rte_config.master_lcore);
 
-	ms = rte_eal_get_physmem_layout();
-
-	for (i = 0; i < RTE_MAX_MEMSEG; i++)
-		if (ms[i].socket_id == socket_id &&
-				ms[i].len > 0)
-			return;
-
-	RTE_LOG(WARNING, EAL, "WARNING: Master core has no "
-			"memory on local socket!\n");
+	if (rte_memseg_walk(check_mem, &socket_id) == 0)
+		RTE_LOG(WARNING, EAL, "WARNING: Master core has no memory on local socket!\n");
 }
 
 static int

index 4660cc2..9731d4c 100644
@@ -99,23 +99,23 @@ static unsigned optimize_object_size(unsigned obj_size)
 	return new_obj_size * RTE_MEMPOOL_ALIGN;
 }
 
+static int
+find_min_pagesz(const struct rte_memseg *ms, void *arg)
+{
+	size_t *min = arg;
+
+	if (ms->hugepage_sz < *min)
+		*min = ms->hugepage_sz;
+
+	return 0;
+}
+
 static size_t
 get_min_page_size(void)
 {
-	const struct rte_mem_config *mcfg =
-			rte_eal_get_configuration()->mem_config;
-	int i;
 	size_t min_pagesz = SIZE_MAX;
 
-	for (i = 0; i < RTE_MAX_MEMSEG; i++) {
-		const struct rte_memseg *ms = &mcfg->memseg[i];
-
-		if (ms->addr == NULL)
-			continue;
-
-		if (ms->hugepage_sz < min_pagesz)
-			min_pagesz = ms->hugepage_sz;
-	}
+	rte_memseg_walk(find_min_pagesz, &min_pagesz);
 
 	return min_pagesz == SIZE_MAX ? (size_t) getpagesize() : min_pagesz;
 }

index ccc5fea..28c241f 100644
@@ -705,16 +705,34 @@ test_malloc_bad_params(void)
 	return -1;
 }
 
+static int
+check_socket_mem(const struct rte_memseg *ms, void *arg)
+{
+	int32_t *socket = arg;
+
+	return *socket == ms->socket_id;
+}
+
 /* Check if memory is available on a specific socket */
 static int
 is_mem_on_socket(int32_t socket)
 {
-	const struct rte_memseg *ms = rte_eal_get_physmem_layout();
-	unsigned i;
+	return rte_memseg_walk(check_socket_mem, &socket);
+}
 
-	for (i = 0; i < RTE_MAX_MEMSEG; i++) {
-		if (socket == ms[i].socket_id)
-			return 1;
+struct walk_param {
+	void *addr;
+	int32_t socket;
+};
+static int
+find_socket(const struct rte_memseg *ms, void *arg)
+{
+	struct walk_param *param = arg;
+
+	if (param->addr >= ms->addr &&
+			param->addr < RTE_PTR_ADD(ms->addr, ms->len)) {
+		param->socket = ms->socket_id;
+		return 1;
 	}
 	return 0;
 }

@@ -726,15 +744,9 @@ is_mem_on_socket(int32_t socket)
 static int32_t
 addr_to_socket(void * addr)
 {
-	const struct rte_memseg *ms = rte_eal_get_physmem_layout();
-	unsigned i;
-
-	for (i = 0; i < RTE_MAX_MEMSEG; i++) {
-		if ((ms[i].addr <= addr) &&
-				((uintptr_t)addr <
-				((uintptr_t)ms[i].addr + (uintptr_t)ms[i].len)))
-			return ms[i].socket_id;
-	}
+	struct walk_param param = {.addr = addr, .socket = 0};
+	if (rte_memseg_walk(find_socket, &param) > 0)
+		return param.socket;
 	return -1;
 }
 

index 972321f..c9b287c 100644
@@ -23,12 +23,20 @@
  */
 
 static int
+check_mem(const struct rte_memseg *ms, void *arg __rte_unused)
+{
+	volatile uint8_t *mem = (volatile uint8_t *) ms->addr;
+	size_t i;
+
+	for (i = 0; i < ms->len; i++, mem++)
+		*mem;
+	return 0;
+}
+
+static int
 test_memory(void)
 {
 	uint64_t s;
-	unsigned i;
-	size_t j;
-	const struct rte_memseg *mem;
 
 	/*
 	 * dump the mapped memory: the python-expect script checks

@@ -45,14 +53,7 @@ test_memory(void)
 	}
 
 	/* try to read memory (should not segfault) */
-	mem = rte_eal_get_physmem_layout();
-	for (i = 0; i < RTE_MAX_MEMSEG && mem[i].addr != NULL ; i++) {
-
-		/* check memory */
-		for (j = 0; j<mem[i].len; j++) {
-			*((volatile uint8_t *) mem[i].addr + j);
-		}
-	}
+	rte_memseg_walk(check_mem, NULL);
 
 	return 0;
 }

64 lines not on this page · raw message

index 2421d51..2a34ae9 100644
@@ -665,39 +665,36 @@ vfio_get_group_no(const char *sysfs_base,
 }
 
 static int
-vfio_type1_dma_map(int vfio_container_fd)
+type1_map(const struct rte_memseg *ms, void *arg)
 {
-	const struct rte_memseg *ms = rte_eal_get_physmem_layout();
-	int i, ret;
+	int *vfio_container_fd = arg;
+	struct vfio_iommu_type1_dma_map dma_map;
+	int ret;
 
-	/* map all DPDK segments for DMA. use 1:1 PA to IOVA mapping */
-	for (i = 0; i < RTE_MAX_MEMSEG; i++) {
-		struct vfio_iommu_type1_dma_map dma_map;
+	memset(&dma_map, 0, sizeof(dma_map));
+	dma_map.argsz = sizeof(struct vfio_iommu_type1_dma_map);
+	dma_map.vaddr = ms->addr_64;
+	dma_map.size = ms->len;
+	dma_map.iova = ms->iova;
+	dma_map.flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE;
 
-		if (ms[i].addr == NULL)
-			break;
-
-		memset(&dma_map, 0, sizeof(dma_map));
-		dma_map.argsz = sizeof(struct vfio_iommu_type1_dma_map);
-		dma_map.vaddr = ms[i].addr_64;
-		dma_map.size = ms[i].len;
-		dma_map.iova = ms[i].iova;
-		dma_map.flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE;
+	ret = ioctl(*vfio_container_fd, VFIO_IOMMU_MAP_DMA, &dma_map);
 
-		ret = ioctl(vfio_container_fd, VFIO_IOMMU_MAP_DMA, &dma_map);
-
-		if (ret) {
-			RTE_LOG(ERR, EAL, "  cannot set up DMA remapping, "
-					  "error %i (%s)\n", errno,
-					  strerror(errno));
-			return -1;
-		}
+	if (ret) {
+		RTE_LOG(ERR, EAL, "  cannot set up DMA remapping, error %i (%s)\n",
+				errno, strerror(errno));
+		return -1;
 	}
-
 	return 0;
 }
 
 static int
+vfio_type1_dma_map(int vfio_container_fd)
+{
+	return rte_memseg_walk(type1_map, &vfio_container_fd);
+}
+
+static int
 vfio_spapr_dma_map(int vfio_container_fd)
 {
 	const struct rte_memseg *ms = rte_eal_get_physmem_layout();

index 2a34ae9..e18e413 100644
@@ -694,16 +694,69 @@ vfio_type1_dma_map(int vfio_container_fd)
 	return rte_memseg_walk(type1_map, &vfio_container_fd);
 }
 
+struct spapr_walk_param {
+	uint64_t window_size;
+	uint64_t hugepage_sz;
+};
 static int
-vfio_spapr_dma_map(int vfio_container_fd)
+spapr_window_size(const struct rte_memseg *ms, void *arg)
 {
-	const struct rte_memseg *ms = rte_eal_get_physmem_layout();
-	int i, ret;
+	struct spapr_walk_param *param = arg;
+	uint64_t max = ms->iova + ms->len;
+
+	if (max > param->window_size) {
+		param->hugepage_sz = ms->hugepage_sz;
+		param->window_size = max;
+	}
 
+	return 0;
+}
+
+static int
+spapr_map(const struct rte_memseg *ms, void *arg)
+{
+	struct vfio_iommu_type1_dma_map dma_map;
 	struct vfio_iommu_spapr_register_memory reg = {
 		.argsz = sizeof(reg),
 		.flags = 0
 	};
+	int *vfio_container_fd = arg;
+	int ret;
+
+	reg.vaddr = (uintptr_t) ms->addr;
+	reg.size = ms->len;
+	ret = ioctl(*vfio_container_fd,
+		VFIO_IOMMU_SPAPR_REGISTER_MEMORY, &reg);
+	if (ret) {
+		RTE_LOG(ERR, EAL, "  cannot register vaddr for IOMMU, error %i (%s)\n",
+				errno, strerror(errno));
+		return -1;
+	}
+
+	memset(&dma_map, 0, sizeof(dma_map));
+	dma_map.argsz = sizeof(struct vfio_iommu_type1_dma_map);
+	dma_map.vaddr = ms->addr_64;
+	dma_map.size = ms->len;
+	dma_map.iova = ms->iova;
+	dma_map.flags = VFIO_DMA_MAP_FLAG_READ |
+			 VFIO_DMA_MAP_FLAG_WRITE;
+
+	ret = ioctl(*vfio_container_fd, VFIO_IOMMU_MAP_DMA, &dma_map);
+
+	if (ret) {
+		RTE_LOG(ERR, EAL, "  cannot set up DMA remapping, error %i (%s)\n",
+				errno, strerror(errno));
+		return -1;
+	}
+
+	return 0;
+}
+
+static int
+vfio_spapr_dma_map(int vfio_container_fd)
+{
+	struct spapr_walk_param param;
+	int ret;
 	struct vfio_iommu_spapr_tce_info info = {
 		.argsz = sizeof(info),
 	};

@@ -714,6 +767,8 @@ vfio_spapr_dma_map(int vfio_container_fd)
 		.argsz = sizeof(remove),
 	};
 
+	memset(&param, 0, sizeof(param));
+
 	/* query spapr iommu info */
 	ret = ioctl(vfio_container_fd, VFIO_IOMMU_SPAPR_TCE_GET_INFO, &info);
 	if (ret) {

@@ -732,17 +787,11 @@ vfio_spapr_dma_map(int vfio_container_fd)
 	}
 
 	/* create DMA window from 0 to max(phys_addr + len) */
-	for (i = 0; i < RTE_MAX_MEMSEG; i++) {
-		if (ms[i].addr == NULL)
-			break;
-
-		create.window_size = RTE_MAX(create.window_size,
-				ms[i].iova + ms[i].len);
-	}
+	rte_memseg_walk(spapr_window_size, &param);
 
 	/* sPAPR requires window size to be a power of 2 */
-	create.window_size = rte_align64pow2(create.window_size);
-	create.page_shift = __builtin_ctzll(ms->hugepage_sz);
+	create.window_size = rte_align64pow2(param.window_size);
+	create.page_shift = __builtin_ctzll(param.hugepage_sz);
 	create.levels = 1;
 
 	ret = ioctl(vfio_container_fd, VFIO_IOMMU_SPAPR_TCE_CREATE, &create);

@@ -757,41 +806,15 @@ vfio_spapr_dma_map(int vfio_container_fd)
 		return -1;
 	}
 
-	/* map all DPDK segments for DMA. use 1:1 PA to IOVA mapping */
-	for (i = 0; i < RTE_MAX_MEMSEG; i++) {
-		struct vfio_iommu_type1_dma_map dma_map;
-
-		if (ms[i].addr == NULL)
-			break;
-
-		reg.vaddr = (uintptr_t) ms[i].addr;
-		reg.size = ms[i].len;
-		ret = ioctl(vfio_container_fd,
-			VFIO_IOMMU_SPAPR_REGISTER_MEMORY, &reg);
-		if (ret) {
-			RTE_LOG(ERR, EAL, "  cannot register vaddr for IOMMU, "
-				"error %i (%s)\n", errno, strerror(errno));
-			return -1;
-		}
-
-		memset(&dma_map, 0, sizeof(dma_map));
-		dma_map.argsz = sizeof(struct vfio_iommu_type1_dma_map);
-		dma_map.vaddr = ms[i].addr_64;
-		dma_map.size = ms[i].len;
-		dma_map.iova = ms[i].iova;
-		dma_map.flags = VFIO_DMA_MAP_FLAG_READ |
-				 VFIO_DMA_MAP_FLAG_WRITE;
-
-		ret = ioctl(vfio_container_fd, VFIO_IOMMU_MAP_DMA, &dma_map);
-
-		if (ret) {
-			RTE_LOG(ERR, EAL, "  cannot set up DMA remapping, "
-				"error %i (%s)\n", errno, strerror(errno));
-			return -1;
-		}
-
+	if (vfio_spapr_create_new_dma_window(vfio_container_fd, &create) < 0) {
+		RTE_LOG(ERR, EAL, "Could not create new DMA window\n");
+		return -1;
 	}
 
+	/* map all DPDK segments for DMA. use 1:1 PA to IOVA mapping */
+	if (rte_memseg_walk(spapr_map, &vfio_container_fd) < 0)
+		return -1;
+
 	return 0;
 }
 

index 4f588c7..4b528b0 100644
@@ -242,6 +242,43 @@ rte_memseg_walk(rte_memseg_walk_t func, void *arg)
 	return 0;
 }
 
+int __rte_experimental
+rte_memseg_contig_walk(rte_memseg_contig_walk_t func, void *arg)
+{
+	struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
+	int i, j, ret;
+
+	for (i = 0; i < RTE_MAX_MEMSEG; i++) {
+		const struct rte_memseg *ms = &mcfg->memseg[i];
+		size_t total_len;
+		void *end_addr;
+
+		if (ms->addr == NULL)
+			continue;
+
+		end_addr = RTE_PTR_ADD(ms->addr, ms->len);
+
+		/* check how many more segments are contiguous to this one */
+		for (j = i + 1; j < RTE_MAX_MEMSEG; j++) {
+			const struct rte_memseg *next = &mcfg->memseg[j];
+
+			if (next->addr != end_addr)
+				break;
+
+			end_addr = RTE_PTR_ADD(next->addr, next->len);
+			i++;
+		}
+		total_len = RTE_PTR_DIFF(end_addr, ms->addr);
+
+		ret = func(ms, total_len, arg);
+		if (ret < 0)
+			return -1;
+		if (ret > 0)
+			return 1;
+	}
+	return 0;
+}
+
 /* init memory subsystem */
 int
 rte_eal_memory_init(void)

index 93eadaa..45d067f 100644
@@ -140,6 +140,18 @@ rte_iova_t rte_mem_virt2iova(const void *virt);
 typedef int (*rte_memseg_walk_t)(const struct rte_memseg *ms, void *arg);
 
 /**
+ * Memseg contig walk function prototype. This will trigger a callback on every
+ * VA-contiguous are starting at memseg ``ms``, so total valid VA space at each
+ * callback call will be [``ms->addr``, ``ms->addr + len``).
+ *
+ * Returning 0 will continue walk
+ * Returning 1 will stop the walk
+ * Returning -1 will stop the walk and report error
+ */
+typedef int (*rte_memseg_contig_walk_t)(const struct rte_memseg *ms,
+		size_t len, void *arg);
+
+/**
  * Walk list of all memsegs.
  *
  * @param func

@@ -155,6 +167,21 @@ int __rte_experimental
 rte_memseg_walk(rte_memseg_walk_t func, void *arg);
 
 /**
+ * Walk each VA-contiguous area.
+ *
+ * @param func
+ *   Iterator function
+ * @param arg
+ *   Argument passed to iterator
+ * @return
+ *   0 if walked over the entire list
+ *   1 if stopped by the user
+ *   -1 if user function reported error
+ */
+int __rte_experimental
+rte_memseg_contig_walk(rte_memseg_contig_walk_t func, void *arg);
+
+/**
  * Get the layout of the available physical memory.
  *
  * It can be useful for an application to have the full physical

index 1711ead..93d7efe 100644
@@ -70,6 +70,32 @@ static uint64_t vhost_req_user_to_kernel[] = {
 	[VHOST_USER_SET_MEM_TABLE] = VHOST_SET_MEM_TABLE,
 };
 
+struct walk_arg {
+	struct vhost_memory_kernel *vm;
+	uint32_t region_nr;
+};
+static int
+add_memory_region(const struct rte_memseg *ms, size_t len, void *arg)
+{
+	struct walk_arg *wa = arg;
+	struct vhost_memory_region *mr;
+	void *start_addr;
+
+	if (wa->region_nr >= max_regions)
+		return -1;
+
+	mr = &wa->vm->regions[wa->region_nr++];
+	start_addr = ms->addr;
+
+	mr->guest_phys_addr = (uint64_t)(uintptr_t)start_addr;
+	mr->userspace_addr = (uint64_t)(uintptr_t)start_addr;
+	mr->memory_size = len;
+	mr->mmap_offset = 0;
+
+	return 0;
+}
+
+
 /* By default, vhost kernel module allows 64 regions, but DPDK allows
  * 256 segments. As a relief, below function merges those virtually
  * adjacent memsegs into one region.

@@ -77,63 +103,24 @@ static uint64_t vhost_req_user_to_kernel[] = {
 static struct vhost_memory_kernel *
 prepare_vhost_memory_kernel(void)
 {
-	uint32_t i, j, k = 0;
-	struct rte_memseg *seg;
-	struct vhost_memory_region *mr;
 	struct vhost_memory_kernel *vm;
+	struct walk_arg wa;
 
 	vm = malloc(sizeof(struct vhost_memory_kernel) +
-		    max_regions *
-		    sizeof(struct vhost_memory_region));
+			max_regions *
+			sizeof(struct vhost_memory_region));
 	if (!vm)
 		return NULL;
 
-	for (i = 0; i < RTE_MAX_MEMSEG; ++i) {
-		seg = &rte_eal_get_configuration()->mem_config->memseg[i];
-		if (!seg->addr)
-			break;
-
-		int new_region = 1;
-
-		for (j = 0; j < k; ++j) {
-			mr = &vm->regions[j];
+	wa.region_nr = 0;
+	wa.vm = vm;
 
-			if (mr->userspace_addr + mr->memory_size ==
-			    (uint64_t)(uintptr_t)seg->addr) {
-				mr->memory_size += seg->len;
-				new_region = 0;
-				break;
-			}
-
-			if ((uint64_t)(uintptr_t)seg->addr + seg->len ==
-			    mr->userspace_addr) {
-				mr->guest_phys_addr =
-					(uint64_t)(uintptr_t)seg->addr;
-				mr->userspace_addr =
-					(uint64_t)(uintptr_t)seg->addr;
-				mr->memory_size += seg->len;
-				new_region = 0;
-				break;
-			}
-		}
-
-		if (new_region == 0)
-			continue;
-
-		mr = &vm->regions[k++];
-		/* use vaddr here! */
-		mr->guest_phys_addr = (uint64_t)(uintptr_t)seg->addr;
-		mr->userspace_addr = (uint64_t)(uintptr_t)seg->addr;
-		mr->memory_size = seg->len;
-		mr->mmap_offset = 0;
-
-		if (k >= max_regions) {
-			free(vm);
-			return NULL;
-		}
+	if (rte_memseg_contig_walk(add_memory_region, &wa) < 0) {
+		free(vm);
+		return NULL;
 	}
 
-	vm->nregions = k;
+	vm->nregions = wa.region_nr;
 	vm->padding = 0;
 	return vm;
 }

index 4b528b0..ea3c5a7 100644
@@ -131,6 +131,36 @@ rte_eal_get_physmem_layout(void)
 	return rte_eal_get_configuration()->mem_config->memseg;
 }
 
+struct virtiova {
+	rte_iova_t iova;
+	void *virt;
+};
+static int
+find_virt(const struct rte_memseg *ms, void *arg)
+{
+	struct virtiova *vi = arg;
+	if (vi->iova >= ms->iova && vi->iova < (ms->iova + ms->len)) {
+		size_t offset = vi->iova - ms->iova;
+		vi->virt = RTE_PTR_ADD(ms->addr, offset);
+		/* stop the walk */
+		return 1;
+	}
+	return 0;
+}
+
+__rte_experimental void *
+rte_mem_iova2virt(rte_iova_t iova)
+{
+	struct virtiova vi;
+
+	memset(&vi, 0, sizeof(vi));
+
+	vi.iova = iova;
+	rte_memseg_walk(find_virt, &vi);
+
+	return vi.virt;
+}
+
 static int
 physmem_size(const struct rte_memseg *ms, void *arg)
 {

index 45d067f..5c60b91 100644
@@ -131,6 +131,18 @@ phys_addr_t rte_mem_virt2phy(const void *virt);
 rte_iova_t rte_mem_virt2iova(const void *virt);
 
 /**
+ * Get virtual memory address corresponding to iova address.
+ *
+ * @param iova
+ *   The iova address.
+ * @return
+ *   Virtual address corresponding to iova address (or NULL if address does not
+ *   exist within DPDK memory map).
+ */
+__rte_experimental void *
+rte_mem_iova2virt(rte_iova_t iova);
+
+/**
  * Memseg walk function prototype.
  *
  * Returning 0 will continue walk

index 718701b..89aeac2 100644
@@ -98,17 +98,7 @@ struct dpaa_portal {
 /* TODO - this is costly, need to write a fast coversion routine */
 static inline void *rte_dpaa_mem_ptov(phys_addr_t paddr)
 {
-	const struct rte_memseg *memseg = rte_eal_get_physmem_layout();
-	int i;
-
-	for (i = 0; i < RTE_MAX_MEMSEG && memseg[i].addr != NULL; i++) {
-		if (paddr >= memseg[i].iova && paddr <
-			memseg[i].iova + memseg[i].len)
-			return (uint8_t *)(memseg[i].addr) +
-			       (paddr - memseg[i].iova);
-	}
-
-	return NULL;
+	return rte_mem_iova2virt(paddr);
 }
 
 /**

index 4a19d42..d38fc49 100644
@@ -260,21 +260,10 @@ static void *dpaa2_mem_ptov(phys_addr_t paddr) __attribute__((unused));
 /* todo - this is costly, need to write a fast coversion routine */
 static void *dpaa2_mem_ptov(phys_addr_t paddr)
 {
-	const struct rte_memseg *memseg;
-	int i;
-
 	if (dpaa2_virt_mode)
 		return (void *)(size_t)paddr;
 
-	memseg = rte_eal_get_physmem_layout();
-
-	for (i = 0; i < RTE_MAX_MEMSEG && memseg[i].addr_64 != 0; i++) {
-		if (paddr >= memseg[i].iova &&
-		    paddr < memseg[i].iova + memseg[i].len)
-			return (void *)(size_t)(memseg[i].addr_64
-				+ (paddr - memseg[i].iova));
-	}
-	return NULL;
+	return rte_mem_iova2virt(paddr);
 }
 
 static phys_addr_t dpaa2_mem_vtop(uint64_t vaddr) __attribute__((unused));

index c5191ce..b04510f 100644
@@ -120,16 +120,7 @@ dpaa_mem_vtop_ctx(struct dpaa_sec_op_ctx *ctx, void *vaddr)
 static inline void *
 dpaa_mem_ptov(rte_iova_t paddr)
 {
-	const struct rte_memseg *memseg = rte_eal_get_physmem_layout();
-	int i;
-
-	for (i = 0; i < RTE_MAX_MEMSEG && memseg[i].addr_64 != 0; i++) {
-		if (paddr >= memseg[i].iova &&
-		    paddr < memseg[i].iova + memseg[i].len)
-			return (void *)(size_t)(memseg[i].addr_64 +
-					(paddr - memseg[i].iova));
-	}
-	return NULL;
+	return rte_mem_iova2virt(paddr);
 }
 
 static void

index ea3c5a7..fd78d2f 100644
@@ -161,6 +161,43 @@ rte_mem_iova2virt(rte_iova_t iova)
 	return vi.virt;
 }
 
+struct virtms {
+	const void *virt;
+	struct rte_memseg *ms;
+};
+static int
+find_memseg(const struct rte_memseg *ms, void *arg)
+{
+	struct virtms *vm = arg;
+
+	if (arg >= ms->addr && arg < RTE_PTR_ADD(ms->addr, ms->len)) {
+		struct rte_memseg *memseg, *found_ms;
+		int idx;
+
+		memseg = rte_eal_get_configuration()->mem_config->memseg;
+		idx = ms - memseg;
+		found_ms = &memseg[idx];
+
+		vm->ms = found_ms;
+		return 1;
+	}
+	return 0;
+}
+
+__rte_experimental struct rte_memseg *
+rte_mem_virt2memseg(const void *addr)
+{
+	struct virtms vm;
+
+	memset(&vm, 0, sizeof(vm));
+
+	vm.virt = addr;
+
+	rte_memseg_walk(find_memseg, &vm);
+
+	return vm.ms;
+}
+
 static int
 physmem_size(const struct rte_memseg *ms, void *arg)
 {

index 5c60b91..b3d7e61 100644
@@ -143,6 +143,17 @@ __rte_experimental void *
 rte_mem_iova2virt(rte_iova_t iova);
 
 /**
+ * Get memseg to which a particular virtual address belongs.
+ *
+ * @param virt
+ *   The virtual address.
+ * @return
+ *   Memseg pointer on success, or NULL on error.
+ */
+__rte_experimental struct rte_memseg *
+rte_mem_virt2memseg(const void *virt);
+
+/**
  * Memseg walk function prototype.
  *
  * Returning 0 will continue walk

index d38fc49..45fd41e 100644
@@ -270,20 +270,14 @@ static phys_addr_t dpaa2_mem_vtop(uint64_t vaddr) __attribute__((unused));
 static phys_addr_t dpaa2_mem_vtop(uint64_t vaddr)
 {
 	const struct rte_memseg *memseg;
-	int i;
 
 	if (dpaa2_virt_mode)
 		return vaddr;
 
-	memseg = rte_eal_get_physmem_layout();
-
-	for (i = 0; i < RTE_MAX_MEMSEG && memseg[i].addr_64 != 0; i++) {
-		if (vaddr >= memseg[i].addr_64 &&
-		    vaddr < memseg[i].addr_64 + memseg[i].len)
-			return memseg[i].iova
-				+ (vaddr - memseg[i].addr_64);
-	}
-	return (size_t)(NULL);
+	memseg = rte_mem_virt2memseg((void *)(uintptr_t)vaddr);
+	if (memseg)
+		return memseg->phys_addr + RTE_PTR_DIFF(vaddr, memseg->addr);
+	return (size_t)NULL;
 }
 
 /**

index b04510f..a14e669 100644
@@ -93,20 +93,11 @@ dpaa_sec_alloc_ctx(dpaa_sec_session *ses)
 static inline rte_iova_t
 dpaa_mem_vtop(void *vaddr)
 {
-	const struct rte_memseg *memseg = rte_eal_get_physmem_layout();
-	uint64_t vaddr_64, paddr;
-	int i;
-
-	vaddr_64 = (size_t)vaddr;
-	for (i = 0; i < RTE_MAX_MEMSEG && memseg[i].addr_64 != 0; i++) {
-		if (vaddr_64 >= memseg[i].addr_64 &&
-		    vaddr_64 < memseg[i].addr_64 + memseg[i].len) {
-			paddr = memseg[i].iova +
-				(vaddr_64 - memseg[i].addr_64);
-
-			return (rte_iova_t)paddr;
-		}
-	}
+	const struct rte_memseg *ms;
+
+	ms = rte_mem_virt2memseg(vaddr);
+	if (ms)
+		return ms->iova + RTE_PTR_DIFF(vaddr, ms->addr);
 	return (size_t)NULL;
 }
 

index 9a1e4de..7ca1560 100644
@@ -126,10 +126,9 @@ mlx4_check_mempool(struct rte_mempool *mp, uintptr_t *start, uintptr_t *end)
 struct mlx4_mr *
 mlx4_mr_get(struct priv *priv, struct rte_mempool *mp)
 {
-	const struct rte_memseg *ms = rte_eal_get_physmem_layout();
+	const struct rte_memseg *ms;
 	uintptr_t start;
 	uintptr_t end;
-	unsigned int i;
 	struct mlx4_mr *mr;
 
 	if (mlx4_check_mempool(mp, &start, &end) != 0) {

@@ -142,16 +141,13 @@ mlx4_mr_get(struct priv *priv, struct rte_mempool *mp)
 	      (void *)mp, (void *)start, (void *)end,
 	      (size_t)(end - start));
 	/* Round start and end to page boundary if found in memory segments. */
-	for (i = 0; (i < RTE_MAX_MEMSEG) && (ms[i].addr != NULL); ++i) {
-		uintptr_t addr = (uintptr_t)ms[i].addr;
-		size_t len = ms[i].len;
-		unsigned int align = ms[i].hugepage_sz;
+	ms = rte_mem_virt2memseg((void *)start);
+	if (ms != NULL)
+		start = RTE_ALIGN_FLOOR(start, ms->hugepage_sz);
+	ms = rte_mem_virt2memseg((void *)end);
+	if (ms != NULL)
+		end = RTE_ALIGN_CEIL(end, ms->hugepage_sz);
 
-		if ((start > addr) && (start < addr + len))
-			start = RTE_ALIGN_FLOOR(start, align);
-		if ((end > addr) && (end < addr + len))
-			end = RTE_ALIGN_CEIL(end, align);
-	}
 	DEBUG("mempool %p using start=%p end=%p size=%zu for MR",
 	      (void *)mp, (void *)start, (void *)end,
 	      (size_t)(end - start));

index 2bf1f9c..ef9b5ba 100644
@@ -234,10 +234,9 @@ struct mlx5_mr *
 mlx5_mr_new(struct rte_eth_dev *dev, struct rte_mempool *mp)
 {
 	struct priv *priv = dev->data->dev_private;
-	const struct rte_memseg *ms = rte_eal_get_physmem_layout();
+	const struct rte_memseg *ms;
 	uintptr_t start;
 	uintptr_t end;
-	unsigned int i;
 	struct mlx5_mr *mr;
 
 	mr = rte_zmalloc_socket(__func__, sizeof(*mr), 0, mp->socket_id);

@@ -261,17 +260,15 @@ mlx5_mr_new(struct rte_eth_dev *dev, struct rte_mempool *mp)
 	/* Save original addresses for exact MR lookup. */
 	mr->start = start;
 	mr->end = end;
+
 	/* Round start and end to page boundary if found in memory segments. */
-	for (i = 0; (i < RTE_MAX_MEMSEG) && (ms[i].addr != NULL); ++i) {
-		uintptr_t addr = (uintptr_t)ms[i].addr;
-		size_t len = ms[i].len;
-		unsigned int align = ms[i].hugepage_sz;
+	ms = rte_mem_virt2memseg((void *)start);
+	if (ms != NULL)
+		start = RTE_ALIGN_FLOOR(start, ms->hugepage_sz);
+	ms = rte_mem_virt2memseg((void *)end);
+	if (ms != NULL)
+		end = RTE_ALIGN_CEIL(end, ms->hugepage_sz);
 
-		if ((start > addr) && (start < addr + len))
-			start = RTE_ALIGN_FLOOR(start, align);
-		if ((end > addr) && (end < addr + len))
-			end = RTE_ALIGN_CEIL(end, align);
-	}
 	DRV_LOG(DEBUG,
 		"port %u mempool %p using start=%p end=%p size=%zu for memory"
 		" region",

index af68c00..d60bde7 100644
@@ -360,31 +360,31 @@ rte_memzone_lookup(const char *name)
 	return memzone;
 }
 
+static void
+dump_memzone(const struct rte_memzone *mz, void *arg)
+{
+	struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
+	FILE *f = arg;
+	int mz_idx;
+
+	mz_idx = mz - mcfg->memzone;
+
+	fprintf(f, "Zone %u: name:<%s>, IO:0x%"PRIx64", len:0x%zx, virt:%p, "
+				"socket_id:%"PRId32", flags:%"PRIx32"\n",
+			mz_idx,
+			mz->name,
+			mz->iova,
+			mz->len,
+			mz->addr,
+			mz->socket_id,
+			mz->flags);
+}
+
 /* Dump all reserved memory zones on console */
 void
 rte_memzone_dump(FILE *f)
 {
-	struct rte_mem_config *mcfg;
-	unsigned i = 0;
-
-	/* get pointer to global configuration */
-	mcfg = rte_eal_get_configuration()->mem_config;
-
-	rte_rwlock_read_lock(&mcfg->mlock);
-	/* dump all zones */
-	for (i=0; i<RTE_MAX_MEMZONE; i++) {
-		if (mcfg->memzone[i].addr == NULL)
-			break;
-		fprintf(f, "Zone %u: name:<%s>, IO:0x%"PRIx64", len:0x%zx"
-		       ", virt:%p, socket_id:%"PRId32", flags:%"PRIx32"\n", i,
-		       mcfg->memzone[i].name,
-		       mcfg->memzone[i].iova,
-		       mcfg->memzone[i].len,
-		       mcfg->memzone[i].addr,
-		       mcfg->memzone[i].socket_id,
-		       mcfg->memzone[i].flags);
-	}
-	rte_rwlock_read_unlock(&mcfg->mlock);
+	rte_memzone_walk(dump_memzone, f);
 }
 
 /*

index 8e25d78..032a5ea 100644
@@ -749,6 +749,8 @@ int rte_vfio_enable(const char *modname);
 int rte_vfio_is_enabled(const char *modname);
 int rte_vfio_noiommu_is_enabled(void);
 int rte_vfio_clear_group(int vfio_group_fd);
+int rte_vfio_dma_map(uint64_t vaddr, uint64_t iova, uint64_t len);
+int rte_vfio_dma_unmap(uint64_t vaddr, uint64_t iova, uint64_t len);
 
 int rte_vfio_setup_device(__rte_unused const char *sysfs_base,
 		      __rte_unused const char *dev_addr,

@@ -784,3 +786,17 @@ int rte_vfio_clear_group(__rte_unused int vfio_group_fd)
 {
 	return 0;
 }
+
+int __rte_experimental
+rte_vfio_dma_map(uint64_t __rte_unused vaddr, __rte_unused uint64_t iova,
+		  __rte_unused uint64_t len)
+{
+	return -1;
+}
+
+int __rte_experimental
+rte_vfio_dma_unmap(uint64_t __rte_unused vaddr, uint64_t __rte_unused iova,
+		    __rte_unused uint64_t len)
+{
+	return -1;
+}

47 lines not on this page · raw message

832 lines not on this page · raw message

index 8059577..549f442 100644
@@ -19,6 +19,7 @@
 
 #ifdef VFIO_PRESENT
 
+#include <stdint.h>
 #include <linux/vfio.h>
 
 #define RTE_VFIO_TYPE1 VFIO_TYPE1_IOMMU

@@ -26,6 +27,7 @@
 #ifndef VFIO_SPAPR_TCE_v2_IOMMU
 #define RTE_VFIO_SPAPR 7
 #define VFIO_IOMMU_SPAPR_REGISTER_MEMORY _IO(VFIO_TYPE, VFIO_BASE + 17)
+#define VFIO_IOMMU_SPAPR_UNREGISTER_MEMORY _IO(VFIO_TYPE, VFIO_BASE + 18)
 #define VFIO_IOMMU_SPAPR_TCE_CREATE _IO(VFIO_TYPE, VFIO_BASE + 19)
 #define VFIO_IOMMU_SPAPR_TCE_REMOVE _IO(VFIO_TYPE, VFIO_BASE + 20)
 

@@ -110,6 +112,7 @@ struct vfio_config {
 	int vfio_enabled;
 	int vfio_container_fd;
 	int vfio_active_groups;
+	const struct vfio_iommu_type *vfio_iommu_type;
 	struct vfio_group vfio_groups[VFIO_MAX_GROUPS];
 };
 

@@ -119,9 +122,18 @@ struct vfio_config {
  * */
 typedef int (*vfio_dma_func_t)(int);
 
+/* Custom memory region DMA mapping function prototype.
+ * Takes VFIO container fd, virtual address, phisical address, length and
+ * operation type (0 to unmap 1 for map) as a parameters.
+ * Returns 0 on success, -1 on error.
+ **/
+typedef int (*vfio_dma_user_func_t)(int fd, uint64_t vaddr, uint64_t iova,
+		uint64_t len, int do_map);
+
 struct vfio_iommu_type {
 	int type_id;
 	const char *name;
+	vfio_dma_user_func_t dma_user_map_func;
 	vfio_dma_func_t dma_map_func;
 };
 

index 032a5ea..f44b904 100644
@@ -534,6 +534,9 @@ rte_eal_init(int argc, char **argv)
 		return -1;
 	}
 
+	/* FreeBSD always uses legacy memory model */
+	internal_config.legacy_mem = true;
+
 	if (eal_plugins_init() < 0) {
 		rte_eal_init_alert("Cannot init plugins\n");
 		rte_errno = EINVAL;

index 8a51ade..fb5ea03 100644
@@ -73,6 +73,7 @@ eal_long_options[] = {
 	{OPT_VDEV,              1, NULL, OPT_VDEV_NUM             },
 	{OPT_VFIO_INTR,         1, NULL, OPT_VFIO_INTR_NUM        },
 	{OPT_VMWARE_TSC_MAP,    0, NULL, OPT_VMWARE_TSC_MAP_NUM   },
+	{OPT_LEGACY_MEM,        0, NULL, OPT_LEGACY_MEM_NUM       },
 	{0,                     0, NULL, 0                        }
 };
 

@@ -1184,6 +1185,9 @@ eal_parse_common_option(int opt, const char *optarg,
 
 		core_parsed = LCORE_OPT_MAP;
 		break;
+	case OPT_LEGACY_MEM_NUM:
+		conf->legacy_mem = 1;
+		break;
 
 	/* don't know what to do, leave this to caller */
 	default:

index a0082d1..fda087b 100644
@@ -47,6 +47,10 @@ struct internal_config {
 	volatile unsigned force_sockets;
 	volatile uint64_t socket_mem[RTE_MAX_NUMA_NODES]; /**< amount of memory per socket */
 	uintptr_t base_virtaddr;          /**< base address to try and reserve memory from */
+	volatile unsigned legacy_mem;
+	/**< true to enable legacy memory behavior (no dynamic allocation,
+	 * IOVA-contiguous segments).
+	 */
 	volatile int syslog_facility;	  /**< facility passed to openlog() */
 	/** default interrupt mode for VFIO */
 	volatile enum rte_intr_mode vfio_intr_mode;

index e86c711..d301d0b 100644
@@ -55,6 +55,8 @@ enum {
 	OPT_VFIO_INTR_NUM,
 #define OPT_VMWARE_TSC_MAP    "vmware-tsc-map"
 	OPT_VMWARE_TSC_MAP_NUM,
+#define OPT_LEGACY_MEM    "legacy-mem"
+	OPT_LEGACY_MEM_NUM,
 	OPT_LONG_MAX_NUM
 };
 

index 77f6cb7..b34e57a 100644
@@ -348,6 +348,7 @@ eal_usage(const char *prgname)
 	       "  --"OPT_BASE_VIRTADDR"     Base virtual address\n"
 	       "  --"OPT_CREATE_UIO_DEV"    Create /dev/uioX (usually done by hotplug)\n"
 	       "  --"OPT_VFIO_INTR"         Interrupt mode for VFIO (legacy|msi|msix)\n"
+	       "  --"OPT_LEGACY_MEM"        Legacy memory mode (no dynamic allocation, contiguous segments)\n"
 	       "\n");
 	/* Allow the application to print its usage message too if hook is set */
 	if ( rte_application_usage_hook ) {

@@ -767,6 +768,8 @@ rte_eal_init(int argc, char **argv)
 		rte_atomic32_clear(&run_once);
 		return -1;
 	}
+	/* for now, always set legacy mem */
+	internal_config.legacy_mem = 1;
 
 	if (eal_plugins_init() < 0) {
 		rte_eal_init_alert("Cannot init plugins\n");

index 24e6b50..17c559f 100644
@@ -922,8 +922,8 @@ huge_recover_sigbus(void)
  *  6. unmap the first mapping
  *  7. fill memsegs in configuration with contiguous zones
  */
-int
-rte_eal_hugepage_init(void)
+static int
+eal_legacy_hugepage_init(void)
 {
 	struct rte_mem_config *mcfg;
 	struct hugepage_file *hugepage = NULL, *tmp_hp = NULL;

@@ -1266,8 +1266,8 @@ getFileSize(int fd)
  * configuration and finds the hugepages which form that segment, mapping them
  * in order to form a contiguous block in the virtual memory space
  */
-int
-rte_eal_hugepage_attach(void)
+static int
+eal_legacy_hugepage_attach(void)
 {
 	const struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
 	struct hugepage_file *hp = NULL;

@@ -1403,6 +1403,22 @@ rte_eal_hugepage_attach(void)
 }
 
 int
+rte_eal_hugepage_init(void)
+{
+	if (internal_config.legacy_mem)
+		return eal_legacy_hugepage_init();
+	return -1;
+}
+
+int
+rte_eal_hugepage_attach(void)
+{
+	if (internal_config.legacy_mem)
+		return eal_legacy_hugepage_attach();
+	return -1;
+}
+
+int
 rte_eal_using_phys_addrs(void)
 {
 	return phys_addrs_available;

859 lines not on this page · raw message

index 4708dd5..1c6048b 100644
@@ -13,6 +13,7 @@
 
 /** Path of rte config file. */
 #define RUNTIME_CONFIG_FMT "%s/.%s_config"
+#define FBARRAY_FMT "%s/%s_%s"
 
 #include <stdint.h>
 #include <limits.h>

@@ -55,6 +56,18 @@ eal_mp_socket_path(void)
 	return buffer;
 }
 
+static inline const char *
+eal_get_fbarray_path(char *buffer, size_t buflen, const char *name) {
+	const char *directory = "/tmp";
+	const char *home_dir = getenv("HOME");
+
+	if (getuid() != 0 && home_dir != NULL)
+		directory = home_dir;
+	snprintf(buffer, buflen - 1, FBARRAY_FMT, directory,
+			internal_config.hugefile_prefix, name);
+	return buffer;
+}
+
 /** Path of hugepage info file. */
 #define HUGEPAGE_INFO_FMT "%s/.%s_hugepage_info"
 

353 lines not on this page · raw message

index db6ceb6..f293d9e 100644
@@ -21,7 +21,12 @@
 /****** library defines ********/
 
 /* EAL defines */
-#define RTE_MAX_MEMSEG 512
+#define RTE_MAX_MEMSEG_LISTS 128
+#define RTE_MAX_MEMSEG_PER_LIST 8192
+#define RTE_MAX_MEM_MB_PER_LIST 32768
+#define RTE_MAX_MEMSEG_PER_TYPE 32768
+#define RTE_MAX_MEM_MB_PER_TYPE 65536
+#define RTE_MAX_MEM_MB 524288
 #define RTE_MAX_MEMZONE 2560
 #define RTE_MAX_TAILQ 32
 #define RTE_LOG_DP_LEVEL RTE_LOG_INFO

index 0c048dc..8b15312 100644
@@ -190,7 +190,8 @@ static int vfio_map_irq_region(struct fslmc_vfio_group *group)
 }
 
 static int
-fslmc_vfio_map(const struct rte_memseg *ms, void *arg)
+fslmc_vfio_map(const struct rte_memseg_list *msl __rte_unused,
+		const struct rte_memseg *ms, void *arg)
 {
 	int *n_segs = arg;
 	struct fslmc_vfio_group *group;

@@ -232,18 +233,11 @@ fslmc_vfio_map(const struct rte_memseg *ms, void *arg)
 
 int rte_fslmc_vfio_dmamap(void)
 {
-	const struct rte_memseg *memseg;
 	int i = 0;
 
 	if (is_dma_done)
 		return 0;
 
-	memseg = rte_eal_get_physmem_layout();
-	if (memseg == NULL) {
-		DPAA2_BUS_ERR("Cannot get physical layout");
-		return -ENODEV;
-	}
-
 	if (rte_memseg_walk(fslmc_vfio_map, &i) < 0)
 		return -1;
 

index 45fd41e..72aae43 100644
@@ -274,7 +274,7 @@ static phys_addr_t dpaa2_mem_vtop(uint64_t vaddr)
 	if (dpaa2_virt_mode)
 		return vaddr;
 
-	memseg = rte_mem_virt2memseg((void *)(uintptr_t)vaddr);
+	memseg = rte_mem_virt2memseg((void *)(uintptr_t)vaddr, NULL);
 	if (memseg)
 		return memseg->phys_addr + RTE_PTR_DIFF(vaddr, memseg->addr);
 	return (size_t)NULL;

index 6dda054..4630a80 100644
@@ -117,9 +117,10 @@ rte_pci_unmap_device(struct rte_pci_device *dev)
 }
 
 static int
-find_max_end_va(const struct rte_memseg *ms, void *arg)
+find_max_end_va(const struct rte_memseg_list *msl, void *arg)
 {
-	void *end_va = RTE_PTR_ADD(ms->addr, ms->len);
+	size_t sz = msl->memseg_arr.len * msl->page_sz;
+	void *end_va = RTE_PTR_ADD(msl->base_va, sz);
 	void **max_va = arg;
 
 	if (*max_va < end_va)

@@ -132,10 +133,11 @@ pci_find_max_end_va(void)
 {
 	void *va = NULL;
 
-	rte_memseg_walk(find_max_end_va, &va);
+	rte_memseg_list_walk(find_max_end_va, &va);
 	return va;
 }
 
+
 /* parse one line of the "resource" sysfs file (note that the 'line'
  * string is modified)
  */

index a14e669..b685220 100644
@@ -95,7 +95,7 @@ dpaa_mem_vtop(void *vaddr)
 {
 	const struct rte_memseg *ms;
 
-	ms = rte_mem_virt2memseg(vaddr);
+	ms = rte_mem_virt2memseg(vaddr, NULL);
 	if (ms)
 		return ms->iova + RTE_PTR_DIFF(vaddr, ms->addr);
 	return (size_t)NULL;

index 7ca1560..7f0e52c 100644
@@ -141,10 +141,10 @@ mlx4_mr_get(struct priv *priv, struct rte_mempool *mp)
 	      (void *)mp, (void *)start, (void *)end,
 	      (size_t)(end - start));
 	/* Round start and end to page boundary if found in memory segments. */
-	ms = rte_mem_virt2memseg((void *)start);
+	ms = rte_mem_virt2memseg((void *)start, NULL);
 	if (ms != NULL)
 		start = RTE_ALIGN_FLOOR(start, ms->hugepage_sz);
-	ms = rte_mem_virt2memseg((void *)end);
+	ms = rte_mem_virt2memseg((void *)end, NULL);
 	if (ms != NULL)
 		end = RTE_ALIGN_CEIL(end, ms->hugepage_sz);
 

index 1724b65..e228356 100644
@@ -478,7 +478,8 @@ static struct rte_pci_driver mlx5_driver;
 static void *uar_base;
 
 static int
-find_lower_va_bound(const struct rte_memseg *ms, void *arg)
+find_lower_va_bound(const struct rte_memseg_list *msl __rte_unused,
+		const struct rte_memseg *ms, void *arg)
 {
 	void **addr = arg;
 

index ef9b5ba..dd499b7 100644
@@ -262,10 +262,10 @@ mlx5_mr_new(struct rte_eth_dev *dev, struct rte_mempool *mp)
 	mr->end = end;
 
 	/* Round start and end to page boundary if found in memory segments. */
-	ms = rte_mem_virt2memseg((void *)start);
+	ms = rte_mem_virt2memseg((void *)start, NULL);
 	if (ms != NULL)
 		start = RTE_ALIGN_FLOOR(start, ms->hugepage_sz);
-	ms = rte_mem_virt2memseg((void *)end);
+	ms = rte_mem_virt2memseg((void *)end, NULL);
 	if (ms != NULL)
 		end = RTE_ALIGN_CEIL(end, ms->hugepage_sz);
 

index 93d7efe..b244409 100644
@@ -75,7 +75,8 @@ struct walk_arg {
 	uint32_t region_nr;
 };
 static int
-add_memory_region(const struct rte_memseg *ms, size_t len, void *arg)
+add_memory_region(const struct rte_memseg_list *msl __rte_unused,
+		const struct rte_memseg *ms, size_t len, void *arg)
 {
 	struct walk_arg *wa = arg;
 	struct vhost_memory_region *mr;

@@ -95,7 +96,6 @@ add_memory_region(const struct rte_memseg *ms, size_t len, void *arg)
 	return 0;
 }
 
-
 /* By default, vhost kernel module allows 64 regions, but DPDK allows
  * 256 segments. As a relief, below function merges those virtually
  * adjacent memsegs into one region.

index f44b904..d009cf0 100644
@@ -64,8 +64,8 @@ static int mem_cfg_fd = -1;
 static struct flock wr_lock = {
 		.l_type = F_WRLCK,
 		.l_whence = SEEK_SET,
-		.l_start = offsetof(struct rte_mem_config, memseg),
-		.l_len = sizeof(early_mem_config.memseg),
+		.l_start = offsetof(struct rte_mem_config, memsegs),
+		.l_len = sizeof(early_mem_config.memsegs),
 };
 
 /* Address of global and public configuration */

@@ -430,11 +430,11 @@ eal_parse_args(int argc, char **argv)
 }
 
 static int
-check_socket(const struct rte_memseg *ms, void *arg)
+check_socket(const struct rte_memseg_list *msl, void *arg)
 {
 	int *socket_id = arg;
 
-	if (ms->socket_id == *socket_id)
+	if (msl->socket_id == *socket_id && msl->memseg_arr.count != 0)
 		return 1;
 
 	return 0;

@@ -447,10 +447,11 @@ eal_check_mem_on_local_socket(void)
 
 	socket_id = rte_lcore_to_socket_id(rte_config.master_lcore);
 
-	if (rte_memseg_walk(check_socket, &socket_id) == 0)
+	if (rte_memseg_list_walk(check_socket, &socket_id) == 0)
 		RTE_LOG(WARNING, EAL, "WARNING: Master core has no memory on local socket!\n");
 }
 
+
 static int
 sync_func(__attribute__((unused)) void *arg)
 {

@@ -561,7 +562,6 @@ rte_eal_init(int argc, char **argv)
 	rte_eal_get_configuration()->iova_mode = rte_bus_get_iommu_class();
 
 	if (internal_config.no_hugetlbfs == 0 &&
-			internal_config.process_type != RTE_PROC_SECONDARY &&
 			eal_hugepage_info_init() < 0) {
 		rte_eal_init_alert("Cannot get hugepage information.");
 		rte_errno = EACCES;

index be2dbf0..ba44da0 100644
@@ -47,12 +47,18 @@ eal_hugepage_info_init(void)
 	struct hugepage_info *hpi = &internal_config.hugepage_info[0];
 	struct hugepage_info *tmp_hpi;
 
+	internal_config.num_hugepage_sizes = 1;
+
+	/* nothing more to be done for secondary */
+	if (rte_eal_process_type() == RTE_PROC_SECONDARY)
+		return 0;
+
 	sysctl_size = sizeof(num_buffers);
 	error = sysctlbyname("hw.contigmem.num_buffers", &num_buffers,
 			&sysctl_size, NULL, 0);
 
 	if (error != 0) {
-		RTE_LOG(ERR, EAL, "could not read sysctl hw.contigmem.num_buffers");
+		RTE_LOG(ERR, EAL, "could not read sysctl hw.contigmem.num_buffers\n");
 		return -1;
 	}
 

@@ -61,7 +67,7 @@ eal_hugepage_info_init(void)
 			&sysctl_size, NULL, 0);
 
 	if (error != 0) {
-		RTE_LOG(ERR, EAL, "could not read sysctl hw.contigmem.buffer_size");
+		RTE_LOG(ERR, EAL, "could not read sysctl hw.contigmem.buffer_size\n");
 		return -1;
 	}
 

@@ -81,22 +87,21 @@ eal_hugepage_info_init(void)
 		RTE_LOG(INFO, EAL, "Contigmem driver has %d buffers, each of size %dKB\n",
 				num_buffers, (int)(buffer_size>>10));
 
-	internal_config.num_hugepage_sizes = 1;
 	hpi->hugedir = CONTIGMEM_DEV;
 	hpi->hugepage_sz = buffer_size;
 	hpi->num_pages[0] = num_buffers;
 	hpi->lock_descriptor = fd;
 
 	tmp_hpi = create_shared_memory(eal_hugepage_info_path(),
-					sizeof(struct hugepage_info));
+			sizeof(internal_config.hugepage_info));
 	if (tmp_hpi == NULL ) {
 		RTE_LOG(ERR, EAL, "Failed to create shared memory!\n");
 		return -1;
 	}
 
-	memcpy(tmp_hpi, hpi, sizeof(struct hugepage_info));
+	memcpy(tmp_hpi, hpi, sizeof(internal_config.hugepage_info));
 
-	if ( munmap(tmp_hpi, sizeof(struct hugepage_info)) < 0) {
+	if (munmap(tmp_hpi, sizeof(internal_config.hugepage_info)) < 0) {
 		RTE_LOG(ERR, EAL, "Failed to unmap shared memory!\n");
 		return -1;
 	}

287 lines not on this page · raw message

706 lines not on this page · raw message

84 lines not on this page · raw message

index 1d519bb..ad1b0b6 100644
@@ -22,7 +22,6 @@ struct hugepage_file {
 	size_t size;        /**< the page size */
 	int socket_id;      /**< NUMA socket ID */
 	int file_id;        /**< the '%d' in HUGEFILE_FMT */
-	int memseg_id;      /**< the memory segment to which page belongs */
 	char filepath[MAX_HUGEPAGE_PATH]; /**< path to backing file on filesystem */
 };
 

index fda087b..5cf7102 100644
@@ -23,7 +23,7 @@ struct hugepage_info {
 	uint64_t hugepage_sz;   /**< size of a huge page */
 	const char *hugedir;    /**< dir where hugetlbfs is mounted */
 	uint32_t num_pages[RTE_MAX_NUMA_NODES];
-				/**< number of hugepages of that size on each socket */
+	/**< number of hugepages of that size on each socket */
 	int lock_descriptor;    /**< file descriptor for hugepage dir */
 };
 

index 29fa0b6..b745e18 100644
@@ -12,12 +12,30 @@
 #include <rte_malloc_heap.h>
 #include <rte_rwlock.h>
 #include <rte_pause.h>
+#include <rte_fbarray.h>
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
 /**
+ * memseg list is a special case as we need to store a bunch of other data
+ * together with the array itself.
+ */
+struct rte_memseg_list {
+	RTE_STD_C11
+	union {
+		void *base_va;
+		/**< Base virtual address for this memseg list. */
+		uint64_t addr_64;
+		/**< Makes sure addr is always 64-bits */
+	};
+	int socket_id; /**< Socket ID for all memsegs in this list. */
+	uint64_t page_sz; /**< Page size for all memsegs in this list. */
+	struct rte_fbarray memseg_arr;
+};
+
+/**
  * the structure for the memory configuration for the RTE.
  * Used by the rte_config structure. It is separated out, as for multi-process
  * support, the memory details should be shared across instances

@@ -43,9 +61,11 @@ struct rte_mem_config {
 	uint32_t memzone_cnt; /**< Number of allocated memzones */
 
 	/* memory segments and zones */
-	struct rte_memseg memseg[RTE_MAX_MEMSEG];    /**< Physmem descriptors. */
 	struct rte_memzone memzone[RTE_MAX_MEMZONE]; /**< Memzone descriptors. */
 
+	struct rte_memseg_list memsegs[RTE_MAX_MEMSEG_LISTS];
+	/**< list of dynamic arrays holding memsegs */
+
 	struct rte_tailq_head tailq_head[RTE_MAX_TAILQ]; /**< Tailqs for objects */
 
 	/* Heaps of Malloc per socket */

89 lines not on this page · raw message

index e2630fd..0eeb94f 100644
@@ -68,7 +68,6 @@ struct rte_memzone {
 	int32_t socket_id;                /**< NUMA socket ID. */
 
 	uint32_t flags;                   /**< Characteristics of this memzone. */
-	uint32_t memseg_id;               /**< Memseg it belongs. */
 } __attribute__((__packed__));
 
 /**

index 87695b9..685aac4 100644
@@ -27,11 +27,11 @@
  * Initialize a general malloc_elem header structure
  */
 void
-malloc_elem_init(struct malloc_elem *elem,
-		struct malloc_heap *heap, const struct rte_memseg *ms, size_t size)
+malloc_elem_init(struct malloc_elem *elem, struct malloc_heap *heap,
+		struct rte_memseg_list *msl, size_t size)
 {
 	elem->heap = heap;
-	elem->ms = ms;
+	elem->msl = msl;
 	elem->prev = NULL;
 	elem->next = NULL;
 	memset(&elem->free_list, 0, sizeof(elem->free_list));

@@ -100,7 +100,7 @@ malloc_elem_insert(struct malloc_elem *elem)
  * so we just check the page addresses.
  */
 static bool
-elem_check_phys_contig(const struct rte_memseg *ms __rte_unused,
+elem_check_phys_contig(const struct rte_memseg_list *msl __rte_unused,
 		void *start, size_t size)
 {
 	rte_iova_t cur, expected;

@@ -191,7 +191,7 @@ elem_start_pt(struct malloc_elem *elem, size_t size, unsigned align,
 			 * couldn't fit all data into one physically contiguous
 			 * block, try again with lower addresses.
 			 */
-			if (!elem_check_phys_contig(elem->ms,
+			if (!elem_check_phys_contig(elem->msl,
 					(void *)new_data_start,
 					new_data_size)) {
 				elem_size -= align;

@@ -225,7 +225,7 @@ split_elem(struct malloc_elem *elem, struct malloc_elem *split_pt)
 	const size_t old_elem_size = (uintptr_t)split_pt - (uintptr_t)elem;
 	const size_t new_elem_size = elem->size - old_elem_size;
 
-	malloc_elem_init(split_pt, elem->heap, elem->ms, new_elem_size);
+	malloc_elem_init(split_pt, elem->heap, elem->msl, new_elem_size);
 	split_pt->prev = elem;
 	split_pt->next = next_elem;
 	if (next_elem)

index 34bd268..620dd44 100644
@@ -7,7 +7,7 @@
 
 #include <stdbool.h>
 
-#include <rte_memory.h>
+#include <rte_eal_memconfig.h>
 
 /* dummy definition of struct so we can use pointers to it in malloc_elem struct */
 struct malloc_heap;

@@ -26,7 +26,7 @@ struct malloc_elem {
 	/**< points to next elem in memseg */
 	LIST_ENTRY(malloc_elem) free_list;
 	/**< list of free elements in heap */
-	const struct rte_memseg *ms;
+	struct rte_memseg_list *msl;
 	volatile enum elem_state state;
 	uint32_t pad;
 	size_t size;

@@ -113,7 +113,7 @@ malloc_elem_from_data(const void *data)
 void
 malloc_elem_init(struct malloc_elem *elem,
 		struct malloc_heap *heap,
-		const struct rte_memseg *ms,
+		struct rte_memseg_list *msl,
 		size_t size);
 
 void

94 lines not on this page · raw message

index 436818a..c6d3e57 100644
@@ -242,17 +242,21 @@ rte_malloc_set_limit(__rte_unused const char *type,
 rte_iova_t
 rte_malloc_virt2iova(const void *addr)
 {
-	rte_iova_t iova;
-	const struct malloc_elem *elem = malloc_elem_from_data(addr);
+	const struct rte_memseg *ms;
+	struct malloc_elem *elem = malloc_elem_from_data(addr);
+
 	if (elem == NULL)
 		return RTE_BAD_IOVA;
-	if (elem->ms->iova == RTE_BAD_IOVA)
-		return RTE_BAD_IOVA;
 
 	if (rte_eal_iova_mode() == RTE_IOVA_VA)
-		iova = (uintptr_t)addr;
-	else
-		iova = elem->ms->iova +
-			RTE_PTR_DIFF(addr, elem->ms->addr);
-	return iova;
+		return (uintptr_t) addr;
+
+	ms = rte_mem_virt2memseg(addr, elem->msl);
+	if (ms == NULL)
+		return RTE_BAD_IOVA;
+
+	if (ms->iova == RTE_BAD_IOVA)
+		return RTE_BAD_IOVA;
+
+	return ms->iova + RTE_PTR_DIFF(addr, ms->addr);
 }

index b34e57a..ffcbd71 100644
@@ -74,8 +74,8 @@ static int mem_cfg_fd = -1;
 static struct flock wr_lock = {
 		.l_type = F_WRLCK,
 		.l_whence = SEEK_SET,
-		.l_start = offsetof(struct rte_mem_config, memseg),
-		.l_len = sizeof(early_mem_config.memseg),
+		.l_start = offsetof(struct rte_mem_config, memsegs),
+		.l_len = sizeof(early_mem_config.memsegs),
 };
 
 /* Address of global and public configuration */

@@ -640,11 +640,14 @@ eal_parse_args(int argc, char **argv)
 }
 
 static int
-check_mem(const struct rte_memseg *ms, void *arg)
+check_socket(const struct rte_memseg_list *msl, void *arg)
 {
-	int *socket = arg;
+	int *socket_id = arg;
 
-	return ms->socket_id == *socket;
+	if (msl->socket_id == *socket_id && msl->memseg_arr.count != 0)
+		return 1;
+
+	return 0;
 }
 
 static void

@@ -654,7 +657,7 @@ eal_check_mem_on_local_socket(void)
 
 	socket_id = rte_lcore_to_socket_id(rte_config.master_lcore);
 
-	if (rte_memseg_walk(check_mem, &socket_id) == 0)
+	if (rte_memseg_list_walk(check_socket, &socket_id) == 0)
 		RTE_LOG(WARNING, EAL, "WARNING: Master core has no memory on local socket!\n");
 }
 

index 8bbf771..afebd42 100644
@@ -15,6 +15,7 @@
 #include <unistd.h>
 #include <errno.h>
 #include <sys/queue.h>
+#include <sys/stat.h>
 
 #include <rte_memory.h>
 #include <rte_eal.h>

@@ -160,6 +161,18 @@ get_hugepage_dir(uint64_t hugepage_sz)
 }
 
 /*
+ * uses fstat to report the size of a file on disk
+ */
+static off_t
+get_file_size(int fd)
+{
+	struct stat st;
+	if (fstat(fd, &st) < 0)
+		return 0;
+	return st.st_size;
+}
+
+/*
  * Clear the hugepage directory of whatever hugepage files
  * there are. Checks if the file is locked (i.e.
  * if it's in use by another DPDK process).

@@ -189,6 +202,8 @@ clear_hugedir(const char * hugedir)
 	}
 
 	while(dirent != NULL){
+		struct flock lck = {0};
+
 		/* skip files that don't match the hugepage pattern */
 		if (fnmatch(filter, dirent->d_name, 0) > 0) {
 			dirent = readdir(dir);

@@ -205,11 +220,17 @@ clear_hugedir(const char * hugedir)
 		}
 
 		/* non-blocking lock */
-		lck_result = flock(fd, LOCK_EX | LOCK_NB);
+		lck.l_type = F_RDLCK;
+		lck.l_whence = SEEK_SET;
+		lck.l_start = 0;
+		lck.l_len = get_file_size(fd);
+
+		lck_result = fcntl(fd, F_SETLK, &lck);
 
 		/* if lock succeeds, unlock and remove the file */
 		if (lck_result != -1) {
-			flock(fd, LOCK_UN);
+			lck.l_type = F_UNLCK;
+			fcntl(fd, F_SETLK, &lck);
 			unlinkat(dir_fd, dirent->d_name, 0);
 		}
 		close (fd);

1,111 lines not on this page · raw message

index c1f0f87..5101c04 100644
@@ -908,7 +908,8 @@ vfio_get_group_no(const char *sysfs_base,
 }
 
 static int
-type1_map(const struct rte_memseg *ms, void *arg)
+type1_map(const struct rte_memseg_list *msl __rte_unused,
+		const struct rte_memseg *ms, void *arg)
 {
 	int *vfio_container_fd = arg;
 

@@ -1021,7 +1022,8 @@ vfio_spapr_dma_do_map(int vfio_container_fd, uint64_t vaddr, uint64_t iova,
 }
 
 static int
-vfio_spapr_map_walk(const struct rte_memseg *ms, void *arg)
+vfio_spapr_map_walk(const struct rte_memseg_list *msl __rte_unused,
+		const struct rte_memseg *ms, void *arg)
 {
 	int *vfio_container_fd = arg;
 

@@ -1034,7 +1036,8 @@ struct spapr_walk_param {
 	uint64_t hugepage_sz;
 };
 static int
-vfio_spapr_window_size_walk(const struct rte_memseg *ms, void *arg)
+vfio_spapr_window_size_walk(const struct rte_memseg_list *msl __rte_unused,
+		const struct rte_memseg *ms, void *arg)
 {
 	struct spapr_walk_param *param = arg;
 	uint64_t max = ms->iova + ms->len;

index 9731d4c..103c015 100644
@@ -100,12 +100,12 @@ static unsigned optimize_object_size(unsigned obj_size)
 }
 
 static int
-find_min_pagesz(const struct rte_memseg *ms, void *arg)
+find_min_pagesz(const struct rte_memseg_list *msl, void *arg)
 {
 	size_t *min = arg;
 
-	if (ms->hugepage_sz < *min)
-		*min = ms->hugepage_sz;
+	if (msl->page_sz < *min)
+		*min = msl->page_sz;
 
 	return 0;
 }

@@ -115,11 +115,12 @@ get_min_page_size(void)
 {
 	size_t min_pagesz = SIZE_MAX;
 
-	rte_memseg_walk(find_min_pagesz, &min_pagesz);
+	rte_memseg_list_walk(find_min_pagesz, &min_pagesz);
 
 	return min_pagesz == SIZE_MAX ? (size_t) getpagesize() : min_pagesz;
 }
 
+
 static void
 mempool_add_elem(struct rte_mempool *mp, void *obj, rte_iova_t iova)
 {

index 28c241f..4b5abb4 100644
@@ -12,6 +12,7 @@
 
 #include <rte_common.h>
 #include <rte_memory.h>
+#include <rte_eal_memconfig.h>
 #include <rte_per_lcore.h>
 #include <rte_launch.h>
 #include <rte_eal.h>

@@ -706,36 +707,20 @@ test_malloc_bad_params(void)
 }
 
 static int
-check_socket_mem(const struct rte_memseg *ms, void *arg)
+check_socket_mem(const struct rte_memseg_list *msl, void *arg)
 {
 	int32_t *socket = arg;
 
-	return *socket == ms->socket_id;
+	return *socket == msl->socket_id;
 }
 
 /* Check if memory is available on a specific socket */
 static int
 is_mem_on_socket(int32_t socket)
 {
-	return rte_memseg_walk(check_socket_mem, &socket);
+	return rte_memseg_list_walk(check_socket_mem, &socket);
 }
 
-struct walk_param {
-	void *addr;
-	int32_t socket;
-};
-static int
-find_socket(const struct rte_memseg *ms, void *arg)
-{
-	struct walk_param *param = arg;
-
-	if (param->addr >= ms->addr &&
-			param->addr < RTE_PTR_ADD(ms->addr, ms->len)) {
[… more lines in the raw message]

Lines of this file are not shown. View the raw message.

index c9b287c..b96bca7 100644

Lines of this file are not shown. View the raw message.

index cbf0cfa..0046f04 100644

Lines of this file are not shown. View the raw message.

index 34b2a8d..f7bfc7a 100644
@@ -264,11 +264,15 @@ static const struct eth_dev_ops ena_dev_ops = {
 static inline int ena_cpu_to_node(int cpu)
 {
 	struct rte_config *config = rte_eal_get_configuration();
+	struct rte_fbarray *arr = &config->mem_config->memzones;
+	const struct rte_memzone *mz;
 
-	if (likely(cpu < RTE_MAX_MEMZONE))
-		return config->mem_config->memzone[cpu].socket_id;
+	if (unlikely(cpu >= RTE_MAX_MEMZONE))
+		return NUMA_NO_NODE;
 
-	return NUMA_NO_NODE;
+	mz = rte_fbarray_get(arr, cpu);
+
+	return mz->socket_id;
 }
 
 static inline void ena_rx_mbuf_prepare(struct rte_mbuf *mbuf,

index d009cf0..54330e1 100644
@@ -599,14 +599,24 @@ rte_eal_init(int argc, char **argv)
 		}
 	}
 
+	/* in secondary processes, memory init may allocate additional fbarrays
+	 * not present in primary processes, so to avoid any potential issues,
+	 * initialize memzones first.
+	 */
+	if (rte_eal_memzone_init() < 0) {
+		rte_eal_init_alert("Cannot init memzone\n");
+		rte_errno = ENODEV;
+		return -1;
+	}
+
 	if (rte_eal_memory_init() < 0) {
 		rte_eal_init_alert("Cannot init memory\n");
 		rte_errno = ENOMEM;
 		return -1;
 	}
 
-	if (rte_eal_memzone_init() < 0) {
-		rte_eal_init_alert("Cannot init memzone\n");
+	if (rte_eal_malloc_heap_init() < 0) {
+		rte_eal_init_alert("Cannot init malloc heap\n");
 		rte_errno = ENODEV;
 		return -1;
 	}

208 lines not on this page · raw message

index b745e18..88cde8c 100644
@@ -58,10 +58,8 @@ struct rte_mem_config {
 	rte_rwlock_t qlock;   /**< used for tailq operation for thread safe. */
 	rte_rwlock_t mplock;  /**< only used by mempool LIB for thread-safe. */
 
-	uint32_t memzone_cnt; /**< Number of allocated memzones */
-
 	/* memory segments and zones */
-	struct rte_memzone memzone[RTE_MAX_MEMZONE]; /**< Memzone descriptors. */
+	struct rte_fbarray memzones; /**< Memzone descriptors. */
 
 	struct rte_memseg_list memsegs[RTE_MAX_MEMSEG_LISTS];
 	/**< list of dynamic arrays holding memsegs */

index 0ef2c45..d798675 100644
@@ -278,6 +278,10 @@ rte_eal_malloc_heap_init(void)
 	if (mcfg == NULL)
 		return -1;
 
+	/* secondary process does not need to initialize anything */
+	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
+		return 0;
+
 	/* add all IOVA-contiguous areas to the heap */
 	return rte_memseg_contig_walk(malloc_add_seg, NULL);
 }

index ffcbd71..9832551 100644
@@ -858,6 +858,15 @@ rte_eal_init(int argc, char **argv)
 		return -1;
 	}
 #endif
+	/* in secondary processes, memory init may allocate additional fbarrays
+	 * not present in primary processes, so to avoid any potential issues,
+	 * initialize memzones first.
+	 */
+	if (rte_eal_memzone_init() < 0) {
+		rte_eal_init_alert("Cannot init memzone\n");
+		rte_errno = ENODEV;
+		return -1;
+	}
 
 	if (rte_eal_memory_init() < 0) {
 		rte_eal_init_alert("Cannot init memory\n");

@@ -868,8 +877,8 @@ rte_eal_init(int argc, char **argv)
 	/* the directories are locked during eal_hugepage_info_init */
 	eal_hugedirs_unlock();
 
-	if (rte_eal_memzone_init() < 0) {
-		rte_eal_init_alert("Cannot init memzone\n");
+	if (rte_eal_malloc_heap_init() < 0) {
+		rte_eal_init_alert("Cannot init malloc heap\n");
 		rte_errno = ENODEV;
 		return -1;
 	}

index 0046f04..efcf732 100644
@@ -909,7 +909,7 @@ test_memzone_basic(void)
 	const struct rte_memzone *mz;
 	int memzone_cnt_after, memzone_cnt_expected;
 	int memzone_cnt_before =
-			rte_eal_get_configuration()->mem_config->memzone_cnt;
+			rte_eal_get_configuration()->mem_config->memzones.count;
 
 	memzone1 = rte_memzone_reserve(TEST_MEMZONE_NAME("testzone1"), 100,
 				SOCKET_ID_ANY, 0);

@@ -933,7 +933,7 @@ test_memzone_basic(void)
 			(memzone3 != NULL) + (memzone4 != NULL);
 
 	memzone_cnt_after =
-			rte_eal_get_configuration()->mem_config->memzone_cnt;
+			rte_eal_get_configuration()->mem_config->memzones.count;
 
 	if (memzone_cnt_after != memzone_cnt_expected)
 		return -1;

@@ -1012,7 +1012,7 @@ test_memzone_basic(void)
 	}
 
 	memzone_cnt_after =
-			rte_eal_get_configuration()->mem_config->memzone_cnt;
+			rte_eal_get_configuration()->mem_config->memzones.count;
 	if (memzone_cnt_after != memzone_cnt_before)
 		return -1;
 

@@ -1033,7 +1033,8 @@ static int
 test_memzone(void)
 {
 	/* take note of how many memzones were allocated before running */
-	int memzone_cnt = rte_eal_get_configuration()->mem_config->memzone_cnt;
+	int memzone_cnt =
+			rte_eal_get_configuration()->mem_config->memzones.count;
 
 	printf("test basic memzone API\n");
 	if (test_memzone_basic() < 0)

new file mode 100644
index 0000000..8c30670
@@ -0,0 +1,26 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2017-2018 Intel Corporation
+ */
+
+#include <inttypes.h>
+
+#include <rte_log.h>
+#include <rte_memory.h>
+
+#include "eal_memalloc.h"
+
+int
+eal_memalloc_alloc_seg_bulk(struct rte_memseg **ms __rte_unused,
+		int __rte_unused n_segs, size_t __rte_unused page_sz,
+		int __rte_unused socket, bool __rte_unused exact)
+{
+	RTE_LOG(ERR, EAL, "Memory hotplug not supported on FreeBSD\n");
+	return -1;
+}
+
+struct rte_memseg *
+eal_memalloc_alloc_seg(size_t __rte_unused page_sz, int __rte_unused socket)
+{
+	RTE_LOG(ERR, EAL, "Memory hotplug not supported on FreeBSD\n");
+	return NULL;
+}

new file mode 100644
index 0000000..f628514
@@ -0,0 +1,31 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2017-2018 Intel Corporation
+ */
+
+#ifndef EAL_MEMALLOC_H
+#define EAL_MEMALLOC_H
+
+#include <stdbool.h>
+
+#include <rte_memory.h>
+
+/*
+ * Allocate segment of specified page size.
+ */
+struct rte_memseg *
+eal_memalloc_alloc_seg(size_t page_sz, int socket);
+
+/*
+ * Allocate `n_segs` segments.
+ *
+ * Note: `ms` can be NULL.
+ *
+ * Note: it is possible to request best-effort allocation by setting `exact` to
+ * `false`, in which case allocator will return however many pages it managed to
+ * allocate successfully.
+ */
+int
+eal_memalloc_alloc_seg_bulk(struct rte_memseg **ms, int n_segs, size_t page_sz,
+		int socket, bool exact);
+
+#endif // EAL_MEMALLOC_H

429 lines not on this page · raw message

index 8c30670..e7bcd2b 100644
@@ -24,3 +24,18 @@ eal_memalloc_alloc_seg(size_t __rte_unused page_sz, int __rte_unused socket)
 	RTE_LOG(ERR, EAL, "Memory hotplug not supported on FreeBSD\n");
 	return NULL;
 }
+
+int
+eal_memalloc_free_seg(struct rte_memseg *ms __rte_unused)
+{
+	RTE_LOG(ERR, EAL, "Memory hotplug not supported on FreeBSD\n");
+	return -1;
+}
+
+int
+eal_memalloc_free_seg_bulk(struct rte_memseg **ms __rte_unused,
+		int n_segs __rte_unused)
+{
+	RTE_LOG(ERR, EAL, "Memory hotplug not supported on FreeBSD\n");
+	return -1;
+}