Memory Hotplug for DPDK

index 852f3bb..5b8ced4 100644
@@ -2,10 +2,12 @@
  * Copyright(c) 2010-2014 Intel Corporation
  */
 
+#include <errno.h>
 #include <stdio.h>
 #include <stdint.h>
 #include <stdlib.h>
 #include <stdarg.h>
+#include <string.h>
 #include <unistd.h>
 #include <inttypes.h>
 #include <sys/mman.h>

@@ -14,12 +16,111 @@
 #include <rte_memory.h>
 #include <rte_eal.h>
 #include <rte_eal_memconfig.h>
+#include <rte_errno.h>
 #include <rte_log.h>
 
 #include "eal_private.h"
 #include "eal_internal_cfg.h"
 
 /*
+ * Try to mmap *size bytes in /dev/zero. If it is successful, return the
+ * pointer to the mmap'd area and keep *size unmodified. Else, retry
+ * with a smaller zone: decrease *size by hugepage_sz until it reaches
+ * 0. In this case, return NULL. Note: this function returns an address
+ * which is a multiple of hugepage size.
+ */
+
+static uint64_t baseaddr_offset;
+static uint64_t system_page_sz;
+
+void *
+eal_get_virtual_area(void *requested_addr, size_t *size,
+		size_t page_sz, int flags, int mmap_flags)
+{
+	bool addr_is_hint, allow_shrink, unmap, no_align;
+	uint64_t map_sz;
+	void *mapped_addr, *aligned_addr;
+
+	if (system_page_sz == 0)
+		system_page_sz = sysconf(_SC_PAGESIZE);
+
+	mmap_flags |= MAP_PRIVATE | MAP_ANONYMOUS;
+
+	RTE_LOG(DEBUG, EAL, "Ask a virtual area of 0x%zx bytes\n", *size);
+
+	addr_is_hint = (flags & EAL_VIRTUAL_AREA_ADDR_IS_HINT) > 0;
+	allow_shrink = (flags & EAL_VIRTUAL_AREA_ALLOW_SHRINK) > 0;
+	unmap = (flags & EAL_VIRTUAL_AREA_UNMAP) > 0;
+
+	if (requested_addr == NULL && internal_config.base_virtaddr != 0) {
+		requested_addr = (void *) (internal_config.base_virtaddr +
+				(size_t)baseaddr_offset);
+		requested_addr = RTE_PTR_ALIGN(requested_addr, page_sz);
+		addr_is_hint = true;
+	}
+
+	/* if requested address is not aligned by page size, or if requested
+	 * address is NULL, add page size to requested length as we may get an
+	 * address that's aligned by system page size, which can be smaller than
+	 * our requested page size. additionally, we shouldn't try to align if
+	 * system page size is the same as requested page size.
+	 */
+	no_align = (requested_addr != NULL &&
+		((uintptr_t)requested_addr & (page_sz - 1)) == 0) ||
+		page_sz == system_page_sz;
+
+	do {
+		map_sz = no_align ? *size : *size + page_sz;
+
+		mapped_addr = mmap(requested_addr, map_sz, PROT_READ,
+				mmap_flags, -1, 0);
+		if (mapped_addr == MAP_FAILED && allow_shrink)
+			*size -= page_sz;
+	} while (allow_shrink && mapped_addr == MAP_FAILED && *size > 0);
+
+	/* align resulting address - if map failed, we will ignore the value
+	 * anyway, so no need to add additional checks.
+	 */
+	aligned_addr = no_align ? mapped_addr :
+			RTE_PTR_ALIGN(mapped_addr, page_sz);
+
+	if (*size == 0) {
+		RTE_LOG(ERR, EAL, "Cannot get a virtual area of any size: %s\n",
+			strerror(errno));
+		rte_errno = errno;
+		return NULL;
+	} else if (mapped_addr == MAP_FAILED) {
+		RTE_LOG(ERR, EAL, "Cannot get a virtual area: %s\n",
+			strerror(errno));
+		/* pass errno up the call chain */
+		rte_errno = errno;
+		return NULL;
+	} else if (requested_addr != NULL && !addr_is_hint &&
+			aligned_addr != requested_addr) {
+		RTE_LOG(ERR, EAL, "Cannot get a virtual area at requested address: %p (got %p)\n",
+			requested_addr, aligned_addr);
+		munmap(mapped_addr, map_sz);
+		rte_errno = EADDRNOTAVAIL;
+		return NULL;
+	} else if (requested_addr != NULL && addr_is_hint &&
+			aligned_addr != requested_addr) {
+		RTE_LOG(WARNING, EAL, "WARNING! Base virtual address hint (%p != %p) not respected!\n",
+			requested_addr, aligned_addr);
+		RTE_LOG(WARNING, EAL, "   This may cause issues with mapping memory into secondary processes\n");
+	}
+
+	if (unmap)
+		munmap(mapped_addr, map_sz);
+
+	RTE_LOG(DEBUG, EAL, "Virtual area found at %p (size = 0x%zx)\n",
+		aligned_addr, *size);
+
+	baseaddr_offset += *size;
+
+	return aligned_addr;
+}
+
+/*
  * Return a pointer to a read-only table of struct rte_physmem_desc
  * elements, containing the layout of all addressable physical
  * memory. The last element of the table contains a NULL address.

index 0b28770..3fed436 100644
@@ -127,6 +127,39 @@ int rte_eal_alarm_init(void);
 int rte_eal_check_module(const char *module_name);
 
 /**
+ * Get virtual area of specified size from the OS.
+ *
+ * This function is private to the EAL.
+ *
+ * @param requested_addr
+ *   Address where to request address space.
+ * @param size
+ *   Size of requested area.
+ * @param page_sz
+ *   Page size on which to align requested virtual area.
+ * @param flags
+ *   EAL_VIRTUAL_AREA_* flags.
+ * @param mmap_flags
+ *   Extra flags passed directly to mmap().
+ *
+ * @return
+ *   Virtual area address if successful.
+ *   NULL if unsuccessful.
+ */
+
+#define EAL_VIRTUAL_AREA_ADDR_IS_HINT (1 << 0)
+/**< don't fail if cannot get exact requested address. */
+#define EAL_VIRTUAL_AREA_ALLOW_SHRINK (1 << 1)
+/**< try getting smaller sized (decrement by page size) virtual areas if cannot
+ * get area of requested size.
+ */
+#define EAL_VIRTUAL_AREA_UNMAP (1 << 2)
+/**< immediately unmap reserved virtual area. */
+void *
+eal_get_virtual_area(void *requested_addr, size_t *size,
+		size_t page_sz, int flags, int mmap_flags);
+
+/**
  * Get cpu core_id.
  *
  * This function is private to the EAL.

204 lines not on this page · raw message

index 0cadc8a..ea041e2 100644
@@ -243,10 +243,6 @@ join_elem(struct malloc_elem *elem1, struct malloc_elem *elem2)
 int
 malloc_elem_free(struct malloc_elem *elem)
 {
-	if (!malloc_elem_cookies_ok(elem) || elem->state != ELEM_BUSY)
-		return -1;
-
-	rte_spinlock_lock(&(elem->heap->lock));
 	size_t sz = elem->size - sizeof(*elem) - MALLOC_ELEM_TRAILER_LEN;
 	uint8_t *ptr = (uint8_t *)&elem[1];
 	struct malloc_elem *next = RTE_PTR_ADD(elem, elem->size);

@@ -274,8 +270,6 @@ malloc_elem_free(struct malloc_elem *elem)
 
 	memset(ptr, 0, sz);
 
-	rte_spinlock_unlock(&(elem->heap->lock));
-
 	return 0;
 }
 

@@ -292,11 +286,10 @@ malloc_elem_resize(struct malloc_elem *elem, size_t size)
 		return 0;
 
 	struct malloc_elem *next = RTE_PTR_ADD(elem, elem->size);
-	rte_spinlock_lock(&elem->heap->lock);
 	if (next ->state != ELEM_FREE)
-		goto err_return;
+		return -1;
 	if (elem->size + next->size < new_size)
-		goto err_return;
+		return -1;
 
 	/* we now know the element fits, so remove from free list,
 	 * join the two

@@ -311,10 +304,5 @@ malloc_elem_resize(struct malloc_elem *elem, size_t size)
 		split_elem(elem, split_pt);
 		malloc_elem_free_list_insert(split_pt);
 	}
-	rte_spinlock_unlock(&elem->heap->lock);
 	return 0;
-
-err_return:
-	rte_spinlock_unlock(&elem->heap->lock);
-	return -1;
 }

index 7aafc88..7d8d70a 100644
@@ -145,6 +145,44 @@ malloc_heap_alloc(struct malloc_heap *heap,
 	return elem == NULL ? NULL : (void *)(&elem[1]);
 }
 
+int
+malloc_heap_free(struct malloc_elem *elem)
+{
+	struct malloc_heap *heap;
+	int ret;
+
+	if (!malloc_elem_cookies_ok(elem) || elem->state != ELEM_BUSY)
+		return -1;
+
+	/* elem may be merged with previous element, so keep heap address */
+	heap = elem->heap;
+
+	rte_spinlock_lock(&(heap->lock));
+
+	ret = malloc_elem_free(elem);
+
+	rte_spinlock_unlock(&(heap->lock));
+
+	return ret;
+}
+
+int
+malloc_heap_resize(struct malloc_elem *elem, size_t size)
+{
+	int ret;
+
+	if (!malloc_elem_cookies_ok(elem) || elem->state != ELEM_BUSY)
+		return -1;
+
+	rte_spinlock_lock(&(elem->heap->lock));
+
+	ret = malloc_elem_resize(elem, size);
+
+	rte_spinlock_unlock(&(elem->heap->lock));
+
+	return ret;
+}
+
 /*
  * Function to retrieve data for heap on given socket
  */

index e0defa7..ab0005c 100644
@@ -28,6 +28,12 @@ malloc_heap_alloc(struct malloc_heap *heap,	const char *type, size_t size,
 		unsigned flags, size_t align, size_t bound);
 
 int
+malloc_heap_free(struct malloc_elem *elem);
+
+int
+malloc_heap_resize(struct malloc_elem *elem, size_t size);
+
+int
 malloc_heap_get_stats(struct malloc_heap *heap,
 		struct rte_malloc_socket_stats *socket_stats);
 

index e0e0d0b..970813e 100644
@@ -29,7 +29,7 @@
 void rte_free(void *addr)
 {
 	if (addr == NULL) return;
-	if (malloc_elem_free(malloc_elem_from_data(addr)) < 0)
+	if (malloc_heap_free(malloc_elem_from_data(addr)) < 0)
 		rte_panic("Fatal error: Invalid memory\n");
 }
 

@@ -140,7 +140,7 @@ rte_realloc(void *ptr, size_t size, unsigned align)
 	size = RTE_CACHE_LINE_ROUNDUP(size), align = RTE_CACHE_LINE_ROUNDUP(align);
 	/* check alignment matches first, and if ok, see if we can resize block */
 	if (RTE_PTR_ALIGN(ptr,align) == ptr &&
-			malloc_elem_resize(elem, size) == 0)
+			malloc_heap_resize(elem, size) == 0)
 		return ptr;
 
 	/* either alignment is off, or we have no room to expand,

index ba99ed9..d43fa90 100644
@@ -13,12 +13,18 @@
 /* Number of free lists per heap, grouped by size. */
 #define RTE_HEAP_NUM_FREELISTS  13
 
+/* dummy definition, for pointers */
+struct malloc_elem;
+
 /**
  * Structure to hold malloc heap
  */
 struct malloc_heap {
 	rte_spinlock_t lock;
 	LIST_HEAD(, malloc_elem) free_head[RTE_HEAP_NUM_FREELISTS];
+	struct malloc_elem *volatile first;
+	struct malloc_elem *volatile last;
+
 	unsigned alloc_count;
 	size_t total_size;
 } __rte_cache_aligned;

271 lines not on this page · raw message

index f4c1c7a..238e451 100644
@@ -18,8 +18,12 @@ enum elem_state {
 
 struct malloc_elem {
 	struct malloc_heap *heap;
-	struct malloc_elem *volatile prev;      /* points to prev elem in memseg */
-	LIST_ENTRY(malloc_elem) free_list;      /* list of free elements in heap */
+	struct malloc_elem *volatile prev;
+	/**< points to prev elem in memseg */
+	struct malloc_elem *volatile next;
+	/**< points to next elem in memseg */
+	LIST_ENTRY(malloc_elem) free_list;
+	/**< list of free elements in heap */
 	const struct rte_memseg *ms;
 	volatile enum elem_state state;
 	uint32_t pad;

@@ -110,12 +114,8 @@ malloc_elem_init(struct malloc_elem *elem,
 		const struct rte_memseg *ms,
 		size_t size);
 
-/*
- * initialise a dummy malloc_elem header for the end-of-memseg marker
- */
 void
-malloc_elem_mkend(struct malloc_elem *elem,
-		struct malloc_elem *prev_free);
+malloc_elem_insert(struct malloc_elem *elem);
 
 /*
  * return true if the current malloc_elem can hold a block of data

index 7d8d70a..9c95166 100644
@@ -70,15 +70,11 @@ check_hugepage_sz(unsigned flags, uint64_t hugepage_sz)
 static void
 malloc_heap_add_memseg(struct malloc_heap *heap, struct rte_memseg *ms)
 {
-	/* allocate the memory block headers, one at end, one at start */
 	struct malloc_elem *start_elem = (struct malloc_elem *)ms->addr;
-	struct malloc_elem *end_elem = RTE_PTR_ADD(ms->addr,
-			ms->len - MALLOC_ELEM_OVERHEAD);
-	end_elem = RTE_PTR_ALIGN_FLOOR(end_elem, RTE_CACHE_LINE_SIZE);
-	const size_t elem_size = (uintptr_t)end_elem - (uintptr_t)start_elem;
+	const size_t elem_size = ms->len - MALLOC_ELEM_OVERHEAD;
 
 	malloc_elem_init(start_elem, heap, ms, elem_size);
-	malloc_elem_mkend(end_elem, start_elem);
+	malloc_elem_insert(start_elem);
 	malloc_elem_free_list_insert(start_elem);
 
 	heap->total_size += elem_size;

index f02a8ba..a9fb7e4 100644
@@ -13,6 +13,7 @@
 
 #include <stdio.h>
 #include <stddef.h>
+#include <rte_compat.h>
 #include <rte_memory.h>
 
 #ifdef __cplusplus

@@ -278,6 +279,15 @@ void
 rte_malloc_dump_stats(FILE *f, const char *type);
 
 /**
+ * Dump contents of all malloc heaps to a file.
+ *
+ * @param f
+ *   A pointer to a file for output
+ */
+void __rte_experimental
+rte_malloc_dump_heaps(FILE *f);
+
+/**
  * Set the maximum amount of allocated memory for this type.
  *
  * This is not yet implemented

index eb41200..e02ed88 100644
@@ -1,6 +1,7 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  * Copyright(c) 2010-2014 Intel Corporation
  */
+#include <inttypes.h>
 #include <stdint.h>
 #include <stddef.h>
 #include <stdio.h>

@@ -434,3 +435,26 @@ malloc_elem_resize(struct malloc_elem *elem, size_t size)
 	}
 	return 0;
 }
+
+static inline const char *
+elem_state_to_str(enum elem_state state)
+{
+	switch (state) {
+	case ELEM_PAD:
+		return "PAD";
+	case ELEM_BUSY:
+		return "BUSY";
+	case ELEM_FREE:
+		return "FREE";
+	}
+	return "ERROR";
+}
+
+void
+malloc_elem_dump(const struct malloc_elem *elem, FILE *f)
+{
+	fprintf(f, "Malloc element at %p (%s)\n", elem,
+			elem_state_to_str(elem->state));
+	fprintf(f, "  len: 0x%zx pad: 0x%" PRIx32 "\n", elem->size, elem->pad);
+	fprintf(f, "  prev: %p next: %p\n", elem->prev, elem->next);
+}

index 238e451..40e8eb5 100644
@@ -149,6 +149,12 @@ int
 malloc_elem_resize(struct malloc_elem *elem, size_t size);
 
 /*
+ * dump contents of malloc elem to a file.
+ */
+void
+malloc_elem_dump(const struct malloc_elem *elem, FILE *f);
+
+/*
  * Given an element size, compute its freelist index.
  */
 size_t

index 9c95166..44538d7 100644
@@ -217,6 +217,28 @@ malloc_heap_get_stats(struct malloc_heap *heap,
 	return 0;
 }
 
+/*
+ * Function to retrieve data for heap on given socket
+ */
+void
+malloc_heap_dump(struct malloc_heap *heap, FILE *f)
+{
+	struct malloc_elem *elem;
+
+	rte_spinlock_lock(&heap->lock);
+
+	fprintf(f, "Heap size: 0x%zx\n", heap->total_size);
+	fprintf(f, "Heap alloc count: %u\n", heap->alloc_count);
+
+	elem = heap->first;
+	while (elem) {
+		malloc_elem_dump(elem, f);
+		elem = elem->next;
+	}
+
+	rte_spinlock_unlock(&heap->lock);
+}
+
 int
 rte_eal_malloc_heap_init(void)
 {

index ab0005c..bb28422 100644
@@ -37,6 +37,9 @@ int
 malloc_heap_get_stats(struct malloc_heap *heap,
 		struct rte_malloc_socket_stats *socket_stats);
 
+void
+malloc_heap_dump(struct malloc_heap *heap, FILE *f);
+
 int
 rte_eal_malloc_heap_init(void);
 

index 970813e..f11a822 100644
@@ -182,6 +182,23 @@ rte_malloc_get_socket_stats(int socket,
 }
 
 /*
+ * Function to dump contents of all heaps
+ */
+void __rte_experimental
+rte_malloc_dump_heaps(FILE *f)
+{
+	struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
+	unsigned int idx;
+
+	for (idx = 0; idx < rte_socket_count(); idx++) {
+		unsigned int socket = rte_socket_id_by_idx(idx);
+		fprintf(f, "Heap on socket %i:\n", socket);
+		malloc_heap_dump(&mcfg->malloc_heaps[socket], f);
+	}
+
+}
+
+/*
  * Print stats on memory type. If type is NULL, info on all types is printed
  */
 void

index cf0b726..6bfdc02 100644
@@ -137,6 +137,8 @@ static void cmd_dump_parsed(void *parsed_result,
 		rte_log_dump(stdout);
 	else if (!strcmp(res->dump, "dump_malloc_stats"))
 		rte_malloc_dump_stats(stdout, NULL);
+	else if (!strcmp(res->dump, "dump_malloc_heaps"))
+		rte_malloc_dump_heaps(stdout);
 }
 
 cmdline_parse_token_string_t cmd_dump_dump =

@@ -147,6 +149,7 @@ cmdline_parse_token_string_t cmd_dump_dump =
 				 "dump_ring#"
 				 "dump_mempool#"
 				 "dump_malloc_stats#"
+				 "dump_malloc_heaps#"
 				 "dump_devargs#"
 				 "dump_log_types");
 

index e02ed88..2291ee1 100644
@@ -325,8 +325,8 @@ join_elem(struct malloc_elem *elem1, struct malloc_elem *elem2)
 	elem1->next = next;
 }
 
-static struct malloc_elem *
-elem_join_adjacent_free(struct malloc_elem *elem)
+struct malloc_elem *
+malloc_elem_join_adjacent_free(struct malloc_elem *elem)
 {
 	/*
 	 * check if next element exists, is adjacent and is free, if so join

@@ -388,7 +388,7 @@ malloc_elem_free(struct malloc_elem *elem)
 	ptr = RTE_PTR_ADD(elem, sizeof(*elem));
 	data_len = elem->size - MALLOC_ELEM_OVERHEAD;
 
-	elem = elem_join_adjacent_free(elem);
+	elem = malloc_elem_join_adjacent_free(elem);
 
 	malloc_elem_free_list_insert(elem);
 

index 40e8eb5..99921d2 100644
@@ -141,6 +141,9 @@ malloc_elem_alloc(struct malloc_elem *elem, size_t size,
 int
 malloc_elem_free(struct malloc_elem *elem);
 
+struct malloc_elem *
+malloc_elem_join_adjacent_free(struct malloc_elem *elem);
+
 /*
  * attempt to resize a malloc_elem by expanding into any free space
  * immediately after it in memory.

index 2291ee1..008f5a3 100644
@@ -245,8 +245,8 @@ malloc_elem_free_list_insert(struct malloc_elem *elem)
 /*
  * Remove the specified element from its heap's free list.
  */
-static void
-elem_free_list_remove(struct malloc_elem *elem)
+void
+malloc_elem_free_list_remove(struct malloc_elem *elem)
 {
 	LIST_REMOVE(elem, free_list);
 }

@@ -266,7 +266,7 @@ malloc_elem_alloc(struct malloc_elem *elem, size_t size, unsigned align,
 	const size_t trailer_size = elem->size - old_elem_size - size -
 		MALLOC_ELEM_OVERHEAD;
 
-	elem_free_list_remove(elem);
+	malloc_elem_free_list_remove(elem);
 
 	if (trailer_size > MALLOC_ELEM_OVERHEAD + MIN_DATA_SIZE) {
 		/* split it, too much free space after elem */

@@ -340,7 +340,7 @@ malloc_elem_join_adjacent_free(struct malloc_elem *elem)
 		erase = RTE_PTR_SUB(elem->next, MALLOC_ELEM_TRAILER_LEN);
 
 		/* remove from free list, join to this one */
-		elem_free_list_remove(elem->next);
+		malloc_elem_free_list_remove(elem->next);
 		join_elem(elem, elem->next);
 
 		/* erase header and trailer */

@@ -360,7 +360,7 @@ malloc_elem_join_adjacent_free(struct malloc_elem *elem)
 		erase = RTE_PTR_SUB(elem, MALLOC_ELEM_TRAILER_LEN);
 
 		/* remove from free list, join to this one */
-		elem_free_list_remove(elem->prev);
+		malloc_elem_free_list_remove(elem->prev);
 
 		new_elem = elem->prev;
 		join_elem(new_elem, elem);

@@ -423,7 +423,7 @@ malloc_elem_resize(struct malloc_elem *elem, size_t size)
 	/* we now know the element fits, so remove from free list,
 	 * join the two
 	 */
-	elem_free_list_remove(elem->next);
+	malloc_elem_free_list_remove(elem->next);
 	join_elem(elem, elem->next);
 
 	if (elem->size - new_size >= MIN_DATA_SIZE + MALLOC_ELEM_OVERHEAD) {

index 99921d2..46e2383 100644
@@ -151,6 +151,9 @@ malloc_elem_join_adjacent_free(struct malloc_elem *elem);
 int
 malloc_elem_resize(struct malloc_elem *elem, size_t size);
 
+void
+malloc_elem_free_list_remove(struct malloc_elem *elem);
+
 /*
  * dump contents of malloc elem to a file.
  */

index 008f5a3..c18f050 100644
@@ -379,7 +379,7 @@ malloc_elem_join_adjacent_free(struct malloc_elem *elem)
  * blocks either immediately before or immediately after newly freed block
  * are also free, the blocks are merged together.
  */
-int
+struct malloc_elem *
 malloc_elem_free(struct malloc_elem *elem)
 {
 	void *ptr;

@@ -397,7 +397,7 @@ malloc_elem_free(struct malloc_elem *elem)
 
 	memset(ptr, 0, data_len);
 
-	return 0;
+	return elem;
 }
 
 /*

index 46e2383..9c1614c 100644
@@ -138,7 +138,7 @@ malloc_elem_alloc(struct malloc_elem *elem, size_t size,
  * blocks either immediately before or immediately after newly freed block
  * are also free, the blocks are merged together.
  */
-int
+struct malloc_elem *
 malloc_elem_free(struct malloc_elem *elem);
 
 struct malloc_elem *

index 44538d7..a2c2e4c 100644
@@ -145,7 +145,7 @@ int
 malloc_heap_free(struct malloc_elem *elem)
 {
 	struct malloc_heap *heap;
-	int ret;
+	struct malloc_elem *ret;
 
 	if (!malloc_elem_cookies_ok(elem) || elem->state != ELEM_BUSY)
 		return -1;

@@ -159,7 +159,7 @@ malloc_heap_free(struct malloc_elem *elem)
 
 	rte_spinlock_unlock(&(heap->lock));
 
-	return ret;
+	return ret != NULL ? 0 : -1;
 }
 
 int

index f11a822..2cda48e 100644
@@ -30,7 +30,7 @@ void rte_free(void *addr)
 {
 	if (addr == NULL) return;
 	if (malloc_heap_free(malloc_elem_from_data(addr)) < 0)
-		rte_panic("Fatal error: Invalid memory\n");
+		RTE_LOG(ERR, EAL, "Error: Invalid memory\n");
 }
 
 /*

@@ -134,8 +134,10 @@ rte_realloc(void *ptr, size_t size, unsigned align)
 		return rte_malloc(NULL, size, align);
 
 	struct malloc_elem *elem = malloc_elem_from_data(ptr);
-	if (elem == NULL)
-		rte_panic("Fatal error: memory corruption detected\n");
+	if (elem == NULL) {
+		RTE_LOG(ERR, EAL, "Error: memory corruption detected\n");
+		return NULL;
+	}
 
 	size = RTE_CACHE_LINE_ROUNDUP(size), align = RTE_CACHE_LINE_ROUNDUP(align);
 	/* check alignment matches first, and if ok, see if we can resize block */

index 1ab3ade..16a2e7a 100644
@@ -98,7 +98,8 @@ find_heap_max_free_elem(int *s, unsigned align)
 
 static const struct rte_memzone *
 memzone_reserve_aligned_thread_unsafe(const char *name, size_t len,
-		int socket_id, unsigned flags, unsigned align, unsigned bound)
+		int socket_id, unsigned int flags, unsigned int align,
+		unsigned int bound, bool contig)
 {
 	struct rte_memzone *mz;
 	struct rte_mem_config *mcfg;

@@ -188,7 +189,7 @@ memzone_reserve_aligned_thread_unsafe(const char *name, size_t len,
 
 	/* allocate memory on heap */
 	void *mz_addr = malloc_heap_alloc(&mcfg->malloc_heaps[socket], NULL,
-			requested_len, flags, align, bound);
+			requested_len, flags, align, bound, contig);
 
 	if ((mz_addr == NULL) && (socket_id == SOCKET_ID_ANY)) {
 		/* try other heaps */

@@ -197,7 +198,8 @@ memzone_reserve_aligned_thread_unsafe(const char *name, size_t len,
 				continue;
 
 			mz_addr = malloc_heap_alloc(&mcfg->malloc_heaps[i],
-					NULL, requested_len, flags, align, bound);
+					NULL, requested_len, flags, align,
+					bound, contig);
 			if (mz_addr != NULL)
 				break;
 		}

@@ -235,9 +237,9 @@ memzone_reserve_aligned_thread_unsafe(const char *name, size_t len,
 }
 
 static const struct rte_memzone *
-rte_memzone_reserve_thread_safe(const char *name, size_t len,
-				int socket_id, unsigned flags, unsigned align,
-				unsigned bound)
+rte_memzone_reserve_thread_safe(const char *name, size_t len, int socket_id,
+		unsigned int flags, unsigned int align, unsigned int bound,
+		bool contig)
 {
 	struct rte_mem_config *mcfg;
 	const struct rte_memzone *mz = NULL;

@@ -248,7 +250,7 @@ rte_memzone_reserve_thread_safe(const char *name, size_t len,
 	rte_rwlock_write_lock(&mcfg->mlock);
 
 	mz = memzone_reserve_aligned_thread_unsafe(
-		name, len, socket_id, flags, align, bound);
+		name, len, socket_id, flags, align, bound, contig);
 
 	rte_rwlock_write_unlock(&mcfg->mlock);
 

@@ -265,7 +267,7 @@ rte_memzone_reserve_bounded(const char *name, size_t len, int socket_id,
 			    unsigned flags, unsigned align, unsigned bound)
 {
 	return rte_memzone_reserve_thread_safe(name, len, socket_id, flags,
-					       align, bound);
+					       align, bound, false);
 }
 
 /*

@@ -277,7 +279,7 @@ rte_memzone_reserve_aligned(const char *name, size_t len, int socket_id,
 			    unsigned flags, unsigned align)
 {
 	return rte_memzone_reserve_thread_safe(name, len, socket_id, flags,
-					       align, 0);
+					       align, 0, false);
 }
 
 /*

@@ -289,7 +291,8 @@ rte_memzone_reserve(const char *name, size_t len, int socket_id,
 		    unsigned flags)
 {
 	return rte_memzone_reserve_thread_safe(name, len, socket_id,
-					       flags, RTE_CACHE_LINE_SIZE, 0);
+					       flags, RTE_CACHE_LINE_SIZE, 0,
+					       false);
 }
 
 int

160 lines not on this page · raw message

index 9c1614c..34bd268 100644
@@ -5,6 +5,8 @@
 #ifndef MALLOC_ELEM_H_
 #define MALLOC_ELEM_H_
 
+#include <stdbool.h>
+
 #include <rte_memory.h>
 
 /* dummy definition of struct so we can use pointers to it in malloc_elem struct */

@@ -123,7 +125,7 @@ malloc_elem_insert(struct malloc_elem *elem);
  */
 int
 malloc_elem_can_hold(struct malloc_elem *elem, size_t size,
-		unsigned align, size_t bound);
+		unsigned int align, size_t bound, bool contig);
 
 /*
  * reserve a block of data in an existing malloc_elem. If the malloc_elem

@@ -131,7 +133,7 @@ malloc_elem_can_hold(struct malloc_elem *elem, size_t size,
  */
 struct malloc_elem *
 malloc_elem_alloc(struct malloc_elem *elem, size_t size,
-		unsigned align, size_t bound);
+		unsigned int align, size_t bound, bool contig);
 
 /*
  * free a malloc_elem block by adding it to the free list. If the

index a2c2e4c..564b61a 100644
@@ -88,7 +88,7 @@ malloc_heap_add_memseg(struct malloc_heap *heap, struct rte_memseg *ms)
  */
 static struct malloc_elem *
 find_suitable_element(struct malloc_heap *heap, size_t size,
-		unsigned flags, size_t align, size_t bound)
+		unsigned int flags, size_t align, size_t bound, bool contig)
 {
 	size_t idx;
 	struct malloc_elem *elem, *alt_elem = NULL;

@@ -97,7 +97,8 @@ find_suitable_element(struct malloc_heap *heap, size_t size,
 			idx < RTE_HEAP_NUM_FREELISTS; idx++) {
 		for (elem = LIST_FIRST(&heap->free_head[idx]);
 				!!elem; elem = LIST_NEXT(elem, free_list)) {
-			if (malloc_elem_can_hold(elem, size, align, bound)) {
+			if (malloc_elem_can_hold(elem, size, align, bound,
+					contig)) {
 				if (check_hugepage_sz(flags, elem->ms->hugepage_sz))
 					return elem;
 				if (alt_elem == NULL)

@@ -121,7 +122,7 @@ find_suitable_element(struct malloc_heap *heap, size_t size,
 void *
 malloc_heap_alloc(struct malloc_heap *heap,
 		const char *type __attribute__((unused)), size_t size, unsigned flags,
-		size_t align, size_t bound)
+		size_t align, size_t bound, bool contig)
 {
 	struct malloc_elem *elem;
 

@@ -130,9 +131,9 @@ malloc_heap_alloc(struct malloc_heap *heap,
 
 	rte_spinlock_lock(&heap->lock);
 
-	elem = find_suitable_element(heap, size, flags, align, bound);
+	elem = find_suitable_element(heap, size, flags, align, bound, contig);
 	if (elem != NULL) {
-		elem = malloc_elem_alloc(elem, size, align, bound);
+		elem = malloc_elem_alloc(elem, size, align, bound, contig);
 		/* increase heap's count of allocated elements */
 		heap->alloc_count++;
 	}

index bb28422..c57b59a 100644
@@ -5,6 +5,8 @@
 #ifndef MALLOC_HEAP_H_
 #define MALLOC_HEAP_H_
 
+#include <stdbool.h>
+
 #include <rte_malloc.h>
 #include <rte_malloc_heap.h>
 

@@ -25,7 +27,7 @@ malloc_get_numa_socket(void)
 
 void *
 malloc_heap_alloc(struct malloc_heap *heap,	const char *type, size_t size,
-		unsigned flags, size_t align, size_t bound);
+		unsigned int flags, size_t align, size_t bound, bool contig);
 
 int
 malloc_heap_free(struct malloc_elem *elem);

index 2cda48e..436818a 100644
@@ -37,7 +37,8 @@ void rte_free(void *addr)
  * Allocate memory on specified heap.
  */
 void *
-rte_malloc_socket(const char *type, size_t size, unsigned align, int socket_arg)
+rte_malloc_socket(const char *type, size_t size, unsigned int align,
+		int socket_arg)
 {
 	struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
 	int socket, i;

@@ -60,7 +61,7 @@ rte_malloc_socket(const char *type, size_t size, unsigned align, int socket_arg)
 		return NULL;
 
 	ret = malloc_heap_alloc(&mcfg->malloc_heaps[socket], type,
-				size, 0, align == 0 ? 1 : align, 0);
+				size, 0, align == 0 ? 1 : align, 0, false);
 	if (ret != NULL || socket_arg != SOCKET_ID_ANY)
 		return ret;
 

@@ -71,7 +72,7 @@ rte_malloc_socket(const char *type, size_t size, unsigned align, int socket_arg)
 			continue;
 
 		ret = malloc_heap_alloc(&mcfg->malloc_heaps[i], type,
-					size, 0, align == 0 ? 1 : align, 0);
+				size, 0, align == 0 ? 1 : align, 0, false);
 		if (ret != NULL)
 			return ret;
 	}

index 16a2e7a..af68c00 100644
@@ -99,12 +99,13 @@ find_heap_max_free_elem(int *s, unsigned align)
 static const struct rte_memzone *
 memzone_reserve_aligned_thread_unsafe(const char *name, size_t len,
 		int socket_id, unsigned int flags, unsigned int align,
-		unsigned int bound, bool contig)
+		unsigned int bound)
 {
 	struct rte_memzone *mz;
 	struct rte_mem_config *mcfg;
 	size_t requested_len;
 	int socket, i;
+	bool contig;
 
 	/* get pointer to global configuration */
 	mcfg = rte_eal_get_configuration()->mem_config;

@@ -170,7 +171,17 @@ memzone_reserve_aligned_thread_unsafe(const char *name, size_t len,
 	if (!rte_eal_has_hugepages())
 		socket_id = SOCKET_ID_ANY;
 
+	contig = (flags & RTE_MEMZONE_IOVA_CONTIG) != 0;
+	/* malloc only cares about size flags, remove contig flag from flags */
+	flags &= ~RTE_MEMZONE_IOVA_CONTIG;
+
 	if (len == 0) {
+		/* len == 0 is only allowed for non-contiguous zones */
+		if (contig) {
+			RTE_LOG(DEBUG, EAL, "Reserving zero-length contiguous memzones is not supported\n");
+			rte_errno = EINVAL;
+			return NULL;
+		}
 		if (bound != 0)
 			requested_len = bound;
 		else {

@@ -238,8 +249,7 @@ memzone_reserve_aligned_thread_unsafe(const char *name, size_t len,
 
 static const struct rte_memzone *
 rte_memzone_reserve_thread_safe(const char *name, size_t len, int socket_id,
-		unsigned int flags, unsigned int align, unsigned int bound,
-		bool contig)
+		unsigned int flags, unsigned int align, unsigned int bound)
 {
 	struct rte_mem_config *mcfg;
 	const struct rte_memzone *mz = NULL;

@@ -250,7 +260,7 @@ rte_memzone_reserve_thread_safe(const char *name, size_t len, int socket_id,
 	rte_rwlock_write_lock(&mcfg->mlock);
 
 	mz = memzone_reserve_aligned_thread_unsafe(
-		name, len, socket_id, flags, align, bound, contig);
+		name, len, socket_id, flags, align, bound);
 
 	rte_rwlock_write_unlock(&mcfg->mlock);
 

@@ -267,7 +277,7 @@ rte_memzone_reserve_bounded(const char *name, size_t len, int socket_id,
 			    unsigned flags, unsigned align, unsigned bound)
 {
 	return rte_memzone_reserve_thread_safe(name, len, socket_id, flags,
-					       align, bound, false);
+					       align, bound);
 }
 
 /*

@@ -279,7 +289,7 @@ rte_memzone_reserve_aligned(const char *name, size_t len, int socket_id,
 			    unsigned flags, unsigned align)
 {
 	return rte_memzone_reserve_thread_safe(name, len, socket_id, flags,
-					       align, 0, false);
+					       align, 0);
 }
 
 /*

@@ -291,8 +301,7 @@ rte_memzone_reserve(const char *name, size_t len, int socket_id,
 		    unsigned flags)
 {
 	return rte_memzone_reserve_thread_safe(name, len, socket_id,
-					       flags, RTE_CACHE_LINE_SIZE, 0,
-					       false);
+					       flags, RTE_CACHE_LINE_SIZE, 0);
 }
 
 int

index 2bfb273..e2630fd 100644
@@ -23,6 +23,7 @@
  */
 
 #include <stdio.h>
+#include <rte_compat.h>
 #include <rte_memory.h>
 #include <rte_common.h>
 

@@ -39,6 +40,7 @@ extern "C" {
 #define RTE_MEMZONE_512MB          0x00040000   /**< Use 512MB pages. */
 #define RTE_MEMZONE_4GB            0x00080000   /**< Use 4GB pages. */
 #define RTE_MEMZONE_SIZE_HINT_ONLY 0x00000004   /**< Use available page size */
+#define RTE_MEMZONE_IOVA_CONTIG    0x00100000   /**< Ask for IOVA-contiguous memzone. */
 
 /**
  * A structure describing a memzone, which is a contiguous portion of

@@ -102,6 +104,9 @@ struct rte_memzone {
  *                                  If this flag is not set, the function
  *                                  will return error on an unavailable size
  *                                  request.
+ *   - RTE_MEMZONE_IOVA_CONTIG - Ensure reserved memzone is IOVA-contiguous.
+ *                               This option should be used when allocating
+ *                               memory intended for hardware rings etc.
  * @return
  *   A pointer to a correctly-filled read-only memzone descriptor, or NULL
  *   on error.

@@ -152,6 +157,9 @@ const struct rte_memzone *rte_memzone_reserve(const char *name,
  *                                  If this flag is not set, the function
  *                                  will return error on an unavailable size
  *                                  request.
+ *   - RTE_MEMZONE_IOVA_CONTIG - Ensure reserved memzone is IOVA-contiguous.
+ *                               This option should be used when allocating
+ *                               memory intended for hardware rings etc.
  * @param align
  *   Alignment for resulting memzone. Must be a power of 2.
  * @return

@@ -207,6 +215,9 @@ const struct rte_memzone *rte_memzone_reserve_aligned(const char *name,
  *                                  If this flag is not set, the function
  *                                  will return error on an unavailable size
  *                                  request.
+ *   - RTE_MEMZONE_IOVA_CONTIG - Ensure reserved memzone is IOVA-contiguous.
+ *                               This option should be used when allocating
+ *                               memory intended for hardware rings etc.
  * @param align
  *   Alignment for resulting memzone. Must be a power of 2.
  * @param bound

index 2c74f7e..d0cf0e7 100644
@@ -3403,7 +3403,8 @@ rte_eth_dma_zone_reserve(const struct rte_eth_dev *dev, const char *ring_name,
 	if (mz)
 		return mz;
 
-	return rte_memzone_reserve_aligned(z_name, size, socket_id, 0, align);
+	return rte_memzone_reserve_aligned(z_name, size, socket_id,
+			RTE_MEMZONE_IOVA_CONTIG, align);
 }
 
 int

index 87b9ce0..478b7ba 100644
@@ -54,8 +54,6 @@ queue_dma_zone_reserve(const char *queue_name, uint32_t queue_size,
 			int socket_id)
 {
 	const struct rte_memzone *mz;
-	unsigned memzone_flags = 0;
-	const struct rte_memseg *ms;
 
 	PMD_INIT_FUNC_TRACE();
 	mz = rte_memzone_lookup(queue_name);

@@ -78,25 +76,8 @@ queue_dma_zone_reserve(const char *queue_name, uint32_t queue_size,
 
 	PMD_DRV_LOG(DEBUG, "Allocate memzone for %s, size %u on socket %u",
 					queue_name, queue_size, socket_id);
-	ms = rte_eal_get_physmem_layout();
-	switch (ms[0].hugepage_sz) {
-	case(RTE_PGSIZE_2M):
-		memzone_flags = RTE_MEMZONE_2MB;
-	break;
-	case(RTE_PGSIZE_1G):
-		memzone_flags = RTE_MEMZONE_1GB;
-	break;
-	case(RTE_PGSIZE_16M):
-		memzone_flags = RTE_MEMZONE_16MB;
-	break;
-	case(RTE_PGSIZE_16G):
-		memzone_flags = RTE_MEMZONE_16GB;
-	break;
-	default:
-		memzone_flags = RTE_MEMZONE_SIZE_HINT_ONLY;
-	}
-	return rte_memzone_reserve_aligned(queue_name, queue_size, socket_id,
-		memzone_flags, queue_size);
+	return rte_memzone_reserve_aligned(queue_name, queue_size,
+		socket_id, RTE_MEMZONE_IOVA_CONTIG, queue_size);
 }
 
 int qat_crypto_sym_qp_setup(struct rte_cryptodev *dev, uint16_t queue_pair_id,

index 4442c3c..68a59b4 100644
@@ -1365,8 +1365,8 @@ avf_allocate_dma_mem_d(__rte_unused struct avf_hw *hw,
 		return AVF_ERR_PARAM;
 
 	snprintf(z_name, sizeof(z_name), "avf_dma_%"PRIu64, rte_rand());
-	mz = rte_memzone_reserve_bounded(z_name, size, SOCKET_ID_ANY, 0,
-					 alignment, RTE_PGSIZE_2M);
+	mz = rte_memzone_reserve_bounded(z_name, size, SOCKET_ID_ANY,
+			RTE_MEMZONE_IOVA_CONTIG, alignment, RTE_PGSIZE_2M);
 	if (!mz)
 		return AVF_ERR_NO_MEMORY;
 

index fb02d0f..81f5dae 100644
@@ -177,7 +177,7 @@ bnx2x_dma_alloc(struct bnx2x_softc *sc, size_t size, struct bnx2x_dma *dma,
 			rte_get_timer_cycles());
 
 	/* Caller must take care that strlen(mz_name) < RTE_MEMZONE_NAMESIZE */
-	z = rte_memzone_reserve_aligned(mz_name, (uint64_t) (size),
+	z = rte_memzone_reserve_aligned_contig(mz_name, (uint64_t)size,
 					SOCKET_ID_ANY,
 					0, align);
 	if (z == NULL) {

index a0d4ac9..6be7277 100644
@@ -26,7 +26,8 @@ ring_dma_zone_reserve(struct rte_eth_dev *dev, const char *ring_name,
 	if (mz)
 		return mz;
 
-	return rte_memzone_reserve_aligned(z_name, ring_size, socket_id, 0, BNX2X_PAGE_SIZE);
+	return rte_memzone_reserve_aligned(z_name, ring_size, socket_id,
+			RTE_MEMZONE_IOVA_CONTIG, BNX2X_PAGE_SIZE);
 }
 
 static void

index 0b21653..ad7d925 100644
@@ -3147,9 +3147,10 @@ bnxt_dev_init(struct rte_eth_dev *eth_dev)
 				sizeof(struct rx_port_stats) + 512);
 		if (!mz) {
 			mz = rte_memzone_reserve(mz_name, total_alloc_len,
-						 SOCKET_ID_ANY,
-						 RTE_MEMZONE_2MB |
-						 RTE_MEMZONE_SIZE_HINT_ONLY);
+					SOCKET_ID_ANY,
+					RTE_MEMZONE_2MB |
+					RTE_MEMZONE_SIZE_HINT_ONLY |
+					RTE_MEMZONE_IOVA_CONTIG);
 			if (mz == NULL)
 				return -ENOMEM;
 		}

@@ -3181,10 +3182,12 @@ bnxt_dev_init(struct rte_eth_dev *eth_dev)
 		total_alloc_len = RTE_CACHE_LINE_ROUNDUP(
 				sizeof(struct tx_port_stats) + 512);
 		if (!mz) {
-			mz = rte_memzone_reserve(mz_name, total_alloc_len,
-						 SOCKET_ID_ANY,
-						 RTE_MEMZONE_2MB |
-						 RTE_MEMZONE_SIZE_HINT_ONLY);
+			mz = rte_memzone_reserve(mz_name,
+					total_alloc_len,
+					SOCKET_ID_ANY,
+					RTE_MEMZONE_2MB |
+					RTE_MEMZONE_SIZE_HINT_ONLY |
+					RTE_MEMZONE_IOVA_CONTIG);
 			if (mz == NULL)
 				return -ENOMEM;
 		}

index 8fb8972..0e8a6a2 100644
@@ -166,10 +166,11 @@ int bnxt_alloc_rings(struct bnxt *bp, uint16_t qidx,
 	mz = rte_memzone_lookup(mz_name);
 	if (!mz) {
 		mz = rte_memzone_reserve_aligned(mz_name, total_alloc_len,
-					 SOCKET_ID_ANY,
-					 RTE_MEMZONE_2MB |
-					 RTE_MEMZONE_SIZE_HINT_ONLY,
-					 getpagesize());
+				SOCKET_ID_ANY,
+				RTE_MEMZONE_2MB |
+				RTE_MEMZONE_SIZE_HINT_ONLY |
+				RTE_MEMZONE_IOVA_CONTIG,
+				getpagesize());
 		if (mz == NULL)
 			return -ENOMEM;
 	}

index d4aeb4c..9ccc67e 100644
@@ -185,10 +185,10 @@ int bnxt_alloc_vnic_attributes(struct bnxt *bp)
 	mz = rte_memzone_lookup(mz_name);
 	if (!mz) {
 		mz = rte_memzone_reserve(mz_name,
-					 entry_length * max_vnics,
-					 SOCKET_ID_ANY,
-					 RTE_MEMZONE_2MB |
-					 RTE_MEMZONE_SIZE_HINT_ONLY);
+				entry_length * max_vnics, SOCKET_ID_ANY,
+				RTE_MEMZONE_2MB |
+				RTE_MEMZONE_SIZE_HINT_ONLY |
+				RTE_MEMZONE_IOVA_CONTIG);
 		if (!mz)
 			return -ENOMEM;
 	}

index 83e26d0..85846fc 100644
@@ -1344,7 +1344,8 @@ static void *alloc_ring(size_t nelem, size_t elem_size,
 	 * handle the maximum ring size is allocated in order to allow for
 	 * resizing in later calls to the queue setup function.
 	 */
-	tz = rte_memzone_reserve_aligned(z_name, len, socket_id, 0, 4096);
+	tz = rte_memzone_reserve_aligned(z_name, len, socket_id,
+			RTE_MEMZONE_IOVA_CONTIG, 4096);
 	if (!tz)
 		return NULL;
 

index 8cba319..9334519 100644
@@ -188,7 +188,8 @@ typedef uint64_t dma_addr_t;
 		ENA_TOUCH(dmadev); ENA_TOUCH(handle);			\
 		snprintf(z_name, sizeof(z_name),			\
 				"ena_alloc_%d", ena_alloc_cnt++);	\
-		mz = rte_memzone_reserve(z_name, size, SOCKET_ID_ANY, 0); \
+		mz = rte_memzone_reserve(z_name, size, SOCKET_ID_ANY,	\
+				RTE_MEMZONE_IOVA_CONTIG);		\
 		memset(mz->addr, 0, size);				\
 		virt = mz->addr;					\
 		phys = mz->iova;					\

@@ -206,7 +207,8 @@ typedef uint64_t dma_addr_t;
 		ENA_TOUCH(dmadev); ENA_TOUCH(dev_node);			\
 		snprintf(z_name, sizeof(z_name),			\
 				"ena_alloc_%d", ena_alloc_cnt++);	\
-		mz = rte_memzone_reserve(z_name, size, node, 0); \
+		mz = rte_memzone_reserve(z_name, size, node,		\
+				RTE_MEMZONE_IOVA_CONTIG);		\
 		memset(mz->addr, 0, size);				\
 		virt = mz->addr;					\
 		phys = mz->iova;					\

@@ -219,7 +221,8 @@ typedef uint64_t dma_addr_t;
 		ENA_TOUCH(dmadev); ENA_TOUCH(dev_node);			\
 		snprintf(z_name, sizeof(z_name),			\
 				"ena_alloc_%d", ena_alloc_cnt++);	\
-		mz = rte_memzone_reserve(z_name, size, node, 0); \
+		mz = rte_memzone_reserve(z_name, size, node,		\
+				RTE_MEMZONE_IOVA_CONTIG);		\
 		memset(mz->addr, 0, size);				\
 		virt = mz->addr;					\
 	} while (0)

index 69ad425..94e8e68 100644
@@ -343,8 +343,8 @@ enic_alloc_consistent(void *priv, size_t size,
 	struct enic *enic = (struct enic *)priv;
 	struct enic_memzone_entry *mze;
 
-	rz = rte_memzone_reserve_aligned((const char *)name,
-					 size, SOCKET_ID_ANY, 0, ENIC_ALIGN);
+	rz = rte_memzone_reserve_aligned((const char *)name, size,
+			SOCKET_ID_ANY, RTE_MEMZONE_IOVA_CONTIG, ENIC_ALIGN);
 	if (!rz) {
 		pr_err("%s : Failed to allocate memory requested for %s\n",
 			__func__, name);

@@ -888,9 +888,8 @@ int enic_alloc_wq(struct enic *enic, uint16_t queue_idx,
 		instance++);
 
 	wq->cqmsg_rz = rte_memzone_reserve_aligned((const char *)name,
-						   sizeof(uint32_t),
-						   SOCKET_ID_ANY, 0,
-						   ENIC_ALIGN);
+			sizeof(uint32_t), SOCKET_ID_ANY,
+			RTE_MEMZONE_IOVA_CONTIG, ENIC_ALIGN);
 	if (!wq->cqmsg_rz)
 		return -ENOMEM;
 

index d0bf4e3..e00f402 100644
@@ -4053,8 +4053,8 @@ i40e_allocate_dma_mem_d(__attribute__((unused)) struct i40e_hw *hw,
 		return I40E_ERR_PARAM;
 
 	snprintf(z_name, sizeof(z_name), "i40e_dma_%"PRIu64, rte_rand());
-	mz = rte_memzone_reserve_bounded(z_name, size, SOCKET_ID_ANY, 0,
-					 alignment, RTE_PGSIZE_2M);
+	mz = rte_memzone_reserve_bounded(z_name, size, SOCKET_ID_ANY,
+			RTE_MEMZONE_IOVA_CONTIG, alignment, RTE_PGSIZE_2M);
 	if (!mz)
 		return I40E_ERR_NO_MEMORY;
 

index 1217e5a..56a854c 100644
@@ -2189,8 +2189,8 @@ i40e_memzone_reserve(const char *name, uint32_t len, int socket_id)
 	if (mz)
 		return mz;
 
-	mz = rte_memzone_reserve_aligned(name, len,
-					 socket_id, 0, I40E_RING_BASE_ALIGN);
+	mz = rte_memzone_reserve_aligned(name, len, socket_id,
+			RTE_MEMZONE_IOVA_CONTIG, I40E_RING_BASE_ALIGN);
 	return mz;
 }
 

index 91017b8..f550412 100644
@@ -135,8 +135,8 @@ void *osal_dma_alloc_coherent(struct ecore_dev *p_dev,
 	if (core_id == (unsigned int)LCORE_ID_ANY)
 		core_id = rte_get_master_lcore();
 	socket_id = rte_lcore_to_socket_id(core_id);
-	mz = rte_memzone_reserve_aligned(mz_name, size,
-					 socket_id, 0, RTE_CACHE_LINE_SIZE);
+	mz = rte_memzone_reserve_aligned(mz_name, size, socket_id,
+			RTE_MEMZONE_IOVA_CONTIG, RTE_CACHE_LINE_SIZE);
 	if (!mz) {
 		DP_ERR(p_dev, "Unable to allocate DMA memory "
 		       "of size %zu bytes - %s\n",

@@ -174,7 +174,8 @@ void *osal_dma_alloc_coherent_aligned(struct ecore_dev *p_dev,
 	if (core_id == (unsigned int)LCORE_ID_ANY)
 		core_id = rte_get_master_lcore();
 	socket_id = rte_lcore_to_socket_id(core_id);
-	mz = rte_memzone_reserve_aligned(mz_name, size, socket_id, 0, align);
+	mz = rte_memzone_reserve_aligned(mz_name, size, socket_id,
+			RTE_MEMZONE_IOVA_CONTIG, align);
 	if (!mz) {
 		DP_ERR(p_dev, "Unable to allocate DMA memory "
 		       "of size %zu bytes - %s\n",

index 2ef213d..f03d790 100644
@@ -391,8 +391,8 @@ virtio_init_queue(struct rte_eth_dev *dev, uint16_t vtpci_queue_idx)
 		     size, vq->vq_ring_size);
 
 	mz = rte_memzone_reserve_aligned(vq_name, vq->vq_ring_size,
-					 SOCKET_ID_ANY,
-					 0, VIRTIO_PCI_VRING_ALIGN);
+			SOCKET_ID_ANY, RTE_MEMZONE_IOVA_CONTIG,
+			VIRTIO_PCI_VRING_ALIGN);
 	if (mz == NULL) {
 		if (rte_errno == EEXIST)
 			mz = rte_memzone_lookup(vq_name);

@@ -417,8 +417,8 @@ virtio_init_queue(struct rte_eth_dev *dev, uint16_t vtpci_queue_idx)
 		snprintf(vq_hdr_name, sizeof(vq_hdr_name), "port%d_vq%d_hdr",
 			 dev->data->port_id, vtpci_queue_idx);
 		hdr_mz = rte_memzone_reserve_aligned(vq_hdr_name, sz_hdr_mz,
-						     SOCKET_ID_ANY, 0,
-						     RTE_CACHE_LINE_SIZE);
+				SOCKET_ID_ANY, RTE_MEMZONE_IOVA_CONTIG,
+				RTE_CACHE_LINE_SIZE);
 		if (hdr_mz == NULL) {
 			if (rte_errno == EEXIST)
 				hdr_mz = rte_memzone_lookup(vq_hdr_name);

index 4260087..104664a 100644
@@ -150,13 +150,14 @@ gpa_zone_reserve(struct rte_eth_dev *dev, uint32_t size,
 		if (mz)
 			rte_memzone_free(mz);
 		return rte_memzone_reserve_aligned(z_name, size, socket_id,
-						   0, align);
+				RTE_MEMZONE_IOVA_CONTIG, align);
 	}
 
 	if (mz)
 		return mz;
 
-	return rte_memzone_reserve_aligned(z_name, size, socket_id, 0, align);
+	return rte_memzone_reserve_aligned(z_name, size, socket_id,
+			RTE_MEMZONE_IOVA_CONTIG, align);
 }
 
 /*

index 54f7f4b..85fbdca 100644
@@ -3,6 +3,7 @@
  * Copyright(c) 2016 6WIND S.A.
  */
 
+#include <stdbool.h>
 #include <stdio.h>
 #include <string.h>
 #include <stdint.h>

@@ -98,6 +99,27 @@ static unsigned optimize_object_size(unsigned obj_size)
 	return new_obj_size * RTE_MEMPOOL_ALIGN;
 }
 
+static size_t
+get_min_page_size(void)
+{
+	const struct rte_mem_config *mcfg =
+			rte_eal_get_configuration()->mem_config;
+	int i;
+	size_t min_pagesz = SIZE_MAX;
+
+	for (i = 0; i < RTE_MAX_MEMSEG; i++) {
+		const struct rte_memseg *ms = &mcfg->memseg[i];
+
+		if (ms->addr == NULL)
+			continue;
+
+		if (ms->hugepage_sz < min_pagesz)
+			min_pagesz = ms->hugepage_sz;
+	}
+
+	return min_pagesz == SIZE_MAX ? (size_t) getpagesize() : min_pagesz;
+}
+
 static void
 mempool_add_elem(struct rte_mempool *mp, void *obj, rte_iova_t iova)
 {

@@ -204,7 +226,6 @@ rte_mempool_calc_obj_size(uint32_t elt_size, uint32_t flags,
 	return sz->total_size;
 }
 
-
 /*
  * Calculate maximum amount of memory required to store given number of objects.
  */

@@ -367,16 +388,6 @@ rte_mempool_populate_iova(struct rte_mempool *mp, char *vaddr,
 	/* update mempool capabilities */
 	mp->flags |= mp_capa_flags;
 
-	/* Detect pool area has sufficient space for elements */
-	if (mp_capa_flags & MEMPOOL_F_CAPA_PHYS_CONTIG) {
-		if (len < total_elt_sz * mp->size) {
-			RTE_LOG(ERR, MEMPOOL,
-				"pool area %" PRIx64 " not enough\n",
-				(uint64_t)len);
-			return -ENOSPC;
-		}
-	}
-
 	memhdr = rte_zmalloc("MEMPOOL_MEMHDR", sizeof(*memhdr), 0);
 	if (memhdr == NULL)
 		return -ENOMEM;

@@ -549,6 +560,7 @@ rte_mempool_populate_default(struct rte_mempool *mp)
 	unsigned mz_id, n;
 	unsigned int mp_flags;
 	int ret;
+	bool force_contig, no_contig, try_contig, no_pageshift;
 
 	/* mempool must not be populated */
 	if (mp->nb_mem_chunks != 0)

@@ -563,9 +575,68 @@ rte_mempool_populate_default(struct rte_mempool *mp)
 	/* update mempool capabilities */
 	mp->flags |= mp_flags;
 
-	if (rte_eal_has_hugepages()) {
-		pg_shift = 0; /* not needed, zone is physically contiguous */
+	no_contig = mp->flags & MEMPOOL_F_NO_PHYS_CONTIG;
+	force_contig = mp->flags & MEMPOOL_F_CAPA_PHYS_CONTIG;
+
+	/*
+	 * the following section calculates page shift and page size values.
+	 *
+	 * these values impact the result of rte_mempool_xmem_size(), which
+	 * returns the amount of memory that should be allocated to store the
+	 * desired number of objects. when not zero, it allocates more memory
+	 * for the padding between objects, to ensure that an object does not
+	 * cross a page boundary. in other words, page size/shift are to be set
+	 * to zero if mempool elements won't care about page boundaries.
+	 * there are several considerations for page size and page shift here.
+	 *
+	 * if we don't need our mempools to have physically contiguous objects,
+	 * then just set page shift and page size to 0, because the user has
+	 * indicated that there's no need to care about anything.
+	 *
+	 * if we do need contiguous objects, there is also an option to reserve
+	 * the entire mempool memory as one contiguous block of memory, in
+	 * which case the page shift and alignment wouldn't matter as well.
+	 *
+	 * if we require contiguous objects, but not necessarily the entire
+	 * mempool reserved space to be contiguous, then there are two options.
+	 *
+	 * if our IO addresses are virtual, not actual physical (IOVA as VA
+	 * case), then no page shift needed - our memory allocation will give us
+	 * contiguous physical memory as far as the hardware is concerned, so
+	 * act as if we're getting contiguous memory.
+	 *
+	 * if our IO addresses are physical, we may get memory from bigger
+	 * pages, or we might get memory from smaller pages, and how much of it
+	 * we require depends on whether we want bigger or smaller pages.
+	 * However, requesting each and every memory size is too much work, so
+	 * what we'll do instead is walk through the page sizes available, pick
+	 * the smallest one and set up page shift to match that one. We will be
+	 * wasting some space this way, but it's much nicer than looping around
+	 * trying to reserve each and every page size.
+	 *
+	 * However, since size calculation will produce page-aligned sizes, it
+	 * makes sense to first try and see if we can reserve the entire memzone
+	 * in one contiguous chunk as well (otherwise we might end up wasting a
+	 * 1G page on a 10MB memzone). If we fail to get enough contiguous
+	 * memory, then we'll go and reserve space page-by-page.
+	 */
+	no_pageshift = no_contig || force_contig ||
+			rte_eal_iova_mode() == RTE_IOVA_VA;
+	try_contig = !no_contig && !no_pageshift && rte_eal_has_hugepages();
+	if (force_contig)
+		mz_flags |= RTE_MEMZONE_IOVA_CONTIG;
+
+	if (no_pageshift) {
 		pg_sz = 0;
+		pg_shift = 0;
+		align = RTE_CACHE_LINE_SIZE;
+	} else if (try_contig) {
+		pg_sz = get_min_page_size();
+		pg_shift = rte_bsf32(pg_sz);
+		/* we're trying to reserve contiguous memzone first, so try
+		 * align to cache line; if we fail to reserve a contiguous
+		 * memzone, we'll adjust alignment to equal pagesize later.
+		 */
 		align = RTE_CACHE_LINE_SIZE;
 	} else {
 		pg_sz = getpagesize();

@@ -575,8 +646,13 @@ rte_mempool_populate_default(struct rte_mempool *mp)
 
 	total_elt_sz = mp->header_size + mp->elt_size + mp->trailer_size;
 	for (mz_id = 0, n = mp->size; n > 0; mz_id++, n -= ret) {
-		size = rte_mempool_xmem_size(n, total_elt_sz, pg_shift,
-						mp->flags);
+		unsigned int flags;
+		if (try_contig || no_pageshift)
+			size = rte_mempool_xmem_size(n, total_elt_sz, 0,
+				mp->flags);
+		else
+			size = rte_mempool_xmem_size(n, total_elt_sz, pg_shift,
+				mp->flags);
 
 		ret = snprintf(mz_name, sizeof(mz_name),
 			RTE_MEMPOOL_MZ_FORMAT "_%d", mp->name, mz_id);

@@ -585,23 +661,52 @@ rte_mempool_populate_default(struct rte_mempool *mp)
 			goto fail;
 		}
 
-		mz = rte_memzone_reserve_aligned(mz_name, size,
-			mp->socket_id, mz_flags, align);
-		/* not enough memory, retry with the biggest zone we have */
-		if (mz == NULL)
+		flags = mz_flags;
+
+		/* if we're trying to reserve contiguous memory, add appropriate
+		 * memzone flag.
+		 */
+		if (try_contig)
+			flags |= RTE_MEMZONE_IOVA_CONTIG;
+
+		mz = rte_memzone_reserve_aligned(mz_name, size, mp->socket_id,
+				flags, align);
+
+		/* if we were trying to allocate contiguous memory, adjust
+		 * memzone size and page size to fit smaller page sizes, and
+		 * try again.
+		 */
+		if (mz == NULL && try_contig) {
+			try_contig = false;
+			flags &= ~RTE_MEMZONE_IOVA_CONTIG;
+			align = pg_sz;
+			size = rte_mempool_xmem_size(n, total_elt_sz,
+				pg_shift, mp->flags);
+
+			mz = rte_memzone_reserve_aligned(mz_name, size,
+				mp->socket_id, flags, align);
+		}
+		/* don't try reserving with 0 size if we were asked to reserve
+		 * IOVA-contiguous memory.
+		 */
+		if (!force_contig && mz == NULL) {
+			/* not enough memory, retry with the biggest zone we
+			 * have
+			 */
 			mz = rte_memzone_reserve_aligned(mz_name, 0,
-				mp->socket_id, mz_flags, align);
+					mp->socket_id, flags, align);
+		}
 		if (mz == NULL) {
 			ret = -rte_errno;
 			goto fail;
 		}
 
-		if (mp->flags & MEMPOOL_F_NO_PHYS_CONTIG)
+		if (no_contig)
 			iova = RTE_BAD_IOVA;
 		else
 			iova = mz->iova;
 
-		if (rte_eal_has_hugepages())
+		if (no_pageshift || try_contig)
 			ret = rte_mempool_populate_iova(mp, mz->addr,
 				iova, mz->len,
 				rte_mempool_memchunk_mz_free,

index 5b8ced4..947db1f 100644
@@ -218,6 +218,27 @@ rte_mem_lock_page(const void *virt)
 	return mlock((void *)aligned, page_size);
 }
 
+int __rte_experimental
+rte_memseg_walk(rte_memseg_walk_t func, void *arg)
+{
+	struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
+	int i, ret;
+
+	for (i = 0; i < RTE_MAX_MEMSEG; i++) {
+		const struct rte_memseg *ms = &mcfg->memseg[i];
+
+		if (ms->addr == NULL)
+			continue;
+
+		ret = func(ms, arg);
+		if (ret < 0)
+			return -1;
+		if (ret > 0)
+			return 1;
+	}
+	return 0;
+}
+
 /* init memory subsystem */
 int
 rte_eal_memory_init(void)

index 302f865..93eadaa 100644
@@ -20,6 +20,7 @@ extern "C" {
 #endif
 
 #include <rte_common.h>
+#include <rte_compat.h>
 #include <rte_config.h>
 
 __extension__

@@ -130,6 +131,30 @@ phys_addr_t rte_mem_virt2phy(const void *virt);
 rte_iova_t rte_mem_virt2iova(const void *virt);
 
 /**
+ * Memseg walk function prototype.
+ *
+ * Returning 0 will continue walk
+ * Returning 1 will stop the walk
+ * Returning -1 will stop the walk and report error
+ */
+typedef int (*rte_memseg_walk_t)(const struct rte_memseg *ms, void *arg);
+
+/**
+ * Walk list of all memsegs.
+ *
+ * @param func
+ *   Iterator function
+ * @param arg
+ *   Argument passed to iterator
+ * @return
+ *   0 if walked over the entire list
+ *   1 if stopped by the user
+ *   -1 if user function reported error
+ */
+int __rte_experimental
+rte_memseg_walk(rte_memseg_walk_t func, void *arg);
+
+/**
  * Get the layout of the available physical memory.
  *
  * It can be useful for an application to have the full physical

index 4291871..0c048dc 100644
@@ -189,17 +189,51 @@ static int vfio_map_irq_region(struct fslmc_vfio_group *group)
 	return -errno;
 }
 
-int rte_fslmc_vfio_dmamap(void)
+static int
+fslmc_vfio_map(const struct rte_memseg *ms, void *arg)
 {
-	int ret;
+	int *n_segs = arg;
 	struct fslmc_vfio_group *group;
 	struct vfio_iommu_type1_dma_map dma_map = {
 		.argsz = sizeof(struct vfio_iommu_type1_dma_map),
 		.flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE,
 	};
+	int ret;
+
+	dma_map.size = ms->len;
+	dma_map.vaddr = ms->addr_64;
+#ifdef RTE_LIBRTE_DPAA2_USE_PHYS_IOVA
+	dma_map.iova = ms->iova;
+#else
+	dma_map.iova = dma_map.vaddr;
+#endif
+
+	/* SET DMA MAP for IOMMU */
+	group = &vfio_group;
+
+	if (!group->container) {
+		DPAA2_BUS_ERR("Container is not connected ");
+		return -1;
+	}
+
+	DPAA2_BUS_DEBUG("-->Initial SHM Virtual ADDR %llX",
+			dma_map.vaddr);
+	DPAA2_BUS_DEBUG("-----> DMA size 0x%llX", dma_map.size);
+	ret = ioctl(group->container->fd, VFIO_IOMMU_MAP_DMA,
+			&dma_map);
+	if (ret) {
+		DPAA2_BUS_ERR("VFIO_IOMMU_MAP_DMA API(errno = %d)",
+				errno);
+		return -1;
+	}
+	(*n_segs)++;
+	return 0;
+}
 
-	int i;
+int rte_fslmc_vfio_dmamap(void)
+{
 	const struct rte_memseg *memseg;
+	int i = 0;
 
 	if (is_dma_done)
 		return 0;

@@ -210,51 +244,21 @@ int rte_fslmc_vfio_dmamap(void)
 		return -ENODEV;
 	}
 
-	for (i = 0; i < RTE_MAX_MEMSEG; i++) {
-		if (memseg[i].addr == NULL && memseg[i].len == 0) {
-			DPAA2_BUS_DEBUG("Total %d segments found", i);
-			break;
-		}
-
-		dma_map.size = memseg[i].len;
-		dma_map.vaddr = memseg[i].addr_64;
-#ifdef RTE_LIBRTE_DPAA2_USE_PHYS_IOVA
-		dma_map.iova = memseg[i].iova;
-#else
-		dma_map.iova = dma_map.vaddr;
-#endif
-
-		/* SET DMA MAP for IOMMU */
-		group = &vfio_group;
-
-		if (!group->container) {
-			DPAA2_BUS_ERR("Container is not connected");
-			return -1;
-		}
-
-		DPAA2_BUS_DEBUG("-->Initial SHM Virtual ADDR %llX",
-				dma_map.vaddr);
-		DPAA2_BUS_DEBUG("-----> DMA size 0x%llX", dma_map.size);
-		ret = ioctl(group->container->fd, VFIO_IOMMU_MAP_DMA,
-			    &dma_map);
-		if (ret) {
-			DPAA2_BUS_ERR("Unable to map DMA address (errno = %d)",
-				      errno);
-			return ret;
-		}
-	}
+	if (rte_memseg_walk(fslmc_vfio_map, &i) < 0)
+		return -1;
 
 	/* Verifying that at least single segment is available */
 	if (i <= 0) {
 		DPAA2_BUS_ERR("No Segments found for VFIO Mapping");
 		return -1;
 	}
+	DPAA2_BUS_DEBUG("Total %d segments found.", i);
 
 	/* TODO - This is a W.A. as VFIO currently does not add the mapping of
 	 * the interrupt region to SMMU. This should be removed once the
 	 * support is added in the Kernel.
 	 */
-	vfio_map_irq_region(group);
+	vfio_map_irq_region(&vfio_group);
 
 	is_dma_done = 1;
 

index abde641..6dda054 100644
@@ -116,22 +116,24 @@ rte_pci_unmap_device(struct rte_pci_device *dev)
 	}
 }
 
-void *
-pci_find_max_end_va(void)
+static int
+find_max_end_va(const struct rte_memseg *ms, void *arg)
 {
-	const struct rte_memseg *seg = rte_eal_get_physmem_layout();
-	const struct rte_memseg *last = seg;
-	unsigned i = 0;
+	void *end_va = RTE_PTR_ADD(ms->addr, ms->len);
+	void **max_va = arg;
 
-	for (i = 0; i < RTE_MAX_MEMSEG; i++, seg++) {
-		if (seg->addr == NULL)
-			break;
+	if (*max_va < end_va)
+		*max_va = end_va;
+	return 0;
+}
 
-		if (seg->addr > last->addr)
-			last = seg;
+void *
+pci_find_max_end_va(void)
+{
+	void *va = NULL;
 
-	}
-	return RTE_PTR_ADD(last->addr, last->len);
+	rte_memseg_walk(find_max_end_va, &va);
+	return va;
 }
 
 /* parse one line of the "resource" sysfs file (note that the 'line'

index 7d58d66..1724b65 100644
@@ -477,6 +477,19 @@ static struct rte_pci_driver mlx5_driver;
  */
 static void *uar_base;
 
+static int
+find_lower_va_bound(const struct rte_memseg *ms, void *arg)
+{
+	void **addr = arg;
+
+	if (*addr == NULL)
+		*addr = ms->addr;
+	else
+		*addr = RTE_MIN(*addr, ms->addr);
+
+	return 0;
+}
+
 /**
  * Reserve UAR address space for primary process.
  *

@@ -491,21 +504,14 @@ mlx5_uar_init_primary(struct rte_eth_dev *dev)
 {
 	struct priv *priv = dev->data->dev_private;
 	void *addr = (void *)0;
-	int i;
-	const struct rte_mem_config *mcfg;
 
 	if (uar_base) { /* UAR address space mapped. */
 		priv->uar_base = uar_base;
 		return 0;
 	}
 	/* find out lower bound of hugepage segments */
-	mcfg = rte_eal_get_configuration()->mem_config;
-	for (i = 0; i < RTE_MAX_MEMSEG && mcfg->memseg[i].addr; i++) {
-		if (addr)
-			addr = RTE_MIN(addr, mcfg->memseg[i].addr);
-		else
-			addr = mcfg->memseg[i].addr;
-	}
+	rte_memseg_walk(find_lower_va_bound, &addr);
+
 	/* keep distance to hugepages to minimize potential conflicts. */
 	addr = RTE_PTR_SUB(addr, MLX5_UAR_OFFSET + MLX5_UAR_SIZE);
 	/* anonymous mmap, no real memory consumption. */

index 4eafcb5..8e25d78 100644
@@ -429,23 +429,26 @@ eal_parse_args(int argc, char **argv)
 	return ret;
 }
 
+static int
+check_socket(const struct rte_memseg *ms, void *arg)
+{
+	int *socket_id = arg;
+
+	if (ms->socket_id == *socket_id)
+		return 1;
+
+	return 0;
+}
+
 static void
 eal_check_mem_on_local_socket(void)
 {
-	const struct rte_memseg *ms;
-	int i, socket_id;
+	int socket_id;
 
 	socket_id = rte_lcore_to_socket_id(rte_config.master_lcore);
 
-	ms = rte_eal_get_physmem_layout();
-
-	for (i = 0; i < RTE_MAX_MEMSEG; i++)
-		if (ms[i].socket_id == socket_id &&
-				ms[i].len > 0)
-			return;
-
-	RTE_LOG(WARNING, EAL, "WARNING: Master core has no "
-			"memory on local socket!\n");
+	if (rte_memseg_walk(check_socket, &socket_id) == 0)
+		RTE_LOG(WARNING, EAL, "WARNING: Master core has no memory on local socket!\n");
 }
 
 static int

89 lines not on this page · raw message

index 564b61a..79914fc 100644
@@ -67,17 +67,32 @@ check_hugepage_sz(unsigned flags, uint64_t hugepage_sz)
  * to prevent overflow. The rest of the zone is added to free list as a single
  * large free block
  */
-static void
-malloc_heap_add_memseg(struct malloc_heap *heap, struct rte_memseg *ms)
+static int
+malloc_heap_add_memseg(const struct rte_memseg *ms, void *arg __rte_unused)
 {
-	struct malloc_elem *start_elem = (struct malloc_elem *)ms->addr;
-	const size_t elem_size = ms->len - MALLOC_ELEM_OVERHEAD;
+	struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
+	struct malloc_elem *start_elem;
+	struct rte_memseg *found_ms;
+	struct malloc_heap *heap;
+	size_t elem_size;
+	int ms_idx;
+
+	heap = &mcfg->malloc_heaps[ms->socket_id];
+
+	/* ms is const, so find it */
+	ms_idx = ms - mcfg->memseg;
+	found_ms = &mcfg->memseg[ms_idx];
 
-	malloc_elem_init(start_elem, heap, ms, elem_size);
+	start_elem = (struct malloc_elem *)found_ms->addr;
+	elem_size = ms->len - MALLOC_ELEM_OVERHEAD;
+
+	malloc_elem_init(start_elem, heap, found_ms, elem_size);
 	malloc_elem_insert(start_elem);
 	malloc_elem_free_list_insert(start_elem);
 
 	heap->total_size += elem_size;
+
+	return 0;
 }
 
 /*

@@ -244,17 +259,11 @@ int
 rte_eal_malloc_heap_init(void)
 {
 	struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
-	unsigned ms_cnt;
-	struct rte_memseg *ms;
 
 	if (mcfg == NULL)
 		return -1;
 
-	for (ms = &mcfg->memseg[0], ms_cnt = 0;
-			(ms_cnt < RTE_MAX_MEMSEG) && (ms->len > 0);
-			ms_cnt++, ms++) {
-		malloc_heap_add_memseg(&mcfg->malloc_heaps[ms->socket_id], ms);
-	}
+	rte_memseg_walk(malloc_heap_add_memseg, NULL);
 
 	return 0;
 }

index 2ecd07b..77f6cb7 100644
@@ -638,23 +638,23 @@ eal_parse_args(int argc, char **argv)
 	return ret;
 }
 
+static int
+check_mem(const struct rte_memseg *ms, void *arg)
+{
+	int *socket = arg;
+
+	return ms->socket_id == *socket;
+}
+
 static void
 eal_check_mem_on_local_socket(void)
 {
-	const struct rte_memseg *ms;
-	int i, socket_id;
+	int socket_id;
 
 	socket_id = rte_lcore_to_socket_id(rte_config.master_lcore);
 
-	ms = rte_eal_get_physmem_layout();
-
-	for (i = 0; i < RTE_MAX_MEMSEG; i++)
-		if (ms[i].socket_id == socket_id &&
-				ms[i].len > 0)
-			return;
-
-	RTE_LOG(WARNING, EAL, "WARNING: Master core has no "
-			"memory on local socket!\n");
+	if (rte_memseg_walk(check_mem, &socket_id) == 0)
+		RTE_LOG(WARNING, EAL, "WARNING: Master core has no memory on local socket!\n");
 }
 
 static int

index 85fbdca..ea62b6b 100644
@@ -99,23 +99,23 @@ static unsigned optimize_object_size(unsigned obj_size)
 	return new_obj_size * RTE_MEMPOOL_ALIGN;
 }
 
+static int
+find_min_pagesz(const struct rte_memseg *ms, void *arg)
+{
+	size_t *min = arg;
+
+	if (ms->hugepage_sz < *min)
+		*min = ms->hugepage_sz;
+
+	return 0;
+}
+
 static size_t
 get_min_page_size(void)
 {
-	const struct rte_mem_config *mcfg =
-			rte_eal_get_configuration()->mem_config;
-	int i;
 	size_t min_pagesz = SIZE_MAX;
 
-	for (i = 0; i < RTE_MAX_MEMSEG; i++) {
-		const struct rte_memseg *ms = &mcfg->memseg[i];
-
-		if (ms->addr == NULL)
-			continue;
-
-		if (ms->hugepage_sz < min_pagesz)
-			min_pagesz = ms->hugepage_sz;
-	}
+	rte_memseg_walk(find_min_pagesz, &min_pagesz);
 
 	return min_pagesz == SIZE_MAX ? (size_t) getpagesize() : min_pagesz;
 }

index ccc5fea..28c241f 100644
@@ -705,16 +705,34 @@ test_malloc_bad_params(void)
 	return -1;
 }
 
+static int
+check_socket_mem(const struct rte_memseg *ms, void *arg)
+{
+	int32_t *socket = arg;
+
+	return *socket == ms->socket_id;
+}
+
 /* Check if memory is available on a specific socket */
 static int
 is_mem_on_socket(int32_t socket)
 {
-	const struct rte_memseg *ms = rte_eal_get_physmem_layout();
-	unsigned i;
+	return rte_memseg_walk(check_socket_mem, &socket);
+}
 
-	for (i = 0; i < RTE_MAX_MEMSEG; i++) {
-		if (socket == ms[i].socket_id)
-			return 1;
+struct walk_param {
+	void *addr;
+	int32_t socket;
+};
+static int
+find_socket(const struct rte_memseg *ms, void *arg)
+{
+	struct walk_param *param = arg;
+
+	if (param->addr >= ms->addr &&
+			param->addr < RTE_PTR_ADD(ms->addr, ms->len)) {
+		param->socket = ms->socket_id;
+		return 1;
 	}
 	return 0;
 }

@@ -726,15 +744,9 @@ is_mem_on_socket(int32_t socket)
 static int32_t
 addr_to_socket(void * addr)
 {
-	const struct rte_memseg *ms = rte_eal_get_physmem_layout();
-	unsigned i;
-
-	for (i = 0; i < RTE_MAX_MEMSEG; i++) {
-		if ((ms[i].addr <= addr) &&
-				((uintptr_t)addr <
-				((uintptr_t)ms[i].addr + (uintptr_t)ms[i].len)))
-			return ms[i].socket_id;
-	}
+	struct walk_param param = {.addr = addr, .socket = 0};
+	if (rte_memseg_walk(find_socket, &param) > 0)
+		return param.socket;
 	return -1;
 }
 

index 972321f..c9b287c 100644
@@ -23,12 +23,20 @@
  */
 
 static int
+check_mem(const struct rte_memseg *ms, void *arg __rte_unused)
+{
+	volatile uint8_t *mem = (volatile uint8_t *) ms->addr;
+	size_t i;
+
+	for (i = 0; i < ms->len; i++, mem++)
+		*mem;
+	return 0;
+}
+
+static int
 test_memory(void)
 {
 	uint64_t s;
-	unsigned i;
-	size_t j;
-	const struct rte_memseg *mem;
 
 	/*
 	 * dump the mapped memory: the python-expect script checks

@@ -45,14 +53,7 @@ test_memory(void)
 	}
 
 	/* try to read memory (should not segfault) */
-	mem = rte_eal_get_physmem_layout();
-	for (i = 0; i < RTE_MAX_MEMSEG && mem[i].addr != NULL ; i++) {
-
-		/* check memory */
-		for (j = 0; j<mem[i].len; j++) {
-			*((volatile uint8_t *) mem[i].addr + j);
-		}
-	}
+	rte_memseg_walk(check_mem, NULL);
 
 	return 0;
 }

64 lines not on this page · raw message

index 2421d51..2a34ae9 100644
@@ -665,39 +665,36 @@ vfio_get_group_no(const char *sysfs_base,
 }
 
 static int
-vfio_type1_dma_map(int vfio_container_fd)
+type1_map(const struct rte_memseg *ms, void *arg)
 {
-	const struct rte_memseg *ms = rte_eal_get_physmem_layout();
-	int i, ret;
+	int *vfio_container_fd = arg;
+	struct vfio_iommu_type1_dma_map dma_map;
+	int ret;
 
-	/* map all DPDK segments for DMA. use 1:1 PA to IOVA mapping */
-	for (i = 0; i < RTE_MAX_MEMSEG; i++) {
-		struct vfio_iommu_type1_dma_map dma_map;
+	memset(&dma_map, 0, sizeof(dma_map));
+	dma_map.argsz = sizeof(struct vfio_iommu_type1_dma_map);
+	dma_map.vaddr = ms->addr_64;
+	dma_map.size = ms->len;
+	dma_map.iova = ms->iova;
+	dma_map.flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE;
 
-		if (ms[i].addr == NULL)
-			break;
-
-		memset(&dma_map, 0, sizeof(dma_map));
-		dma_map.argsz = sizeof(struct vfio_iommu_type1_dma_map);
-		dma_map.vaddr = ms[i].addr_64;
-		dma_map.size = ms[i].len;
-		dma_map.iova = ms[i].iova;
-		dma_map.flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE;
+	ret = ioctl(*vfio_container_fd, VFIO_IOMMU_MAP_DMA, &dma_map);
 
-		ret = ioctl(vfio_container_fd, VFIO_IOMMU_MAP_DMA, &dma_map);
-
-		if (ret) {
-			RTE_LOG(ERR, EAL, "  cannot set up DMA remapping, "
-					  "error %i (%s)\n", errno,
-					  strerror(errno));
-			return -1;
-		}
+	if (ret) {
+		RTE_LOG(ERR, EAL, "  cannot set up DMA remapping, error %i (%s)\n",
+				errno, strerror(errno));
+		return -1;
 	}
-
 	return 0;
 }
 
 static int
+vfio_type1_dma_map(int vfio_container_fd)
+{
+	return rte_memseg_walk(type1_map, &vfio_container_fd);
+}
+
+static int
 vfio_spapr_dma_map(int vfio_container_fd)
 {
 	const struct rte_memseg *ms = rte_eal_get_physmem_layout();

index 2a34ae9..fb41e82 100644
@@ -694,16 +694,69 @@ vfio_type1_dma_map(int vfio_container_fd)
 	return rte_memseg_walk(type1_map, &vfio_container_fd);
 }
 
+struct spapr_walk_param {
+	uint64_t window_size;
+	uint64_t hugepage_sz;
+};
 static int
-vfio_spapr_dma_map(int vfio_container_fd)
+spapr_window_size(const struct rte_memseg *ms, void *arg)
 {
-	const struct rte_memseg *ms = rte_eal_get_physmem_layout();
-	int i, ret;
+	struct spapr_walk_param *param = arg;
+	uint64_t max = ms->iova + ms->len;
+
+	if (max > param->window_size) {
+		param->hugepage_sz = ms->hugepage_sz;
+		param->window_size = max;
+	}
 
+	return 0;
+}
+
+static int
+spapr_map(const struct rte_memseg *ms, void *arg)
+{
+	struct vfio_iommu_type1_dma_map dma_map;
 	struct vfio_iommu_spapr_register_memory reg = {
 		.argsz = sizeof(reg),
 		.flags = 0
 	};
+	int *vfio_container_fd = arg;
+	int ret;
+
+	reg.vaddr = (uintptr_t) ms->addr;
+	reg.size = ms->len;
+	ret = ioctl(*vfio_container_fd,
+		VFIO_IOMMU_SPAPR_REGISTER_MEMORY, &reg);
+	if (ret) {
+		RTE_LOG(ERR, EAL, "  cannot register vaddr for IOMMU, error %i (%s)\n",
+				errno, strerror(errno));
+		return -1;
+	}
+
+	memset(&dma_map, 0, sizeof(dma_map));
+	dma_map.argsz = sizeof(struct vfio_iommu_type1_dma_map);
+	dma_map.vaddr = ms->addr_64;
+	dma_map.size = ms->len;
+	dma_map.iova = ms->iova;
+	dma_map.flags = VFIO_DMA_MAP_FLAG_READ |
+			 VFIO_DMA_MAP_FLAG_WRITE;
+
+	ret = ioctl(*vfio_container_fd, VFIO_IOMMU_MAP_DMA, &dma_map);
+
+	if (ret) {
+		RTE_LOG(ERR, EAL, "  cannot set up DMA remapping, error %i (%s)\n",
+				errno, strerror(errno));
+		return -1;
+	}
+
+	return 0;
+}
+
+static int
+vfio_spapr_dma_map(int vfio_container_fd)
+{
+	struct spapr_walk_param param;
+	int ret;
 	struct vfio_iommu_spapr_tce_info info = {
 		.argsz = sizeof(info),
 	};

@@ -714,6 +767,8 @@ vfio_spapr_dma_map(int vfio_container_fd)
 		.argsz = sizeof(remove),
 	};
 
+	memset(&param, 0, sizeof(param));
+
 	/* query spapr iommu info */
 	ret = ioctl(vfio_container_fd, VFIO_IOMMU_SPAPR_TCE_GET_INFO, &info);
 	if (ret) {

@@ -732,17 +787,11 @@ vfio_spapr_dma_map(int vfio_container_fd)
 	}
 
 	/* create DMA window from 0 to max(phys_addr + len) */
-	for (i = 0; i < RTE_MAX_MEMSEG; i++) {
-		if (ms[i].addr == NULL)
-			break;
-
-		create.window_size = RTE_MAX(create.window_size,
-				ms[i].iova + ms[i].len);
-	}
+	rte_memseg_walk(spapr_window_size, &param);
 
 	/* sPAPR requires window size to be a power of 2 */
-	create.window_size = rte_align64pow2(create.window_size);
-	create.page_shift = __builtin_ctzll(ms->hugepage_sz);
+	create.window_size = rte_align64pow2(param.window_size);
+	create.page_shift = __builtin_ctzll(param.hugepage_sz);
 	create.levels = 1;
 
 	ret = ioctl(vfio_container_fd, VFIO_IOMMU_SPAPR_TCE_CREATE, &create);

@@ -758,39 +807,8 @@ vfio_spapr_dma_map(int vfio_container_fd)
 	}
 
 	/* map all DPDK segments for DMA. use 1:1 PA to IOVA mapping */
-	for (i = 0; i < RTE_MAX_MEMSEG; i++) {
-		struct vfio_iommu_type1_dma_map dma_map;
-
-		if (ms[i].addr == NULL)
-			break;
-
-		reg.vaddr = (uintptr_t) ms[i].addr;
-		reg.size = ms[i].len;
-		ret = ioctl(vfio_container_fd,
-			VFIO_IOMMU_SPAPR_REGISTER_MEMORY, &reg);
-		if (ret) {
-			RTE_LOG(ERR, EAL, "  cannot register vaddr for IOMMU, "
-				"error %i (%s)\n", errno, strerror(errno));
-			return -1;
-		}
-
-		memset(&dma_map, 0, sizeof(dma_map));
-		dma_map.argsz = sizeof(struct vfio_iommu_type1_dma_map);
-		dma_map.vaddr = ms[i].addr_64;
-		dma_map.size = ms[i].len;
-		dma_map.iova = ms[i].iova;
-		dma_map.flags = VFIO_DMA_MAP_FLAG_READ |
-				 VFIO_DMA_MAP_FLAG_WRITE;
-
-		ret = ioctl(vfio_container_fd, VFIO_IOMMU_MAP_DMA, &dma_map);
-
-		if (ret) {
-			RTE_LOG(ERR, EAL, "  cannot set up DMA remapping, "
-				"error %i (%s)\n", errno, strerror(errno));
-			return -1;
-		}
-
-	}
+	if (rte_memseg_walk(spapr_map, &vfio_container_fd) < 0)
+		return -1;
 
 	return 0;
 }

index 4f588c7..4b528b0 100644
@@ -242,6 +242,43 @@ rte_memseg_walk(rte_memseg_walk_t func, void *arg)
 	return 0;
 }
 
+int __rte_experimental
+rte_memseg_contig_walk(rte_memseg_contig_walk_t func, void *arg)
+{
+	struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
+	int i, j, ret;
+
+	for (i = 0; i < RTE_MAX_MEMSEG; i++) {
+		const struct rte_memseg *ms = &mcfg->memseg[i];
+		size_t total_len;
+		void *end_addr;
+
+		if (ms->addr == NULL)
+			continue;
+
+		end_addr = RTE_PTR_ADD(ms->addr, ms->len);
+
+		/* check how many more segments are contiguous to this one */
+		for (j = i + 1; j < RTE_MAX_MEMSEG; j++) {
+			const struct rte_memseg *next = &mcfg->memseg[j];
+
+			if (next->addr != end_addr)
+				break;
+
+			end_addr = RTE_PTR_ADD(next->addr, next->len);
+			i++;
+		}
+		total_len = RTE_PTR_DIFF(end_addr, ms->addr);
+
+		ret = func(ms, total_len, arg);
+		if (ret < 0)
+			return -1;
+		if (ret > 0)
+			return 1;
+	}
+	return 0;
+}
+
 /* init memory subsystem */
 int
 rte_eal_memory_init(void)

index 93eadaa..45d067f 100644
@@ -140,6 +140,18 @@ rte_iova_t rte_mem_virt2iova(const void *virt);
 typedef int (*rte_memseg_walk_t)(const struct rte_memseg *ms, void *arg);
 
 /**
+ * Memseg contig walk function prototype. This will trigger a callback on every
+ * VA-contiguous are starting at memseg ``ms``, so total valid VA space at each
+ * callback call will be [``ms->addr``, ``ms->addr + len``).
+ *
+ * Returning 0 will continue walk
+ * Returning 1 will stop the walk
+ * Returning -1 will stop the walk and report error
+ */
+typedef int (*rte_memseg_contig_walk_t)(const struct rte_memseg *ms,
+		size_t len, void *arg);
+
+/**
  * Walk list of all memsegs.
  *
  * @param func

@@ -155,6 +167,21 @@ int __rte_experimental
 rte_memseg_walk(rte_memseg_walk_t func, void *arg);
 
 /**
+ * Walk each VA-contiguous area.
+ *
+ * @param func
+ *   Iterator function
+ * @param arg
+ *   Argument passed to iterator
+ * @return
+ *   0 if walked over the entire list
+ *   1 if stopped by the user
+ *   -1 if user function reported error
+ */
+int __rte_experimental
+rte_memseg_contig_walk(rte_memseg_contig_walk_t func, void *arg);
+
+/**
  * Get the layout of the available physical memory.
  *
  * It can be useful for an application to have the full physical

index 1711ead..93d7efe 100644
@@ -70,6 +70,32 @@ static uint64_t vhost_req_user_to_kernel[] = {
 	[VHOST_USER_SET_MEM_TABLE] = VHOST_SET_MEM_TABLE,
 };
 
+struct walk_arg {
+	struct vhost_memory_kernel *vm;
+	uint32_t region_nr;
+};
+static int
+add_memory_region(const struct rte_memseg *ms, size_t len, void *arg)
+{
+	struct walk_arg *wa = arg;
+	struct vhost_memory_region *mr;
+	void *start_addr;
+
+	if (wa->region_nr >= max_regions)
+		return -1;
+
+	mr = &wa->vm->regions[wa->region_nr++];
+	start_addr = ms->addr;
+
+	mr->guest_phys_addr = (uint64_t)(uintptr_t)start_addr;
+	mr->userspace_addr = (uint64_t)(uintptr_t)start_addr;
+	mr->memory_size = len;
+	mr->mmap_offset = 0;
+
+	return 0;
+}
+
+
 /* By default, vhost kernel module allows 64 regions, but DPDK allows
  * 256 segments. As a relief, below function merges those virtually
  * adjacent memsegs into one region.

@@ -77,63 +103,24 @@ static uint64_t vhost_req_user_to_kernel[] = {
 static struct vhost_memory_kernel *
 prepare_vhost_memory_kernel(void)
 {
-	uint32_t i, j, k = 0;
-	struct rte_memseg *seg;
-	struct vhost_memory_region *mr;
 	struct vhost_memory_kernel *vm;
+	struct walk_arg wa;
 
 	vm = malloc(sizeof(struct vhost_memory_kernel) +
-		    max_regions *
-		    sizeof(struct vhost_memory_region));
+			max_regions *
+			sizeof(struct vhost_memory_region));
 	if (!vm)
 		return NULL;
 
-	for (i = 0; i < RTE_MAX_MEMSEG; ++i) {
-		seg = &rte_eal_get_configuration()->mem_config->memseg[i];
-		if (!seg->addr)
-			break;
-
-		int new_region = 1;
-
-		for (j = 0; j < k; ++j) {
-			mr = &vm->regions[j];
+	wa.region_nr = 0;
+	wa.vm = vm;
 
-			if (mr->userspace_addr + mr->memory_size ==
-			    (uint64_t)(uintptr_t)seg->addr) {
-				mr->memory_size += seg->len;
-				new_region = 0;
-				break;
-			}
-
-			if ((uint64_t)(uintptr_t)seg->addr + seg->len ==
-			    mr->userspace_addr) {
-				mr->guest_phys_addr =
-					(uint64_t)(uintptr_t)seg->addr;
-				mr->userspace_addr =
-					(uint64_t)(uintptr_t)seg->addr;
-				mr->memory_size += seg->len;
-				new_region = 0;
-				break;
-			}
-		}
-
-		if (new_region == 0)
-			continue;
-
-		mr = &vm->regions[k++];
-		/* use vaddr here! */
-		mr->guest_phys_addr = (uint64_t)(uintptr_t)seg->addr;
-		mr->userspace_addr = (uint64_t)(uintptr_t)seg->addr;
-		mr->memory_size = seg->len;
-		mr->mmap_offset = 0;
-
-		if (k >= max_regions) {
-			free(vm);
-			return NULL;
-		}
+	if (rte_memseg_contig_walk(add_memory_region, &wa) < 0) {
+		free(vm);
+		return NULL;
 	}
 
-	vm->nregions = k;
+	vm->nregions = wa.region_nr;
 	vm->padding = 0;
 	return vm;
 }

index 4b528b0..ea3c5a7 100644
@@ -131,6 +131,36 @@ rte_eal_get_physmem_layout(void)
 	return rte_eal_get_configuration()->mem_config->memseg;
 }
 
+struct virtiova {
+	rte_iova_t iova;
+	void *virt;
+};
+static int
+find_virt(const struct rte_memseg *ms, void *arg)
+{
+	struct virtiova *vi = arg;
+	if (vi->iova >= ms->iova && vi->iova < (ms->iova + ms->len)) {
+		size_t offset = vi->iova - ms->iova;
+		vi->virt = RTE_PTR_ADD(ms->addr, offset);
+		/* stop the walk */
+		return 1;
+	}
+	return 0;
+}
+
+__rte_experimental void *
+rte_mem_iova2virt(rte_iova_t iova)
+{
+	struct virtiova vi;
+
+	memset(&vi, 0, sizeof(vi));
+
+	vi.iova = iova;
+	rte_memseg_walk(find_virt, &vi);
+
+	return vi.virt;
+}
+
 static int
 physmem_size(const struct rte_memseg *ms, void *arg)
 {

index 45d067f..5c60b91 100644
@@ -131,6 +131,18 @@ phys_addr_t rte_mem_virt2phy(const void *virt);
 rte_iova_t rte_mem_virt2iova(const void *virt);
 
 /**
+ * Get virtual memory address corresponding to iova address.
+ *
+ * @param iova
+ *   The iova address.
+ * @return
+ *   Virtual address corresponding to iova address (or NULL if address does not
+ *   exist within DPDK memory map).
+ */
+__rte_experimental void *
+rte_mem_iova2virt(rte_iova_t iova);
+
+/**
  * Memseg walk function prototype.
  *
  * Returning 0 will continue walk

index 718701b..89aeac2 100644
@@ -98,17 +98,7 @@ struct dpaa_portal {
 /* TODO - this is costly, need to write a fast coversion routine */
 static inline void *rte_dpaa_mem_ptov(phys_addr_t paddr)
 {
-	const struct rte_memseg *memseg = rte_eal_get_physmem_layout();
-	int i;
-
-	for (i = 0; i < RTE_MAX_MEMSEG && memseg[i].addr != NULL; i++) {
-		if (paddr >= memseg[i].iova && paddr <
-			memseg[i].iova + memseg[i].len)
-			return (uint8_t *)(memseg[i].addr) +
-			       (paddr - memseg[i].iova);
-	}
-
-	return NULL;
+	return rte_mem_iova2virt(paddr);
 }
 
 /**

index 4a19d42..d38fc49 100644
@@ -260,21 +260,10 @@ static void *dpaa2_mem_ptov(phys_addr_t paddr) __attribute__((unused));
 /* todo - this is costly, need to write a fast coversion routine */
 static void *dpaa2_mem_ptov(phys_addr_t paddr)
 {
-	const struct rte_memseg *memseg;
-	int i;
-
 	if (dpaa2_virt_mode)
 		return (void *)(size_t)paddr;
 
-	memseg = rte_eal_get_physmem_layout();
-
-	for (i = 0; i < RTE_MAX_MEMSEG && memseg[i].addr_64 != 0; i++) {
-		if (paddr >= memseg[i].iova &&
-		    paddr < memseg[i].iova + memseg[i].len)
-			return (void *)(size_t)(memseg[i].addr_64
-				+ (paddr - memseg[i].iova));
-	}
-	return NULL;
+	return rte_mem_iova2virt(paddr);
 }
 
 static phys_addr_t dpaa2_mem_vtop(uint64_t vaddr) __attribute__((unused));

index c5191ce..b04510f 100644
@@ -120,16 +120,7 @@ dpaa_mem_vtop_ctx(struct dpaa_sec_op_ctx *ctx, void *vaddr)
 static inline void *
 dpaa_mem_ptov(rte_iova_t paddr)
 {
-	const struct rte_memseg *memseg = rte_eal_get_physmem_layout();
-	int i;
-
-	for (i = 0; i < RTE_MAX_MEMSEG && memseg[i].addr_64 != 0; i++) {
-		if (paddr >= memseg[i].iova &&
-		    paddr < memseg[i].iova + memseg[i].len)
-			return (void *)(size_t)(memseg[i].addr_64 +
-					(paddr - memseg[i].iova));
-	}
-	return NULL;
+	return rte_mem_iova2virt(paddr);
 }
 
 static void

index ea3c5a7..fd78d2f 100644
@@ -161,6 +161,43 @@ rte_mem_iova2virt(rte_iova_t iova)
 	return vi.virt;
 }
 
+struct virtms {
+	const void *virt;
+	struct rte_memseg *ms;
+};
+static int
+find_memseg(const struct rte_memseg *ms, void *arg)
+{
+	struct virtms *vm = arg;
+
+	if (arg >= ms->addr && arg < RTE_PTR_ADD(ms->addr, ms->len)) {
+		struct rte_memseg *memseg, *found_ms;
+		int idx;
+
+		memseg = rte_eal_get_configuration()->mem_config->memseg;
+		idx = ms - memseg;
+		found_ms = &memseg[idx];
+
+		vm->ms = found_ms;
+		return 1;
+	}
+	return 0;
+}
+
+__rte_experimental struct rte_memseg *
+rte_mem_virt2memseg(const void *addr)
+{
+	struct virtms vm;
+
+	memset(&vm, 0, sizeof(vm));
+
+	vm.virt = addr;
+
+	rte_memseg_walk(find_memseg, &vm);
+
+	return vm.ms;
+}
+
 static int
 physmem_size(const struct rte_memseg *ms, void *arg)
 {

index 5c60b91..b3d7e61 100644
@@ -143,6 +143,17 @@ __rte_experimental void *
 rte_mem_iova2virt(rte_iova_t iova);
 
 /**
+ * Get memseg to which a particular virtual address belongs.
+ *
+ * @param virt
+ *   The virtual address.
+ * @return
+ *   Memseg pointer on success, or NULL on error.
+ */
+__rte_experimental struct rte_memseg *
+rte_mem_virt2memseg(const void *virt);
+
+/**
  * Memseg walk function prototype.
  *
  * Returning 0 will continue walk