Memory Hotplug for DPDK

index 852f3bb..042881b 100644
@@ -2,10 +2,12 @@
  * Copyright(c) 2010-2014 Intel Corporation
  */
 
+#include <errno.h>
 #include <stdio.h>
 #include <stdint.h>
 #include <stdlib.h>
 #include <stdarg.h>
+#include <string.h>
 #include <unistd.h>
 #include <inttypes.h>
 #include <sys/mman.h>

@@ -14,12 +16,111 @@
 #include <rte_memory.h>
 #include <rte_eal.h>
 #include <rte_eal_memconfig.h>
+#include <rte_errno.h>
 #include <rte_log.h>
 
 #include "eal_private.h"
 #include "eal_internal_cfg.h"
 
 /*
+ * Try to mmap *size bytes in /dev/zero. If it is successful, return the
+ * pointer to the mmap'd area and keep *size unmodified. Else, retry
+ * with a smaller zone: decrease *size by hugepage_sz until it reaches
+ * 0. In this case, return NULL. Note: this function returns an address
+ * which is a multiple of hugepage size.
+ */
+
+static uint64_t baseaddr_offset;
+static uint64_t system_page_sz;
+
+void *
+eal_get_virtual_area(void *requested_addr, uint64_t *size,
+		uint64_t page_sz, int flags, int mmap_flags)
+{
+	bool addr_is_hint, allow_shrink, unmap, no_align;
+	uint64_t map_sz;
+	void *mapped_addr, *aligned_addr;
+
+	if (system_page_sz == 0)
+		system_page_sz = sysconf(_SC_PAGESIZE);
+
+	mmap_flags |= MAP_PRIVATE | MAP_ANONYMOUS;
+
+	RTE_LOG(DEBUG, EAL, "Ask a virtual area of 0x%zx bytes\n", *size);
+
+	addr_is_hint = (flags & EAL_VIRTUAL_AREA_ADDR_IS_HINT) > 0;
+	allow_shrink = (flags & EAL_VIRTUAL_AREA_ALLOW_SHRINK) > 0;
+	unmap = (flags & EAL_VIRTUAL_AREA_UNMAP) > 0;
+
+	if (requested_addr == NULL && internal_config.base_virtaddr != 0) {
+		requested_addr = (void *) (internal_config.base_virtaddr +
+				baseaddr_offset);
+		requested_addr = RTE_PTR_ALIGN(requested_addr, page_sz);
+		addr_is_hint = true;
+	}
+
+	/* if requested address is not aligned by page size, or if requested
+	 * address is NULL, add page size to requested length as we may get an
+	 * address that's aligned by system page size, which can be smaller than
+	 * our requested page size. additionally, we shouldn't try to align if
+	 * system page size is the same as requested page size.
+	 */
+	no_align = (requested_addr != NULL &&
+		((uintptr_t)requested_addr & (page_sz - 1)) == 0) ||
+		page_sz == system_page_sz;
+
+	do {
+		map_sz = no_align ? *size : *size + page_sz;
+
+		mapped_addr = mmap(requested_addr, map_sz, PROT_READ,
+				mmap_flags, -1, 0);
+		if (mapped_addr == MAP_FAILED && allow_shrink)
+			*size -= page_sz;
+	} while (allow_shrink && mapped_addr == MAP_FAILED && *size > 0);
+
+	/* align resulting address - if map failed, we will ignore the value
+	 * anyway, so no need to add additional checks.
+	 */
+	aligned_addr = no_align ? mapped_addr :
+			RTE_PTR_ALIGN(mapped_addr, page_sz);
+
+	if (*size == 0) {
+		RTE_LOG(ERR, EAL, "Cannot get a virtual area of any size: %s\n",
+			strerror(errno));
+		rte_errno = errno;
+		return NULL;
+	} else if (mapped_addr == MAP_FAILED) {
+		RTE_LOG(ERR, EAL, "Cannot get a virtual area: %s\n",
+			strerror(errno));
+		/* pass errno up the call chain */
+		rte_errno = errno;
+		return NULL;
+	} else if (requested_addr != NULL && !addr_is_hint &&
+			aligned_addr != requested_addr) {
+		RTE_LOG(ERR, EAL, "Cannot get a virtual area at requested address: %p (got %p)\n",
+			requested_addr, aligned_addr);
+		munmap(mapped_addr, map_sz);
+		rte_errno = EADDRNOTAVAIL;
+		return NULL;
+	} else if (requested_addr != NULL && addr_is_hint &&
+			aligned_addr != requested_addr) {
+		RTE_LOG(WARNING, EAL, "WARNING! Base virtual address hint (%p != %p) not respected!\n",
+			requested_addr, aligned_addr);
+		RTE_LOG(WARNING, EAL, "   This may cause issues with mapping memory into secondary processes\n");
+	}
+
+	if (unmap)
+		munmap(mapped_addr, map_sz);
+
+	RTE_LOG(DEBUG, EAL, "Virtual area found at %p (size = 0x%zx)\n",
+		aligned_addr, *size);
+
+	baseaddr_offset += *size;
+
+	return aligned_addr;
+}
+
+/*
  * Return a pointer to a read-only table of struct rte_physmem_desc
  * elements, containing the layout of all addressable physical
  * memory. The last element of the table contains a NULL address.

index 0b28770..96cebb7 100644
@@ -127,6 +127,39 @@ int rte_eal_alarm_init(void);
 int rte_eal_check_module(const char *module_name);
 
 /**
+ * Get virtual area of specified size from the OS.
+ *
+ * This function is private to the EAL.
+ *
+ * @param requested_addr
+ *   Address where to request address space.
+ * @param size
+ *   Size of requested area.
+ * @param page_sz
+ *   Page size on which to align requested virtual area.
+ * @param flags
+ *   EAL_VIRTUAL_AREA_* flags.
+ * @param mmap_flags
+ *   Extra flags passed directly to mmap().
+ *
+ * @return
+ *   Virtual area address if successful.
+ *   NULL if unsuccessful.
+ */
+
+#define EAL_VIRTUAL_AREA_ADDR_IS_HINT (1 << 0)
+/**< don't fail if cannot get exact requested address. */
+#define EAL_VIRTUAL_AREA_ALLOW_SHRINK (1 << 1)
+/**< try getting smaller sized (decrement by page size) virtual areas if cannot
+ * get area of requested size.
+ */
+#define EAL_VIRTUAL_AREA_UNMAP (1 << 2)
+/**< immediately unmap reserved virtual area. */
+void *
+eal_get_virtual_area(void *requested_addr, uint64_t *size,
+		uint64_t page_sz, int flags, int mmap_flags);
+
+/**
  * Get cpu core_id.
  *
  * This function is private to the EAL.

204 lines not on this page · raw message

index ba99ed9..9ec4b62 100644
@@ -13,12 +13,18 @@
 /* Number of free lists per heap, grouped by size. */
 #define RTE_HEAP_NUM_FREELISTS  13
 
+/* dummy definition, for pointers */
+struct malloc_elem;
+
 /**
  * Structure to hold malloc heap
  */
 struct malloc_heap {
 	rte_spinlock_t lock;
 	LIST_HEAD(, malloc_elem) free_head[RTE_HEAP_NUM_FREELISTS];
+	struct malloc_elem *first;
+	struct malloc_elem *last;
+
 	unsigned alloc_count;
 	size_t total_size;
 } __rte_cache_aligned;

271 lines not on this page · raw message

index f4c1c7a..238e451 100644
@@ -18,8 +18,12 @@ enum elem_state {
 
 struct malloc_elem {
 	struct malloc_heap *heap;
-	struct malloc_elem *volatile prev;      /* points to prev elem in memseg */
-	LIST_ENTRY(malloc_elem) free_list;      /* list of free elements in heap */
+	struct malloc_elem *volatile prev;
+	/**< points to prev elem in memseg */
+	struct malloc_elem *volatile next;
+	/**< points to next elem in memseg */
+	LIST_ENTRY(malloc_elem) free_list;
+	/**< list of free elements in heap */
 	const struct rte_memseg *ms;
 	volatile enum elem_state state;
 	uint32_t pad;

@@ -110,12 +114,8 @@ malloc_elem_init(struct malloc_elem *elem,
 		const struct rte_memseg *ms,
 		size_t size);
 
-/*
- * initialise a dummy malloc_elem header for the end-of-memseg marker
- */
 void
-malloc_elem_mkend(struct malloc_elem *elem,
-		struct malloc_elem *prev_free);
+malloc_elem_insert(struct malloc_elem *elem);
 
 /*
  * return true if the current malloc_elem can hold a block of data

index 7d8d70a..9c95166 100644
@@ -70,15 +70,11 @@ check_hugepage_sz(unsigned flags, uint64_t hugepage_sz)
 static void
 malloc_heap_add_memseg(struct malloc_heap *heap, struct rte_memseg *ms)
 {
-	/* allocate the memory block headers, one at end, one at start */
 	struct malloc_elem *start_elem = (struct malloc_elem *)ms->addr;
-	struct malloc_elem *end_elem = RTE_PTR_ADD(ms->addr,
-			ms->len - MALLOC_ELEM_OVERHEAD);
-	end_elem = RTE_PTR_ALIGN_FLOOR(end_elem, RTE_CACHE_LINE_SIZE);
-	const size_t elem_size = (uintptr_t)end_elem - (uintptr_t)start_elem;
+	const size_t elem_size = ms->len - MALLOC_ELEM_OVERHEAD;
 
 	malloc_elem_init(start_elem, heap, ms, elem_size);
-	malloc_elem_mkend(end_elem, start_elem);
+	malloc_elem_insert(start_elem);
 	malloc_elem_free_list_insert(start_elem);
 
 	heap->total_size += elem_size;

index cf0b726..6bfdc02 100644
@@ -137,6 +137,8 @@ static void cmd_dump_parsed(void *parsed_result,
 		rte_log_dump(stdout);
 	else if (!strcmp(res->dump, "dump_malloc_stats"))
 		rte_malloc_dump_stats(stdout, NULL);
+	else if (!strcmp(res->dump, "dump_malloc_heaps"))
+		rte_malloc_dump_heaps(stdout);
 }
 
 cmdline_parse_token_string_t cmd_dump_dump =

@@ -147,6 +149,7 @@ cmdline_parse_token_string_t cmd_dump_dump =
 				 "dump_ring#"
 				 "dump_mempool#"
 				 "dump_malloc_stats#"
+				 "dump_malloc_heaps#"
 				 "dump_devargs#"
 				 "dump_log_types");
 

index e02ed88..2291ee1 100644
@@ -325,8 +325,8 @@ join_elem(struct malloc_elem *elem1, struct malloc_elem *elem2)
 	elem1->next = next;
 }
 
-static struct malloc_elem *
-elem_join_adjacent_free(struct malloc_elem *elem)
+struct malloc_elem *
+malloc_elem_join_adjacent_free(struct malloc_elem *elem)
 {
 	/*
 	 * check if next element exists, is adjacent and is free, if so join

@@ -388,7 +388,7 @@ malloc_elem_free(struct malloc_elem *elem)
 	ptr = RTE_PTR_ADD(elem, sizeof(*elem));
 	data_len = elem->size - MALLOC_ELEM_OVERHEAD;
 
-	elem = elem_join_adjacent_free(elem);
+	elem = malloc_elem_join_adjacent_free(elem);
 
 	malloc_elem_free_list_insert(elem);
 

index 40e8eb5..99921d2 100644
@@ -141,6 +141,9 @@ malloc_elem_alloc(struct malloc_elem *elem, size_t size,
 int
 malloc_elem_free(struct malloc_elem *elem);
 
+struct malloc_elem *
+malloc_elem_join_adjacent_free(struct malloc_elem *elem);
+
 /*
  * attempt to resize a malloc_elem by expanding into any free space
  * immediately after it in memory.

index 0cadc8a..ea041e2 100644
@@ -243,10 +243,6 @@ join_elem(struct malloc_elem *elem1, struct malloc_elem *elem2)
 int
 malloc_elem_free(struct malloc_elem *elem)
 {
-	if (!malloc_elem_cookies_ok(elem) || elem->state != ELEM_BUSY)
-		return -1;
-
-	rte_spinlock_lock(&(elem->heap->lock));
 	size_t sz = elem->size - sizeof(*elem) - MALLOC_ELEM_TRAILER_LEN;
 	uint8_t *ptr = (uint8_t *)&elem[1];
 	struct malloc_elem *next = RTE_PTR_ADD(elem, elem->size);

@@ -274,8 +270,6 @@ malloc_elem_free(struct malloc_elem *elem)
 
 	memset(ptr, 0, sz);
 
-	rte_spinlock_unlock(&(elem->heap->lock));
-
 	return 0;
 }
 

@@ -292,11 +286,10 @@ malloc_elem_resize(struct malloc_elem *elem, size_t size)
 		return 0;
 
 	struct malloc_elem *next = RTE_PTR_ADD(elem, elem->size);
-	rte_spinlock_lock(&elem->heap->lock);
 	if (next ->state != ELEM_FREE)
-		goto err_return;
+		return -1;
 	if (elem->size + next->size < new_size)
-		goto err_return;
+		return -1;
 
 	/* we now know the element fits, so remove from free list,
 	 * join the two

@@ -311,10 +304,5 @@ malloc_elem_resize(struct malloc_elem *elem, size_t size)
 		split_elem(elem, split_pt);
 		malloc_elem_free_list_insert(split_pt);
 	}
-	rte_spinlock_unlock(&elem->heap->lock);
 	return 0;
-
-err_return:
-	rte_spinlock_unlock(&elem->heap->lock);
-	return -1;
 }

index 7aafc88..7d8d70a 100644
@@ -145,6 +145,44 @@ malloc_heap_alloc(struct malloc_heap *heap,
 	return elem == NULL ? NULL : (void *)(&elem[1]);
 }
 
+int
+malloc_heap_free(struct malloc_elem *elem)
+{
+	struct malloc_heap *heap;
+	int ret;
+
+	if (!malloc_elem_cookies_ok(elem) || elem->state != ELEM_BUSY)
+		return -1;
+
+	/* elem may be merged with previous element, so keep heap address */
+	heap = elem->heap;
+
+	rte_spinlock_lock(&(heap->lock));
+
+	ret = malloc_elem_free(elem);
+
+	rte_spinlock_unlock(&(heap->lock));
+
+	return ret;
+}
+
+int
+malloc_heap_resize(struct malloc_elem *elem, size_t size)
+{
+	int ret;
+
+	if (!malloc_elem_cookies_ok(elem) || elem->state != ELEM_BUSY)
+		return -1;
+
+	rte_spinlock_lock(&(elem->heap->lock));
+
+	ret = malloc_elem_resize(elem, size);
+
+	rte_spinlock_unlock(&(elem->heap->lock));
+
+	return ret;
+}
+
 /*
  * Function to retrieve data for heap on given socket
  */

index e0defa7..ab0005c 100644
@@ -28,6 +28,12 @@ malloc_heap_alloc(struct malloc_heap *heap,	const char *type, size_t size,
 		unsigned flags, size_t align, size_t bound);
 
 int
+malloc_heap_free(struct malloc_elem *elem);
+
+int
+malloc_heap_resize(struct malloc_elem *elem, size_t size);
+
+int
 malloc_heap_get_stats(struct malloc_heap *heap,
 		struct rte_malloc_socket_stats *socket_stats);
 

index e0e0d0b..970813e 100644
@@ -29,7 +29,7 @@
 void rte_free(void *addr)
 {
 	if (addr == NULL) return;
-	if (malloc_elem_free(malloc_elem_from_data(addr)) < 0)
+	if (malloc_heap_free(malloc_elem_from_data(addr)) < 0)
 		rte_panic("Fatal error: Invalid memory\n");
 }
 

@@ -140,7 +140,7 @@ rte_realloc(void *ptr, size_t size, unsigned align)
 	size = RTE_CACHE_LINE_ROUNDUP(size), align = RTE_CACHE_LINE_ROUNDUP(align);
 	/* check alignment matches first, and if ok, see if we can resize block */
 	if (RTE_PTR_ALIGN(ptr,align) == ptr &&
-			malloc_elem_resize(elem, size) == 0)
+			malloc_heap_resize(elem, size) == 0)
 		return ptr;
 
 	/* either alignment is off, or we have no room to expand,

index 2291ee1..008f5a3 100644
@@ -245,8 +245,8 @@ malloc_elem_free_list_insert(struct malloc_elem *elem)
 /*
  * Remove the specified element from its heap's free list.
  */
-static void
-elem_free_list_remove(struct malloc_elem *elem)
+void
+malloc_elem_free_list_remove(struct malloc_elem *elem)
 {
 	LIST_REMOVE(elem, free_list);
 }

@@ -266,7 +266,7 @@ malloc_elem_alloc(struct malloc_elem *elem, size_t size, unsigned align,
 	const size_t trailer_size = elem->size - old_elem_size - size -
 		MALLOC_ELEM_OVERHEAD;
 
-	elem_free_list_remove(elem);
+	malloc_elem_free_list_remove(elem);
 
 	if (trailer_size > MALLOC_ELEM_OVERHEAD + MIN_DATA_SIZE) {
 		/* split it, too much free space after elem */

@@ -340,7 +340,7 @@ malloc_elem_join_adjacent_free(struct malloc_elem *elem)
 		erase = RTE_PTR_SUB(elem->next, MALLOC_ELEM_TRAILER_LEN);
 
 		/* remove from free list, join to this one */
-		elem_free_list_remove(elem->next);
+		malloc_elem_free_list_remove(elem->next);
 		join_elem(elem, elem->next);
 
 		/* erase header and trailer */

@@ -360,7 +360,7 @@ malloc_elem_join_adjacent_free(struct malloc_elem *elem)
 		erase = RTE_PTR_SUB(elem, MALLOC_ELEM_TRAILER_LEN);
 
 		/* remove from free list, join to this one */
-		elem_free_list_remove(elem->prev);
+		malloc_elem_free_list_remove(elem->prev);
 
 		new_elem = elem->prev;
 		join_elem(new_elem, elem);

@@ -423,7 +423,7 @@ malloc_elem_resize(struct malloc_elem *elem, size_t size)
 	/* we now know the element fits, so remove from free list,
 	 * join the two
 	 */
-	elem_free_list_remove(elem->next);
+	malloc_elem_free_list_remove(elem->next);
 	join_elem(elem, elem->next);
 
 	if (elem->size - new_size >= MIN_DATA_SIZE + MALLOC_ELEM_OVERHEAD) {

index 99921d2..46e2383 100644
@@ -151,6 +151,9 @@ malloc_elem_join_adjacent_free(struct malloc_elem *elem);
 int
 malloc_elem_resize(struct malloc_elem *elem, size_t size);
 
+void
+malloc_elem_free_list_remove(struct malloc_elem *elem);
+
 /*
  * dump contents of malloc elem to a file.
  */

index 8bbf771..706b6d5 100644
@@ -30,6 +30,7 @@
 #include "eal_filesystem.h"
 
 static const char sys_dir_path[] = "/sys/kernel/mm/hugepages";
+static const char sys_pages_numa_dir_path[] = "/sys/devices/system/node";
 
 /* this function is only called from eal_hugepage_info_init which itself
  * is only called from a primary process */

@@ -70,6 +71,45 @@ get_num_hugepages(const char *subdir)
 	return num_pages;
 }
 
+static uint32_t
+get_num_hugepages_on_node(const char *subdir, unsigned int socket)
+{
+	char path[PATH_MAX], socketpath[PATH_MAX];
+	DIR *socketdir;
+	unsigned long num_pages = 0;
+	const char *nr_hp_file = "free_hugepages";
+
+	snprintf(socketpath, sizeof(socketpath), "%s/node%u/hugepages",
+		sys_pages_numa_dir_path, socket);
+
+	socketdir = opendir(socketpath);
+	if (socketdir) {
+		/* Keep calm and carry on */
+		closedir(socketdir);
+	} else {
+		/* Can't find socket dir, so ignore it */
+		return 0;
+	}
+
+	snprintf(path, sizeof(path), "%s/%s/%s",
+			socketpath, subdir, nr_hp_file);
+	if (eal_parse_sysfs_value(path, &num_pages) < 0)
+		return 0;
+
+	if (num_pages == 0)
+		RTE_LOG(WARNING, EAL, "No free hugepages reported in %s\n",
+				subdir);
+
+	/*
+	 * we want to return a uint32_t and more than this looks suspicious
+	 * anyway ...
+	 */
+	if (num_pages > UINT32_MAX)
+		num_pages = UINT32_MAX;
+
+	return num_pages;
+}
+
 static uint64_t
 get_default_hp_size(void)
 {

@@ -248,7 +288,7 @@ eal_hugepage_info_init(void)
 {
 	const char dirent_start_text[] = "hugepages-";
 	const size_t dirent_start_len = sizeof(dirent_start_text) - 1;
-	unsigned i, num_sizes = 0;
+	unsigned int i, total_pages, num_sizes = 0;
 	DIR *dir;
 	struct dirent *dirent;
 

@@ -302,9 +342,27 @@ eal_hugepage_info_init(void)
 		if (clear_hugedir(hpi->hugedir) == -1)
 			break;
 
-		/* for now, put all pages into socket 0,
-		 * later they will be sorted */
-		hpi->num_pages[0] = get_num_hugepages(dirent->d_name);
+		/*
+		 * first, try to put all hugepages into relevant sockets, but
+		 * if first attempts fails, fall back to collecting all pages
+		 * in one socket and sorting them later
+		 */
+		total_pages = 0;
+		/* we also don't want to do this for legacy init */
+		if (!internal_config.legacy_mem)
+			for (i = 0; i < rte_num_sockets(); i++) {
+				unsigned int num_pages =
+						get_num_hugepages_on_node(
+							dirent->d_name, i);
+				hpi->num_pages[i] = num_pages;
+				total_pages += num_pages;
+			}
+		/*
+		 * we failed to sort memory from the get go, so fall
+		 * back to old way
+		 */
+		if (total_pages == 0)
+			hpi->num_pages[0] = get_num_hugepages(dirent->d_name);
 
 #ifndef RTE_ARCH_64
 		/* for 32-bit systems, limit number of hugepages to

@@ -328,10 +386,19 @@ eal_hugepage_info_init(void)
 	      sizeof(internal_config.hugepage_info[0]), compare_hpi);
 
 	/* now we have all info, check we have at least one valid size */
-	for (i = 0; i < num_sizes; i++)
+	for (i = 0; i < num_sizes; i++) {
+		/* pages may no longer all be on socket 0, so check all */
+		unsigned int j, num_pages = 0;
+
+		for (j = 0; j < RTE_MAX_NUMA_NODES; j++) {
+			struct hugepage_info *hpi =
+					&internal_config.hugepage_info[i];
+			num_pages += hpi->num_pages[j];
+		}
 		if (internal_config.hugepage_info[i].hugedir != NULL &&
-		    internal_config.hugepage_info[i].num_pages[0] > 0)
+				num_pages > 0)
 			return 0;
+	}
 
 	/* no valid hugepage mounts available, return error */
 	return -1;

index f02a8ba..a3fc83e 100644
@@ -278,6 +278,15 @@ void
 rte_malloc_dump_stats(FILE *f, const char *type);
 
 /**
+ * Dump contents of all malloc heaps to a file.
+ *
+ * @param f
+ *   A pointer to a file for output
+ */
+void
+rte_malloc_dump_heaps(FILE *f);
+
+/**
  * Set the maximum amount of allocated memory for this type.
  *
  * This is not yet implemented

index eb41200..e02ed88 100644
@@ -1,6 +1,7 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  * Copyright(c) 2010-2014 Intel Corporation
  */
+#include <inttypes.h>
 #include <stdint.h>
 #include <stddef.h>
 #include <stdio.h>

@@ -434,3 +435,26 @@ malloc_elem_resize(struct malloc_elem *elem, size_t size)
 	}
 	return 0;
 }
+
+static inline const char *
+elem_state_to_str(enum elem_state state)
+{
+	switch (state) {
+	case ELEM_PAD:
+		return "PAD";
+	case ELEM_BUSY:
+		return "BUSY";
+	case ELEM_FREE:
+		return "FREE";
+	}
+	return "ERROR";
+}
+
+void
+malloc_elem_dump(const struct malloc_elem *elem, FILE *f)
+{
+	fprintf(f, "Malloc element at %p (%s)\n", elem,
+			elem_state_to_str(elem->state));
+	fprintf(f, "  len: 0x%zx pad: 0x%" PRIx32 "\n", elem->size, elem->pad);
+	fprintf(f, "  prev: %p next: %p\n", elem->prev, elem->next);
+}

index 238e451..40e8eb5 100644
@@ -149,6 +149,12 @@ int
 malloc_elem_resize(struct malloc_elem *elem, size_t size);
 
 /*
+ * dump contents of malloc elem to a file.
+ */
+void
+malloc_elem_dump(const struct malloc_elem *elem, FILE *f);
+
+/*
  * Given an element size, compute its freelist index.
  */
 size_t

index 9c95166..44538d7 100644
@@ -217,6 +217,28 @@ malloc_heap_get_stats(struct malloc_heap *heap,
 	return 0;
 }
 
+/*
+ * Function to retrieve data for heap on given socket
+ */
+void
+malloc_heap_dump(struct malloc_heap *heap, FILE *f)
+{
+	struct malloc_elem *elem;
+
+	rte_spinlock_lock(&heap->lock);
+
+	fprintf(f, "Heap size: 0x%zx\n", heap->total_size);
+	fprintf(f, "Heap alloc count: %u\n", heap->alloc_count);
+
+	elem = heap->first;
+	while (elem) {
+		malloc_elem_dump(elem, f);
+		elem = elem->next;
+	}
+
+	rte_spinlock_unlock(&heap->lock);
+}
+
 int
 rte_eal_malloc_heap_init(void)
 {

index ab0005c..bb28422 100644
@@ -37,6 +37,9 @@ int
 malloc_heap_get_stats(struct malloc_heap *heap,
 		struct rte_malloc_socket_stats *socket_stats);
 
+void
+malloc_heap_dump(struct malloc_heap *heap, FILE *f);
+
 int
 rte_eal_malloc_heap_init(void);
 

index 970813e..80fb6cc 100644
@@ -182,6 +182,22 @@ rte_malloc_get_socket_stats(int socket,
 }
 
 /*
+ * Function to dump contents of all heaps
+ */
+void
+rte_malloc_dump_heaps(FILE *f)
+{
+	struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
+	unsigned int socket;
+
+	for (socket = 0; socket < rte_num_sockets(); socket++) {
+		fprintf(f, "Heap on socket %i:\n", socket);
+		malloc_heap_dump(&mcfg->malloc_heaps[socket], f);
+	}
+
+}
+
+/*
  * Print stats on memory type. If type is NULL, info on all types is printed
  */
 void

index 008f5a3..c18f050 100644
@@ -379,7 +379,7 @@ malloc_elem_join_adjacent_free(struct malloc_elem *elem)
  * blocks either immediately before or immediately after newly freed block
  * are also free, the blocks are merged together.
  */
-int
+struct malloc_elem *
 malloc_elem_free(struct malloc_elem *elem)
 {
 	void *ptr;

@@ -397,7 +397,7 @@ malloc_elem_free(struct malloc_elem *elem)
 
 	memset(ptr, 0, data_len);
 
-	return 0;
+	return elem;
 }
 
 /*

index 46e2383..9c1614c 100644
@@ -138,7 +138,7 @@ malloc_elem_alloc(struct malloc_elem *elem, size_t size,
  * blocks either immediately before or immediately after newly freed block
  * are also free, the blocks are merged together.
  */
-int
+struct malloc_elem *
 malloc_elem_free(struct malloc_elem *elem);
 
 struct malloc_elem *

index 44538d7..a2c2e4c 100644
@@ -145,7 +145,7 @@ int
 malloc_heap_free(struct malloc_elem *elem)
 {
 	struct malloc_heap *heap;
-	int ret;
+	struct malloc_elem *ret;
 
 	if (!malloc_elem_cookies_ok(elem) || elem->state != ELEM_BUSY)
 		return -1;

@@ -159,7 +159,7 @@ malloc_heap_free(struct malloc_elem *elem)
 
 	rte_spinlock_unlock(&(heap->lock));
 
-	return ret;
+	return ret != NULL ? 0 : -1;
 }
 
 int

index 4eafcb5..45e5670 100644
@@ -531,6 +531,9 @@ rte_eal_init(int argc, char **argv)
 		return -1;
 	}
 
+	/* FreeBSD always uses legacy memory model */
+	internal_config.legacy_mem = true;
+
 	if (eal_plugins_init() < 0) {
 		rte_eal_init_alert("Cannot init plugins\n");
 		rte_errno = EINVAL;

index dbc3fb5..3e92551 100644
@@ -74,6 +74,7 @@ eal_long_options[] = {
 	{OPT_VFIO_INTR,         1, NULL, OPT_VFIO_INTR_NUM        },
 	{OPT_VMWARE_TSC_MAP,    0, NULL, OPT_VMWARE_TSC_MAP_NUM   },
 	{OPT_SINGLE_FILE_SEGMENTS, 0, NULL, OPT_SINGLE_FILE_SEGMENTS_NUM},
+	{OPT_LEGACY_MEM,        0, NULL, OPT_LEGACY_MEM_NUM       },
 	{0,                     0, NULL, 0                        }
 };
 

@@ -1165,6 +1166,9 @@ eal_parse_common_option(int opt, const char *optarg,
 	case OPT_SINGLE_FILE_SEGMENTS_NUM:
 		conf->single_file_segments = 1;
 		break;
+	case OPT_LEGACY_MEM_NUM:
+		conf->legacy_mem = 1;
+		break;
 
 	/* don't know what to do, leave this to caller */
 	default:

index 3e31ac6..c8a0676 100644
@@ -51,6 +51,10 @@ struct internal_config {
 	/**< true if storing all pages within single files (per-page-size,
 	 * per-node).
 	 */
+	volatile unsigned legacy_mem;
+	/**< true to enable legacy memory behavior (no dynamic allocation,
+	 * contiguous segments).
+	 */
 	volatile int syslog_facility;	  /**< facility passed to openlog() */
 	/** default interrupt mode for VFIO */
 	volatile enum rte_intr_mode vfio_intr_mode;

index a4b80d5..f9a679d 100644
@@ -57,6 +57,8 @@ enum {
 	OPT_VMWARE_TSC_MAP_NUM,
 #define OPT_SINGLE_FILE_SEGMENTS    "single-file-segments"
 	OPT_SINGLE_FILE_SEGMENTS_NUM,
+#define OPT_LEGACY_MEM    "legacy-mem"
+	OPT_LEGACY_MEM_NUM,
 	OPT_LONG_MAX_NUM
 };
 

index c84e6bf..5207713 100644
@@ -349,6 +349,7 @@ eal_usage(const char *prgname)
 	       "  --"OPT_CREATE_UIO_DEV"    Create /dev/uioX (usually done by hotplug)\n"
 	       "  --"OPT_VFIO_INTR"         Interrupt mode for VFIO (legacy|msi|msix)\n"
 	       "  --"OPT_SINGLE_FILE_SEGMENTS" Put all hugepage memory in single files\n"
+	       "  --"OPT_LEGACY_MEM"        Legacy memory mode (no dynamic allocation, contiguous segments)\n"
 	       "\n");
 	/* Allow the application to print its usage message too if hook is set */
 	if ( rte_application_usage_hook ) {

index 5c11d77..b9bcb75 100644
@@ -919,8 +919,8 @@ huge_recover_sigbus(void)
  *  6. unmap the first mapping
  *  7. fill memsegs in configuration with contiguous zones
  */
-int
-rte_eal_hugepage_init(void)
+static int
+eal_legacy_hugepage_init(void)
 {
 	struct rte_mem_config *mcfg;
 	struct hugepage_file *hugepage = NULL, *tmp_hp = NULL;

@@ -1262,8 +1262,8 @@ getFileSize(int fd)
  * configuration and finds the hugepages which form that segment, mapping them
  * in order to form a contiguous block in the virtual memory space
  */
-int
-rte_eal_hugepage_attach(void)
+static int
+eal_legacy_hugepage_attach(void)
 {
 	const struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
 	struct hugepage_file *hp = NULL;

@@ -1399,6 +1399,22 @@ rte_eal_hugepage_attach(void)
 }
 
 int
+rte_eal_hugepage_init(void)
+{
+	if (internal_config.legacy_mem)
+		return eal_legacy_hugepage_init();
+	return -1;
+}
+
+int
+rte_eal_hugepage_attach(void)
+{
+	if (internal_config.legacy_mem)
+		return eal_legacy_hugepage_attach();
+	return -1;
+}
+
+int
 rte_eal_using_phys_addrs(void)
 {
 	return phys_addrs_available;

index abde641..ec05d7c 100644
@@ -119,19 +119,30 @@ rte_pci_unmap_device(struct rte_pci_device *dev)
 void *
 pci_find_max_end_va(void)
 {
-	const struct rte_memseg *seg = rte_eal_get_physmem_layout();
-	const struct rte_memseg *last = seg;
-	unsigned i = 0;
+	void *cur_end, *max_end = NULL;
+	int i = 0;
 
-	for (i = 0; i < RTE_MAX_MEMSEG; i++, seg++) {
-		if (seg->addr == NULL)
-			break;
+	for (i = 0; i < RTE_MAX_MEMSEG_LISTS; i++) {
+		struct rte_mem_config *mcfg =
+				rte_eal_get_configuration()->mem_config;
+		struct rte_memseg_list *msl = &mcfg->memsegs[i];
+		struct rte_fbarray *arr = &msl->memseg_arr;
 
-		if (seg->addr > last->addr)
-			last = seg;
+		if (arr->count == 0)
+			continue;
 
+		/*
+		 * we need to handle legacy mem case, so don't rely on page size
+		 * to calculate max VA end
+		 */
+		while ((i = rte_fbarray_find_next_used(arr, i)) >= 0) {
+			struct rte_memseg *ms = rte_fbarray_get(arr, i);
+			cur_end = RTE_PTR_ADD(ms->addr, ms->len);
+			if (cur_end > max_end)
+				max_end = cur_end;
+		}
 	}
-	return RTE_PTR_ADD(last->addr, last->len);
+	return max_end;
 }
 
 /* parse one line of the "resource" sysfs file (note that the 'line'

132 lines not on this page · raw message

372 lines not on this page · raw message

index 1ab3ade..ed36174 100644
@@ -226,10 +226,9 @@ memzone_reserve_aligned_thread_unsafe(const char *name, size_t len,
 	mz->iova = rte_malloc_virt2iova(mz_addr);
 	mz->addr = mz_addr;
 	mz->len = (requested_len == 0 ? elem->size : requested_len);
-	mz->hugepage_sz = elem->ms->hugepage_sz;
-	mz->socket_id = elem->ms->socket_id;
+	mz->hugepage_sz = elem->msl->hugepage_sz;
+	mz->socket_id = elem->msl->socket_id;
 	mz->flags = 0;
-	mz->memseg_id = elem->ms - rte_eal_get_configuration()->mem_config->memseg;
 
 	return mz;
 }

@@ -382,7 +381,6 @@ int
 rte_eal_memzone_init(void)
 {
 	struct rte_mem_config *mcfg;
-	const struct rte_memseg *memseg;
 
 	/* get pointer to global configuration */
 	mcfg = rte_eal_get_configuration()->mem_config;

@@ -391,12 +389,6 @@ rte_eal_memzone_init(void)
 	if (rte_eal_process_type() == RTE_PROC_SECONDARY)
 		return 0;
 
-	memseg = rte_eal_get_physmem_layout();
-	if (memseg == NULL) {
-		RTE_LOG(ERR, EAL, "%s(): Cannot get physical layout\n", __func__);
-		return -1;
-	}
-
 	rte_rwlock_write_lock(&mcfg->mlock);
 
 	/* delete all zones */

index 1d519bb..f963ae5 100644
@@ -23,6 +23,8 @@ struct hugepage_file {
 	int socket_id;      /**< NUMA socket ID */
 	int file_id;        /**< the '%d' in HUGEFILE_FMT */
 	int memseg_id;      /**< the memory segment to which page belongs */
+	int memseg_list_id;
+	/**< the memory segment list to which page belongs */
 	char filepath[MAX_HUGEPAGE_PATH]; /**< path to backing file on filesystem */
 };
 

index c8a0676..eea8b66 100644
@@ -23,7 +23,7 @@ struct hugepage_info {
 	uint64_t hugepage_sz;   /**< size of a huge page */
 	const char *hugedir;    /**< dir where hugetlbfs is mounted */
 	uint32_t num_pages[RTE_MAX_NUMA_NODES];
-				/**< number of hugepages of that size on each socket */
+	/**< number of hugepages of that size on each socket */
 	int lock_descriptor;    /**< file descriptor for hugepage dir */
 };
 

index 29fa0b6..31fc8e7 100644
@@ -12,12 +12,30 @@
 #include <rte_malloc_heap.h>
 #include <rte_rwlock.h>
 #include <rte_pause.h>
+#include <rte_fbarray.h>
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
 /**
+ * memseg list is a special case as we need to store a bunch of other data
+ * together with the array itself.
+ */
+struct rte_memseg_list {
+	RTE_STD_C11
+	union {
+		void *base_va;
+		/**< Base virtual address for this memseg list. */
+		uint64_t addr_64;
+		/**< Makes sure addr is always 64-bits */
+	};
+	int socket_id; /**< Socket ID for all memsegs in this list. */
+	uint64_t hugepage_sz; /**< page size for all memsegs in this list. */
+	struct rte_fbarray memseg_arr;
+};
+
+/**
  * the structure for the memory configuration for the RTE.
  * Used by the rte_config structure. It is separated out, as for multi-process
  * support, the memory details should be shared across instances

@@ -43,9 +61,11 @@ struct rte_mem_config {
 	uint32_t memzone_cnt; /**< Number of allocated memzones */
 
 	/* memory segments and zones */
-	struct rte_memseg memseg[RTE_MAX_MEMSEG];    /**< Physmem descriptors. */
 	struct rte_memzone memzone[RTE_MAX_MEMZONE]; /**< Memzone descriptors. */
 
+	struct rte_memseg_list memsegs[RTE_MAX_MEMSEG_LISTS];
+	/**< list of dynamic arrays holding memsegs */
+
 	struct rte_tailq_head tailq_head[RTE_MAX_TAILQ]; /**< Tailqs for objects */
 
 	/* Heaps of Malloc per socket */

index 302f865..674d4cb 100644
@@ -22,6 +22,9 @@ extern "C" {
 #include <rte_common.h>
 #include <rte_config.h>
 
+/* forward declaration for pointers */
+struct rte_memseg_list;
+
 __extension__
 enum rte_page_sizes {
 	RTE_PGSIZE_4K    = 1ULL << 12,

@@ -130,21 +133,27 @@ phys_addr_t rte_mem_virt2phy(const void *virt);
 rte_iova_t rte_mem_virt2iova(const void *virt);
 
 /**
- * Get the layout of the available physical memory.
+ * Get memseg corresponding to virtual memory address.
  *
- * It can be useful for an application to have the full physical
- * memory layout to decide the size of a memory zone to reserve. This
- * table is stored in rte_config (see rte_eal_get_configuration()).
+ * @param virt
+ *   The virtual address.
+ * @param msl
+ *   Memseg list in which to look for memsegs (can be NULL).
+ * @return
+ *   Memseg to which this virtual address belongs to.
+ */
+struct rte_memseg *rte_mem_virt2memseg(const void *virt,
+		const struct rte_memseg_list *msl);
+
+/**
+ * Get memseg list corresponding to virtual memory address.
  *
+ * @param virt
+ *   The virtual address.
  * @return
- *  - On success, return a pointer to a read-only table of struct
- *    rte_physmem_desc elements, containing the layout of all
- *    addressable physical memory. The last element of the table
- *    contains a NULL address.
- *  - On error, return NULL. This should not happen since it is a fatal
- *    error that will probably cause the entire system to panic.
- */
-const struct rte_memseg *rte_eal_get_physmem_layout(void);
+ *   Memseg list to which this virtual address belongs to.
+ */
+struct rte_memseg_list *rte_mem_virt2memseg_list(const void *virt);
 
 /**
  * Dump the physical memory layout to a file.

index 2bfb273..a69f068 100644
@@ -66,7 +66,6 @@ struct rte_memzone {
 	int32_t socket_id;                /**< NUMA socket ID. */
 
 	uint32_t flags;                   /**< Characteristics of this memzone. */
-	uint32_t memseg_id;               /**< Memseg it belongs. */
 } __attribute__((__packed__));
 
 /**

index c18f050..701bffd 100644
@@ -26,11 +26,11 @@
  * Initialize a general malloc_elem header structure
  */
 void
-malloc_elem_init(struct malloc_elem *elem,
-		struct malloc_heap *heap, const struct rte_memseg *ms, size_t size)
+malloc_elem_init(struct malloc_elem *elem, struct malloc_heap *heap,
+		struct rte_memseg_list *msl, size_t size)
 {
 	elem->heap = heap;
-	elem->ms = ms;
+	elem->msl = msl;
 	elem->prev = NULL;
 	elem->next = NULL;
 	memset(&elem->free_list, 0, sizeof(elem->free_list));

@@ -145,7 +145,7 @@ split_elem(struct malloc_elem *elem, struct malloc_elem *split_pt)
 	const size_t old_elem_size = (uintptr_t)split_pt - (uintptr_t)elem;
 	const size_t new_elem_size = elem->size - old_elem_size;
 
-	malloc_elem_init(split_pt, elem->heap, elem->ms, new_elem_size);
+	malloc_elem_init(split_pt, elem->heap, elem->msl, new_elem_size);
 	split_pt->prev = elem;
 	split_pt->next = next_elem;
 	if (next_elem)

index 9c1614c..388c16f 100644
@@ -5,7 +5,7 @@
 #ifndef MALLOC_ELEM_H_
 #define MALLOC_ELEM_H_
 
-#include <rte_memory.h>
+#include <rte_eal_memconfig.h>
 
 /* dummy definition of struct so we can use pointers to it in malloc_elem struct */
 struct malloc_heap;

@@ -24,7 +24,7 @@ struct malloc_elem {
 	/**< points to next elem in memseg */
 	LIST_ENTRY(malloc_elem) free_list;
 	/**< list of free elements in heap */
-	const struct rte_memseg *ms;
+	struct rte_memseg_list *msl;
 	volatile enum elem_state state;
 	uint32_t pad;
 	size_t size;

@@ -111,7 +111,7 @@ malloc_elem_from_data(const void *data)
 void
 malloc_elem_init(struct malloc_elem *elem,
 		struct malloc_heap *heap,
-		const struct rte_memseg *ms,
+		struct rte_memseg_list *msl,
 		size_t size);
 
 void

124 lines not on this page · raw message

index 80fb6cc..bd7e757 100644
@@ -238,17 +238,21 @@ rte_malloc_set_limit(__rte_unused const char *type,
 rte_iova_t
 rte_malloc_virt2iova(const void *addr)
 {
-	rte_iova_t iova;
-	const struct malloc_elem *elem = malloc_elem_from_data(addr);
+	const struct rte_memseg *ms;
+	struct malloc_elem *elem = malloc_elem_from_data(addr);
+
 	if (elem == NULL)
 		return RTE_BAD_IOVA;
-	if (elem->ms->iova == RTE_BAD_IOVA)
-		return RTE_BAD_IOVA;
 
 	if (rte_eal_iova_mode() == RTE_IOVA_VA)
-		iova = (uintptr_t)addr;
-	else
-		iova = elem->ms->iova +
-			RTE_PTR_DIFF(addr, elem->ms->addr);
-	return iova;
+		return (uintptr_t) addr;
+
+	ms = rte_mem_virt2memseg(addr, elem->msl);
+	if (ms == NULL)
+		return RTE_BAD_IOVA;
+
+	if (ms->iova == RTE_BAD_IOVA)
+		return RTE_BAD_IOVA;
+
+	return ms->iova + RTE_PTR_DIFF(addr, ms->addr);
 }

index 5207713..7851a7d 100644
@@ -74,8 +74,8 @@ static int mem_cfg_fd = -1;
 static struct flock wr_lock = {
 		.l_type = F_WRLCK,
 		.l_whence = SEEK_SET,
-		.l_start = offsetof(struct rte_mem_config, memseg),
-		.l_len = sizeof(early_mem_config.memseg),
+		.l_start = offsetof(struct rte_mem_config, memsegs),
+		.l_len = sizeof(early_mem_config.memsegs),
 };
 
 /* Address of global and public configuration */

@@ -643,17 +643,20 @@ eal_parse_args(int argc, char **argv)
 static void
 eal_check_mem_on_local_socket(void)
 {
-	const struct rte_memseg *ms;
+	const struct rte_memseg_list *msl;
 	int i, socket_id;
 
 	socket_id = rte_lcore_to_socket_id(rte_config.master_lcore);
 
-	ms = rte_eal_get_physmem_layout();
-
-	for (i = 0; i < RTE_MAX_MEMSEG; i++)
-		if (ms[i].socket_id == socket_id &&
-				ms[i].len > 0)
-			return;
+	for (i = 0; i < RTE_MAX_MEMSEG_LISTS; i++) {
+		msl = &rte_eal_get_configuration()->mem_config->memsegs[i];
+		if (msl->socket_id != socket_id)
+			continue;
+		/* for legacy memory, check if there's anything allocated */
+		if (internal_config.legacy_mem && msl->memseg_arr.count == 0)
+			continue;
+		return;
+	}
 
 	RTE_LOG(WARNING, EAL, "WARNING: Master core has no "
 			"memory on local socket!\n");

385 lines not on this page · raw message

208 lines not on this page · raw message

index d23192c..8484fb6 100644
@@ -12,6 +12,7 @@
 
 #include <rte_common.h>
 #include <rte_memory.h>
+#include <rte_eal_memconfig.h>
 #include <rte_per_lcore.h>
 #include <rte_launch.h>
 #include <rte_eal.h>

@@ -705,15 +706,23 @@ test_malloc_bad_params(void)
 	return -1;
 }
 
-/* Check if memory is available on a specific socket */
+/* Check if memory is avilable on a specific socket */
 static int
 is_mem_on_socket(int32_t socket)
 {
-	const struct rte_memseg *ms = rte_eal_get_physmem_layout();
+	const struct rte_mem_config *mcfg =
+			rte_eal_get_configuration()->mem_config;
 	unsigned i;
 
-	for (i = 0; i < RTE_MAX_MEMSEG; i++) {
-		if (socket == ms[i].socket_id)
+	for (i = 0; i < RTE_MAX_MEMSEG_LISTS; i++) {
+		const struct rte_memseg_list *msl =
+				&mcfg->memsegs[i];
+		const struct rte_fbarray *arr = &msl->memseg_arr;
+
+		if (msl->socket_id != socket)
+			continue;
+
+		if (arr->count)
 			return 1;
 	}
 	return 0;

@@ -726,16 +735,8 @@ is_mem_on_socket(int32_t socket)
 static int32_t
 addr_to_socket(void * addr)
 {
-	const struct rte_memseg *ms = rte_eal_get_physmem_layout();
-	unsigned i;
-
-	for (i = 0; i < RTE_MAX_MEMSEG; i++) {
-		if ((ms[i].addr <= addr) &&
-				((uintptr_t)addr <
-				((uintptr_t)ms[i].addr + (uintptr_t)ms[i].len)))
-			return ms[i].socket_id;
-	}
-	return -1;
+	const struct rte_memseg *ms = rte_mem_virt2memseg(addr, NULL);
+	return ms == NULL ? -1 : ms->socket_id;
 }
 
 /* Test using rte_[c|m|zm]alloc_socket() on a specific socket */

73 lines not on this page · raw message

index 8ece1ac..47f4de8 100644
@@ -108,22 +108,25 @@ static int
 test_memzone_reserve_flags(void)
 {
 	const struct rte_memzone *mz;
-	const struct rte_memseg *ms;
 	int hugepage_2MB_avail = 0;
 	int hugepage_1GB_avail = 0;
 	int hugepage_16MB_avail = 0;
 	int hugepage_16GB_avail = 0;
 	const size_t size = 100;
 	int i = 0;
-	ms = rte_eal_get_physmem_layout();
-	for (i = 0; i < RTE_MAX_MEMSEG; i++) {
-		if (ms[i].hugepage_sz == RTE_PGSIZE_2M)
+
+	for (i = 0; i < RTE_MAX_MEMSEG_LISTS; i++) {
+		struct rte_mem_config *mcfg =
+				rte_eal_get_configuration()->mem_config;
+		struct rte_memseg_list *msl = &mcfg->memsegs[i];
+
+		if (msl->hugepage_sz == RTE_PGSIZE_2M)
 			hugepage_2MB_avail = 1;
-		if (ms[i].hugepage_sz == RTE_PGSIZE_1G)
+		if (msl->hugepage_sz == RTE_PGSIZE_1G)
 			hugepage_1GB_avail = 1;
-		if (ms[i].hugepage_sz == RTE_PGSIZE_16M)
+		if (msl->hugepage_sz == RTE_PGSIZE_16M)
 			hugepage_16MB_avail = 1;
-		if (ms[i].hugepage_sz == RTE_PGSIZE_16G)
+		if (msl->hugepage_sz == RTE_PGSIZE_16G)
 			hugepage_16GB_avail = 1;
 	}
 	/* Display the availability of 2MB ,1GB, 16MB, 16GB pages */

index c1076cf..adf59c4 100644
@@ -16,4 +16,7 @@ int
 eal_memalloc_alloc_page_bulk(struct rte_memseg **ms, int n, uint64_t size,
 		int socket, bool exact);
 
+int
+eal_memalloc_free_page(struct rte_memseg *ms);
+
 #endif // EAL_MEMALLOC_H

190 lines not on this page · raw message

49 lines not on this page · raw message

index adf59c4..08ba70e 100644
@@ -8,6 +8,7 @@
 #include <stdbool.h>
 
 #include <rte_memory.h>
+#include <rte_eal_memconfig.h>
 
 struct rte_memseg *
 eal_memalloc_alloc_page(uint64_t size, int socket);

@@ -19,4 +20,8 @@ eal_memalloc_alloc_page_bulk(struct rte_memseg **ms, int n, uint64_t size,
 int
 eal_memalloc_free_page(struct rte_memseg *ms);
 
+bool
+eal_memalloc_is_contig(struct rte_memseg_list *msl, void *start,
+		size_t len);
+
 #endif // EAL_MEMALLOC_H

index be8340b..255aedc 100644
@@ -24,3 +24,10 @@ eal_memalloc_alloc_page(uint64_t __rte_unused size, int __rte_unused socket)
 	RTE_LOG(ERR, EAL, "Memory hotplug not supported on FreeBSD\n");
 	return NULL;
 }
+
+int
+eal_memalloc_sync_with_primary(void)
+{
+	RTE_LOG(ERR, EAL, "Memory hotplug not supported on FreeBSD\n");
+	return -1;
+}

index 08ba70e..beac296 100644
@@ -24,4 +24,8 @@ bool
 eal_memalloc_is_contig(struct rte_memseg_list *msl, void *start,
 		size_t len);
 
+/* synchronize local memory map to primary process */
+int
+eal_memalloc_sync_with_primary(void);
+
 #endif // EAL_MEMALLOC_H

index b6bdb21..d653d57 100644
@@ -32,6 +32,8 @@ struct rte_memseg_list {
 	};
 	int socket_id; /**< Socket ID for all memsegs in this list. */
 	uint64_t hugepage_sz; /**< page size for all memsegs in this list. */
+	rte_rwlock_t mplock; /**< read-write lock for multiprocess sync. */
+	uint32_t version; /**< version number for multiprocess sync. */
 	struct rte_fbarray memseg_arr;
 };
 

336 lines not on this page · raw message

index 718dee8..75c7dd9 100644
@@ -98,7 +98,8 @@ find_heap_max_free_elem(int *s, unsigned align)
 
 static const struct rte_memzone *
 memzone_reserve_aligned_thread_unsafe(const char *name, size_t len,
-		int socket_id, unsigned flags, unsigned align, unsigned bound)
+		int socket_id, unsigned int flags, unsigned int align,
+		unsigned int bound, bool contig)
 {
 	struct rte_memzone *mz;
 	struct rte_mem_config *mcfg;

@@ -182,7 +183,7 @@ memzone_reserve_aligned_thread_unsafe(const char *name, size_t len,
 
 	/* allocate memory on heap */
 	void *mz_addr = malloc_heap_alloc(NULL, requested_len, socket_id, flags,
-			align, bound);
+			align, bound, contig);
 
 	if (mz_addr == NULL) {
 		rte_errno = ENOMEM;

@@ -215,9 +216,9 @@ memzone_reserve_aligned_thread_unsafe(const char *name, size_t len,
 }
 
 static const struct rte_memzone *
-rte_memzone_reserve_thread_safe(const char *name, size_t len,
-				int socket_id, unsigned flags, unsigned align,
-				unsigned bound)
+rte_memzone_reserve_thread_safe(const char *name, size_t len, int socket_id,
+		unsigned int flags, unsigned int align, unsigned int bound,
+		bool contig)
 {
 	struct rte_mem_config *mcfg;
 	const struct rte_memzone *mz = NULL;

@@ -228,7 +229,7 @@ rte_memzone_reserve_thread_safe(const char *name, size_t len,
 	rte_rwlock_write_lock(&mcfg->mlock);
 
 	mz = memzone_reserve_aligned_thread_unsafe(
-		name, len, socket_id, flags, align, bound);
+		name, len, socket_id, flags, align, bound, contig);
 
 	rte_rwlock_write_unlock(&mcfg->mlock);
 

@@ -245,7 +246,7 @@ rte_memzone_reserve_bounded(const char *name, size_t len, int socket_id,
 			    unsigned flags, unsigned align, unsigned bound)
 {
 	return rte_memzone_reserve_thread_safe(name, len, socket_id, flags,
-					       align, bound);
+					       align, bound, false);
 }
 
 /*

@@ -257,7 +258,7 @@ rte_memzone_reserve_aligned(const char *name, size_t len, int socket_id,
 			    unsigned flags, unsigned align)
 {
 	return rte_memzone_reserve_thread_safe(name, len, socket_id, flags,
-					       align, 0);
+					       align, 0, false);
 }
 
 /*

@@ -269,7 +270,8 @@ rte_memzone_reserve(const char *name, size_t len, int socket_id,
 		    unsigned flags)
 {
 	return rte_memzone_reserve_thread_safe(name, len, socket_id,
-					       flags, RTE_CACHE_LINE_SIZE, 0);
+					       flags, RTE_CACHE_LINE_SIZE, 0,
+					       false);
 }
 
 int

136 lines not on this page · raw message

index 6d979d2..798472e 100644
@@ -123,7 +123,7 @@ malloc_elem_insert(struct malloc_elem *elem);
  */
 int
 malloc_elem_can_hold(struct malloc_elem *elem, size_t size,
-		unsigned align, size_t bound);
+		unsigned int align, size_t bound, bool contig);
 
 /*
  * reserve a block of data in an existing malloc_elem. If the malloc_elem

@@ -131,7 +131,7 @@ malloc_elem_can_hold(struct malloc_elem *elem, size_t size,
  */
 struct malloc_elem *
 malloc_elem_alloc(struct malloc_elem *elem, size_t size,
-		unsigned align, size_t bound);
+		unsigned int align, size_t bound, bool contig);
 
 /*
  * free a malloc_elem block by adding it to the free list. If the

172 lines not on this page · raw message

index 292d578..03b8014 100644
@@ -5,6 +5,8 @@
 #ifndef MALLOC_HEAP_H_
 #define MALLOC_HEAP_H_
 
+#include <stdbool.h>
+
 #include <rte_malloc.h>
 #include <rte_malloc_heap.h>
 

@@ -25,7 +27,7 @@ malloc_get_numa_socket(void)
 
 void *
 malloc_heap_alloc(const char *type, size_t size, int socket, unsigned int flags,
-		size_t align, size_t bound);
+		size_t align, size_t bound, bool contig);
 
 int
 malloc_heap_free(struct malloc_elem *elem);

index b0fe11c..5cd92d1 100644
@@ -37,7 +37,8 @@ void rte_free(void *addr)
  * Allocate memory on specified heap.
  */
 void *
-rte_malloc_socket(const char *type, size_t size, unsigned align, int socket_arg)
+rte_malloc_socket(const char *type, size_t size, unsigned int align,
+		int socket_arg)
 {
 	/* return NULL if size is 0 or alignment is not power-of-2 */
 	if (size == 0 || (align && !rte_is_power_of_2(align)))

@@ -50,8 +51,7 @@ rte_malloc_socket(const char *type, size_t size, unsigned align, int socket_arg)
 	if (socket_arg >= RTE_MAX_NUMA_NODES)
 		return NULL;
 
-	return malloc_heap_alloc(type, size, socket_arg, 0,
-			align == 0 ? 1 : align, 0);
+	return malloc_heap_alloc(type, size, socket_arg, 0, align, 0, false);
 }
 
 /*

index 75c7dd9..8c9aa28 100644
@@ -170,6 +170,12 @@ memzone_reserve_aligned_thread_unsafe(const char *name, size_t len,
 		socket_id = SOCKET_ID_ANY;
 
 	if (len == 0) {
+		/* len == 0 is only allowed for non-contiguous zones */
+		if (contig) {
+			RTE_LOG(DEBUG, EAL, "Reserving zero-length contiguous memzones is not supported\n");
+			rte_errno = EINVAL;
+			return NULL;
+		}
 		if (bound != 0)
 			requested_len = bound;
 		else {

@@ -251,6 +257,19 @@ rte_memzone_reserve_bounded(const char *name, size_t len, int socket_id,
 
 /*
  * Return a pointer to a correctly filled memzone descriptor (with a
+ * specified alignment and boundary). If the allocation cannot be done,
+ * return NULL.
+ */
+const struct rte_memzone *
+rte_memzone_reserve_bounded_contig(const char *name, size_t len, int socket_id,
+		unsigned int flags, unsigned int align, unsigned int bound)
+{
+	return rte_memzone_reserve_thread_safe(name, len, socket_id, flags,
+					       align, bound, true);
+}
+
+/*
+ * Return a pointer to a correctly filled memzone descriptor (with a
  * specified alignment). If the allocation cannot be done, return NULL.
  */
 const struct rte_memzone *

@@ -262,6 +281,18 @@ rte_memzone_reserve_aligned(const char *name, size_t len, int socket_id,
 }
 
 /*
+ * Return a pointer to a correctly filled memzone descriptor (with a
+ * specified alignment). If the allocation cannot be done, return NULL.
+ */
+const struct rte_memzone *
+rte_memzone_reserve_aligned_contig(const char *name, size_t len, int socket_id,
+		unsigned int flags, unsigned int align)
+{
+	return rte_memzone_reserve_thread_safe(name, len, socket_id, flags,
+					       align, 0, true);
+}
+
+/*
  * Return a pointer to a correctly filled memzone descriptor. If the
  * allocation cannot be done, return NULL.
  */

@@ -274,6 +305,19 @@ rte_memzone_reserve(const char *name, size_t len, int socket_id,
 					       false);
 }
 
+/*
+ * Return a pointer to a correctly filled memzone descriptor. If the
+ * allocation cannot be done, return NULL.
+ */
+const struct rte_memzone *
+rte_memzone_reserve_contig(const char *name, size_t len, int socket_id,
+		unsigned int flags)
+{
+	return rte_memzone_reserve_thread_safe(name, len, socket_id,
+					       flags, RTE_CACHE_LINE_SIZE, 0,
+					       true);
+}
+
 int
 rte_memzone_free(const struct rte_memzone *mz)
 {

160 lines not on this page · raw message

index be2dbf0..18e6e5e 100644
@@ -103,3 +103,10 @@ eal_hugepage_info_init(void)
 
 	return 0;
 }
+
+/* memory hotplug is not supported in FreeBSD, so no need to implement this */
+int
+eal_hugepage_info_read(void)
+{
+	return 0;
+}

151 lines not on this page · raw message

index f963ae5..38d0b04 100644
@@ -34,4 +34,9 @@ struct hugepage_file {
  */
 int eal_hugepage_info_init(void);
 
+/**
+ * Read information about hugepages on Linux, but don't clear them out.
+ */
+int eal_hugepage_info_read(void);
+
 #endif /* EAL_HUGEPAGES_H */

index d336c96..7a0d742 100644
@@ -805,13 +805,17 @@ rte_eal_init(int argc, char **argv)
 			"KNI module inserted\n");
 	}
 
-	if (internal_config.no_hugetlbfs == 0 &&
-			internal_config.process_type != RTE_PROC_SECONDARY &&
-			eal_hugepage_info_init() < 0) {
-		rte_eal_init_alert("Cannot get hugepage information.");
-		rte_errno = EACCES;
-		rte_atomic32_clear(&run_once);
-		return -1;
+	if (internal_config.no_hugetlbfs == 0) {
+		/* rte_config isn't initialized yet */
+		ret = internal_config.process_type == RTE_PROC_PRIMARY ?
+				eal_hugepage_info_init() :
+				eal_hugepage_info_read();
+		if (ret < 0) {
+			rte_eal_init_alert("Cannot get hugepage information.");
+			rte_errno = EACCES;
+			rte_atomic32_clear(&run_once);
+			return -1;
+		}
 	}
 
 	if (internal_config.memory == 0 && internal_config.force_sockets == 0) {

82 lines not on this page · raw message

index e0b4988..f74291f 100644
@@ -1569,6 +1569,22 @@ eal_legacy_hugepage_attach(void)
 	return -1;
 }
 
+static int
+eal_hugepage_attach(void)
+{
+	if (eal_memalloc_sync_with_primary()) {
+		RTE_LOG(ERR, EAL, "Could not map memory from primary process\n");
+		if (aslr_enabled() > 0) {
+			RTE_LOG(ERR, EAL, "It is recommended to "
+				"disable ASLR in the kernel "
+				"and retry running both primary "
+				"and secondary processes\n");
+		}
+		return -1;
+	}
+	return 0;
+}
+
 int
 rte_eal_hugepage_init(void)
 {

@@ -1580,11 +1596,9 @@ rte_eal_hugepage_init(void)
 int
 rte_eal_hugepage_attach(void)
 {
-	if (internal_config.legacy_mem)
-		return eal_legacy_hugepage_attach();
-	else
-		RTE_LOG(ERR, EAL, "Secondary processes aren't supported yet\n");
-	return -1;
+	return internal_config.legacy_mem ?
+			eal_legacy_hugepage_attach() :
+			eal_hugepage_attach();
 }
 
 int

index d653d57..c4b36f6 100644
@@ -60,6 +60,9 @@ struct rte_mem_config {
 	rte_rwlock_t qlock;   /**< used for tailq operation for thread safe. */
 	rte_rwlock_t mplock;  /**< only used by mempool LIB for thread-safe. */
 
+	rte_rwlock_t memory_hotplug_lock;
+	/**< indicates whether memory hotplug request is in progress. */
+
 	/* memory segments and zones */
 	struct rte_fbarray memzones; /**< Memzone descriptors. */
 

381 lines not on this page · raw message

723 lines not on this page · raw message

86 lines not on this page · raw message

153 lines not on this page · raw message

index 62e8c16..4fb55f2 100644
@@ -2,16 +2,46 @@
  * Copyright(c) 2017-2018 Intel Corporation
  */
 
+#include <string.h>
+
+#include <rte_errno.h>
 #include <rte_lcore.h>
 #include <rte_fbarray.h>
 #include <rte_memzone.h>
 #include <rte_memory.h>
 #include <rte_eal_memconfig.h>
+#include <rte_rwlock.h>
 
 #include "eal_private.h"
 #include "eal_internal_cfg.h"
 #include "eal_memalloc.h"
 
+struct mem_event_callback_entry {
+	TAILQ_ENTRY(mem_event_callback_entry) next;
+	char name[RTE_MEM_EVENT_CALLBACK_NAME_LEN];
+	rte_mem_event_callback_t clb;
+};
+
+/** Double linked list of actions. */
+TAILQ_HEAD(mem_event_callback_entry_list, mem_event_callback_entry);
+
+static struct mem_event_callback_entry_list callback_list =
+	TAILQ_HEAD_INITIALIZER(callback_list);
+
+static rte_rwlock_t rwlock = RTE_RWLOCK_INITIALIZER;
+
+static struct mem_event_callback_entry *
+find_callback(const char *name)
+{
+	struct mem_event_callback_entry *r;
+
+	TAILQ_FOREACH(r, &callback_list, next) {
+		if (!strcmp(r->name, name))
+			break;
+	}
+	return r;
+}
+
 bool
 eal_memalloc_is_contig(struct rte_memseg_list *msl, void *start,
 		size_t len)

@@ -47,3 +77,105 @@ eal_memalloc_is_contig(struct rte_memseg_list *msl, void *start,
 
 	return true;
 }
+
+int
+eal_memalloc_callback_register(const char *name,
+		rte_mem_event_callback_t clb)
+{
+	struct mem_event_callback_entry *entry;
+	int ret, len;
+	if (name == NULL || clb == NULL) {
+		rte_errno = EINVAL;
+		return -1;
+	}
+	len = strnlen(name, RTE_MEM_EVENT_CALLBACK_NAME_LEN);
+	if (len == 0) {
+		rte_errno = EINVAL;
+		return -1;
+	} else if (len == RTE_MEM_EVENT_CALLBACK_NAME_LEN) {
+		rte_errno = ENAMETOOLONG;
+		return -1;
+	}
+	rte_rwlock_write_lock(&rwlock);
+
+	entry = find_callback(name);
+	if (entry != NULL) {
+		rte_errno = EEXIST;
+		ret = -1;
+		goto unlock;
+	}
+
+	entry = malloc(sizeof(*entry));
+	if (entry == NULL) {
+		rte_errno = ENOMEM;
+		ret = -1;
+		goto unlock;
+	}
+
+	/* callback successfully created and is valid, add it to the list */
+	entry->clb = clb;
+	snprintf(entry->name, RTE_MEM_EVENT_CALLBACK_NAME_LEN, "%s", name);
+	TAILQ_INSERT_TAIL(&callback_list, entry, next);
+
+	ret = 0;
+
+	RTE_LOG(DEBUG, EAL, "Mem event callback '%s' registered\n", name);
+
+unlock:
+	rte_rwlock_write_unlock(&rwlock);
+	return ret;
+}
+
+int
+eal_memalloc_callback_unregister(const char *name)
+{
+	struct mem_event_callback_entry *entry;
+	int ret, len;
+
+	if (name == NULL) {
+		rte_errno = EINVAL;
+		return -1;
+	}
+	len = strnlen(name, RTE_MEM_EVENT_CALLBACK_NAME_LEN);
+	if (len == 0) {
+		rte_errno = EINVAL;
+		return -1;
+	} else if (len == RTE_MEM_EVENT_CALLBACK_NAME_LEN) {
+		rte_errno = ENAMETOOLONG;
+		return -1;
+	}
+	rte_rwlock_write_lock(&rwlock);
+
+	entry = find_callback(name);
+	if (entry == NULL) {
+		rte_errno = ENOENT;
+		ret = -1;
+		goto unlock;
+	}
+	TAILQ_REMOVE(&callback_list, entry, next);
+	free(entry);
+
+	ret = 0;
+
+	RTE_LOG(DEBUG, EAL, "Mem event callback '%s' unregistered\n", name);
+
+unlock:
+	rte_rwlock_write_unlock(&rwlock);
+	return ret;
+}
+
+void
+eal_memalloc_notify(enum rte_mem_event event, const void *start, size_t len)
+{
+	struct mem_event_callback_entry *entry;
+
+	rte_rwlock_read_lock(&rwlock);
+
+	TAILQ_FOREACH(entry, &callback_list, next) {
+		RTE_LOG(DEBUG, EAL, "Calling mem event callback %s",
+			entry->name);
+		entry->clb(event, start, len);
+	}
+
+	rte_rwlock_read_unlock(&rwlock);
+}

index a571e24..dcba099 100644
@@ -466,6 +466,34 @@ rte_eal_get_physmem_size(void)
 	return total_len;
 }
 
+/*
+ * Defining here because declared in rte_memory.h, but the actual implementation
+ * is in eal_common_memalloc.c, like all other memalloc internals.
+ */
+int
+rte_mem_event_register_callback(const char *name, rte_mem_event_callback_t clb)
+{
+	/* FreeBSD boots with legacy mem enabled by default */
+	if (internal_config.legacy_mem) {
+		RTE_LOG(DEBUG, EAL, "Registering mem event callbacks not supported\n");
+		rte_errno = ENOTSUP;
+		return -1;
+	}
+	return eal_memalloc_callback_register(name, clb);
+}
+
+int
+rte_mem_event_unregister_callback(const char *name)
+{
+	/* FreeBSD boots with legacy mem enabled by default */
+	if (internal_config.legacy_mem) {
+		RTE_LOG(DEBUG, EAL, "Registering mem event callbacks not supported\n");
+		rte_errno = ENOTSUP;
+		return -1;
+	}
+	return eal_memalloc_callback_unregister(name);
+}
+
 /* Dump the physical memory layout on console */
 void
 rte_dump_physmem_layout(FILE *f)

index beac296..499cf58 100644
@@ -28,4 +28,14 @@ eal_memalloc_is_contig(struct rte_memseg_list *msl, void *start,
 int
 eal_memalloc_sync_with_primary(void);
 
+int
+eal_memalloc_callback_register(const char *name,
+		rte_mem_event_callback_t clb);
+
+int
+eal_memalloc_callback_unregister(const char *name);
+
+void
+eal_memalloc_notify(enum rte_mem_event event, const void *start, size_t len);
+
 #endif // EAL_MEMALLOC_H

54 lines not on this page · raw message

index 9109555..9d055c8 100644
@@ -223,6 +223,7 @@ try_expand_heap_primary(struct malloc_heap *heap, uint64_t pg_sz,
 	void *map_addr;
 	size_t map_len;
 	int n_pages;
+	bool callback_triggered = false;
 
 	map_len = RTE_ALIGN_CEIL(align + elt_size +
 			MALLOC_ELEM_TRAILER_LEN, pg_sz);

@@ -242,14 +243,25 @@ try_expand_heap_primary(struct malloc_heap *heap, uint64_t pg_sz,
 
 	map_addr = ms[0]->addr;
 
+	/* notify user about changes in memory map */
+	eal_memalloc_notify(RTE_MEM_EVENT_ALLOC, map_addr, map_len);
+
 	/* notify other processes that this has happened */
 	if (request_sync()) {
 		/* we couldn't ensure all processes have mapped memory,
 		 * so free it back and notify everyone that it's been
 		 * freed back.
+		 *
+		 * technically, we could've avoided adding memory addresses to
+		 * the map, but that would've led to inconsistent behavior
+		 * between primary and secondary processes, as those get
+		 * callbacks during sync. therefore, force primary process to
+		 * do alloc-and-rollback syncs as well.
 		 */
+		callback_triggered = true;
 		goto free_elem;
 	}
+
 	heap->total_size += map_len;
 
 	RTE_LOG(DEBUG, EAL, "Heap on socket %d was expanded by %zdMB\n",

@@ -260,6 +272,9 @@ try_expand_heap_primary(struct malloc_heap *heap, uint64_t pg_sz,
 	return 0;
 
 free_elem:
+	if (callback_triggered)
+		eal_memalloc_notify(RTE_MEM_EVENT_FREE, map_addr, map_len);
+
 	rollback_expand_heap(ms, n_pages, elem, map_addr, map_len);
 
 	request_sync();

@@ -615,6 +630,10 @@ malloc_heap_free(struct malloc_elem *elem)
 	heap->total_size -= n_pages * msl->hugepage_sz;
 
 	if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
+		/* notify user about changes in memory map */
+		eal_memalloc_notify(RTE_MEM_EVENT_FREE,
+				aligned_start, aligned_len);
+
 		/* don't care if any of this fails */
 		malloc_heap_free_pages(aligned_start, aligned_len);
 

@@ -637,6 +656,8 @@ malloc_heap_free(struct malloc_elem *elem)
 		 * already removed from the heap, so it is, for all intents and
 		 * purposes, hidden from the rest of DPDK even if some other
 		 * process (including this one) may have these pages mapped.
+		 *
+		 * notifications about deallocated memory happen during sync.
 		 */
 		request_to_primary(&req);
 	}

index 227d703..1008fae 100644
@@ -34,7 +34,6 @@
 #include <rte_eal.h>
 #include <rte_memory.h>
 #include <rte_spinlock.h>
-#include <rte_vfio.h>
 
 #include "eal_filesystem.h"
 #include "eal_internal_cfg.h"

@@ -480,10 +479,6 @@ alloc_page(struct rte_memseg *ms, void *addr, uint64_t size, int socket_id,
 	ms->iova = iova;
 	ms->socket_id = socket_id;
 
-	/* map the segment so that VFIO has access to it */
-	if (rte_eal_iova_mode() == RTE_IOVA_VA &&
-			rte_vfio_dma_map(ms->addr_64, iova, size))
-		RTE_LOG(DEBUG, EAL, "Cannot register segment with VFIO\n");
 	return 0;
 
 mapped:

@@ -515,12 +510,6 @@ free_page(struct rte_memseg *ms, struct hugepage_info *hi,
 	char path[PATH_MAX];
 	int fd, ret;
 
-	/* unmap the segment from VFIO */
-	if (rte_eal_iova_mode() == RTE_IOVA_VA &&
-			rte_vfio_dma_unmap(ms->addr_64, ms->iova, ms->len)) {
-		RTE_LOG(DEBUG, EAL, "Cannot unregister segment with VFIO\n");
-	}
-
 	if (mmap(ms->addr, ms->hugepage_sz, PROT_READ,
 			MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0) ==
 				MAP_FAILED) {

@@ -808,6 +797,19 @@ sync_chunk(struct rte_memseg_list *primary_msl,
 
 	diff_len = RTE_MIN(chunk_len, diff_len);
 
+	/* if we are freeing memory, notif the application */
+	if (!used) {
+		struct rte_memseg *ms;
+		void *start_va;
+		size_t len;
+
+		ms = rte_fbarray_get(l_arr, start);
+		start_va = ms->addr;
+		len = ms->len * diff_len;
+
+		eal_memalloc_notify(RTE_MEM_EVENT_FREE, start_va, len);
+	}
+
 	for (i = 0; i < diff_len; i++) {
 		struct rte_memseg *p_ms, *l_ms;
 		int seg_idx = start + i;

@@ -834,6 +836,19 @@ sync_chunk(struct rte_memseg_list *primary_msl,
 		}
 	}
 
+	/* if we just allocated memory, notify the application */
+	if (used) {
+		struct rte_memseg *ms;
+		void *start_va;
+		size_t len;
+
+		ms = rte_fbarray_get(l_arr, start);
+		start_va = ms->addr;
+		len = ms->len * diff_len;
+
+		eal_memalloc_notify(RTE_MEM_EVENT_ALLOC, start_va, len);
+	}
+
 	/* calculate how much we can advance until next chunk */
 	diff_len = used ?
 			rte_fbarray_find_contig_used(l_arr, start) :

index 8fe8984..d3c3b70 100644
@@ -214,6 +214,37 @@ vfio_group_device_count(int vfio_group_fd)
 	return vfio_cfg.vfio_groups[i].devices;
 }
 
+static void
+vfio_mem_event_callback(enum rte_mem_event type, const void *addr, size_t len)
+{
+	struct rte_memseg_list *msl;
+	struct rte_memseg *ms;
+	size_t cur_len = 0;
+	uint64_t pgsz;
+
+	msl = rte_mem_virt2memseg_list(addr);
+	pgsz = msl->hugepage_sz;
+
+	while (cur_len < len) {
+		const void *va = RTE_PTR_ADD(addr, cur_len);
+		uint64_t vfio_va, iova;
+
+		ms = rte_mem_virt2memseg(va, msl);
+		vfio_va = (uint64_t) (uintptr_t) va;
+		iova = ms->iova;
+
+		/* this never gets called in legacy mode, so we can be sure that
+		 * each segment is a single page.
+		 */
+		if (type == RTE_MEM_EVENT_ALLOC)
+			rte_vfio_dma_map(vfio_va, iova, pgsz);
+		else
+			rte_vfio_dma_unmap(vfio_va, iova, pgsz);
+
+		cur_len += pgsz;
+	}
+}
+
 int
 rte_vfio_clear_group(int vfio_group_fd)
 {

@@ -507,6 +538,10 @@ rte_vfio_enable(const char *modname)
 	if (vfio_cfg.vfio_container_fd != -1) {
 		RTE_LOG(NOTICE, EAL, "VFIO support initialized\n");
 		vfio_cfg.vfio_enabled = 1;
+
+		/* register callback for mem events */
+		rte_mem_event_register_callback("vfio_mem_event_clb",
+				vfio_mem_event_callback);
 	} else {
 		RTE_LOG(NOTICE, EAL, "VFIO support could not be initialized\n");
 	}

index 87b9ce0..3f8ed4d 100644
@@ -95,8 +95,8 @@ queue_dma_zone_reserve(const char *queue_name, uint32_t queue_size,
 	default:
 		memzone_flags = RTE_MEMZONE_SIZE_HINT_ONLY;
 	}
-	return rte_memzone_reserve_aligned(queue_name, queue_size, socket_id,
-		memzone_flags, queue_size);
+	return rte_memzone_reserve_aligned_contig(queue_name, queue_size,
+		socket_id, memzone_flags, queue_size);
 }
 
 int qat_crypto_sym_qp_setup(struct rte_cryptodev *dev, uint16_t queue_pair_id,

859 lines not on this page · raw message

index 4708dd5..1c6048b 100644
@@ -13,6 +13,7 @@
 
 /** Path of rte config file. */
 #define RUNTIME_CONFIG_FMT "%s/.%s_config"
+#define FBARRAY_FMT "%s/%s_%s"
 
 #include <stdint.h>
 #include <limits.h>

@@ -55,6 +56,18 @@ eal_mp_socket_path(void)
 	return buffer;
 }
 
+static inline const char *
+eal_get_fbarray_path(char *buffer, size_t buflen, const char *name) {
+	const char *directory = "/tmp";
+	const char *home_dir = getenv("HOME");
+
+	if (getuid() != 0 && home_dir != NULL)
+		directory = home_dir;
+	snprintf(buffer, buflen - 1, FBARRAY_FMT, directory,
+			internal_config.hugefile_prefix, name);
+	return buffer;
+}
+
 /** Path of hugepage info file. */
 #define HUGEPAGE_INFO_FMT "%s/.%s_hugepage_info"
 

352 lines not on this page · raw message

index bbeeeba..c03e7bc 100644
@@ -34,6 +34,7 @@
 #include <rte_eal.h>
 #include <rte_memory.h>
 #include <rte_spinlock.h>
+#include <rte_vfio.h>
 
 #include "eal_filesystem.h"
 #include "eal_internal_cfg.h"

@@ -476,6 +477,10 @@ alloc_page(struct rte_memseg *ms, void *addr, uint64_t size, int socket_id,
 	ms->iova = iova;
 	ms->socket_id = socket_id;
 
+	/* map the segment so that VFIO has access to it */
+	if (rte_eal_iova_mode() == RTE_IOVA_VA &&
+			rte_vfio_dma_map(ms->addr_64, iova, size))
+		RTE_LOG(DEBUG, EAL, "Cannot register segment with VFIO\n");
 	return 0;
 
 mapped:

@@ -507,6 +512,12 @@ free_page(struct rte_memseg *ms, struct hugepage_info *hi,
 	char path[PATH_MAX];
 	int fd, ret;
 
+	/* unmap the segment from VFIO */
+	if (rte_eal_iova_mode() == RTE_IOVA_VA &&
+			rte_vfio_dma_unmap(ms->addr_64, ms->iova, ms->len)) {
+		RTE_LOG(DEBUG, EAL, "Cannot unregister segment with VFIO\n");
+	}
+
 	if (mmap(ms->addr, ms->hugepage_sz, PROT_READ,
 			MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0) ==
 				MAP_FAILED) {

index 0590f0c..7935230 100644
@@ -3401,7 +3401,8 @@ rte_eth_dma_zone_reserve(const struct rte_eth_dev *dev, const char *ring_name,
 	if (mz)
 		return mz;
 
-	return rte_memzone_reserve_aligned(z_name, size, socket_id, 0, align);
+	return rte_memzone_reserve_aligned_contig(z_name, size, socket_id, 0,
+			align);
 }
 
 int

index fb02d0f..81f5dae 100644
@@ -177,7 +177,7 @@ bnx2x_dma_alloc(struct bnx2x_softc *sc, size_t size, struct bnx2x_dma *dma,
 			rte_get_timer_cycles());
 
 	/* Caller must take care that strlen(mz_name) < RTE_MEMZONE_NAMESIZE */
-	z = rte_memzone_reserve_aligned(mz_name, (uint64_t) (size),
+	z = rte_memzone_reserve_aligned_contig(mz_name, (uint64_t)size,
 					SOCKET_ID_ANY,
 					0, align);
 	if (z == NULL) {

index a0d4ac9..325b94d 100644
@@ -26,7 +26,8 @@ ring_dma_zone_reserve(struct rte_eth_dev *dev, const char *ring_name,
 	if (mz)
 		return mz;
 
-	return rte_memzone_reserve_aligned(z_name, ring_size, socket_id, 0, BNX2X_PAGE_SIZE);
+	return rte_memzone_reserve_aligned_contig(z_name, ring_size, socket_id,
+			0, BNX2X_PAGE_SIZE);
 }
 
 static void

index 3d5aa59..e31474c 100644
@@ -1299,7 +1299,8 @@ static void *alloc_ring(size_t nelem, size_t elem_size,
 	 * handle the maximum ring size is allocated in order to allow for
 	 * resizing in later calls to the queue setup function.
 	 */
-	tz = rte_memzone_reserve_aligned(z_name, len, socket_id, 0, 4096);
+	tz = rte_memzone_reserve_aligned_contig(z_name, len, socket_id, 0,
+			4096);
 	if (!tz)
 		return NULL;
 

index 0be80cb..dbc3fb5 100644
@@ -73,6 +73,7 @@ eal_long_options[] = {
 	{OPT_VDEV,              1, NULL, OPT_VDEV_NUM             },
 	{OPT_VFIO_INTR,         1, NULL, OPT_VFIO_INTR_NUM        },
 	{OPT_VMWARE_TSC_MAP,    0, NULL, OPT_VMWARE_TSC_MAP_NUM   },
+	{OPT_SINGLE_FILE_SEGMENTS, 0, NULL, OPT_SINGLE_FILE_SEGMENTS_NUM},
 	{0,                     0, NULL, 0                        }
 };
 

@@ -1161,6 +1162,9 @@ eal_parse_common_option(int opt, const char *optarg,
 
 		core_parsed = LCORE_OPT_MAP;
 		break;
+	case OPT_SINGLE_FILE_SEGMENTS_NUM:
+		conf->single_file_segments = 1;
+		break;
 
 	/* don't know what to do, leave this to caller */
 	default:

index 4e2c2e6..3e31ac6 100644
@@ -47,6 +47,10 @@ struct internal_config {
 	volatile unsigned force_sockets;
 	volatile uint64_t socket_mem[RTE_MAX_NUMA_NODES]; /**< amount of memory per socket */
 	uintptr_t base_virtaddr;          /**< base address to try and reserve memory from */
+	volatile unsigned single_file_segments;
+	/**< true if storing all pages within single files (per-page-size,
+	 * per-node).
+	 */
 	volatile int syslog_facility;	  /**< facility passed to openlog() */
 	/** default interrupt mode for VFIO */
 	volatile enum rte_intr_mode vfio_intr_mode;

index e86c711..a4b80d5 100644
@@ -55,6 +55,8 @@ enum {
 	OPT_VFIO_INTR_NUM,
 #define OPT_VMWARE_TSC_MAP    "vmware-tsc-map"
 	OPT_VMWARE_TSC_MAP_NUM,
+#define OPT_SINGLE_FILE_SEGMENTS    "single-file-segments"
+	OPT_SINGLE_FILE_SEGMENTS_NUM,
 	OPT_LONG_MAX_NUM
 };
 

index 2ecd07b..c84e6bf 100644
@@ -348,6 +348,7 @@ eal_usage(const char *prgname)
 	       "  --"OPT_BASE_VIRTADDR"     Base virtual address\n"
 	       "  --"OPT_CREATE_UIO_DEV"    Create /dev/uioX (usually done by hotplug)\n"
 	       "  --"OPT_VFIO_INTR"         Interrupt mode for VFIO (legacy|msi|msix)\n"
+	       "  --"OPT_SINGLE_FILE_SEGMENTS" Put all hugepage memory in single files\n"
 	       "\n");
 	/* Allow the application to print its usage message too if hook is set */
 	if ( rte_application_usage_hook ) {

new file mode 100644
index 0000000..be8340b
@@ -0,0 +1,26 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2017-2018 Intel Corporation
+ */
+
+#include <inttypes.h>
+
+#include <rte_log.h>
+#include <rte_memory.h>
+
+#include "eal_memalloc.h"
+
+int
+eal_memalloc_alloc_page_bulk(struct rte_memseg **ms __rte_unused,
+		int __rte_unused n, uint64_t __rte_unused size,
+		int __rte_unused socket, bool __rte_unused exact)
+{
+	RTE_LOG(ERR, EAL, "Memory hotplug not supported on FreeBSD\n");
+	return -1;
+}
+
+struct rte_memseg *
+eal_memalloc_alloc_page(uint64_t __rte_unused size, int __rte_unused socket)
+{
+	RTE_LOG(ERR, EAL, "Memory hotplug not supported on FreeBSD\n");
+	return NULL;
+}

new file mode 100644
index 0000000..c1076cf
@@ -0,0 +1,19 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2017-2018 Intel Corporation
+ */
+
+#ifndef EAL_MEMALLOC_H
+#define EAL_MEMALLOC_H
+
+#include <stdbool.h>
+
+#include <rte_memory.h>
+
+struct rte_memseg *
+eal_memalloc_alloc_page(uint64_t size, int socket);
+
+int
+eal_memalloc_alloc_page_bulk(struct rte_memseg **ms, int n, uint64_t size,
+		int socket, bool exact);
+
+#endif // EAL_MEMALLOC_H

609 lines not on this page · raw message

index 508b417..0fffe2c 100644
@@ -4010,7 +4010,7 @@ i40e_allocate_dma_mem_d(__attribute__((unused)) struct i40e_hw *hw,
 		return I40E_ERR_PARAM;
 
 	snprintf(z_name, sizeof(z_name), "i40e_dma_%"PRIu64, rte_rand());
-	mz = rte_memzone_reserve_bounded(z_name, size, SOCKET_ID_ANY, 0,
+	mz = rte_memzone_reserve_bounded_contig(z_name, size, SOCKET_ID_ANY, 0,
 					 alignment, RTE_PGSIZE_2M);
 	if (!mz)
 		return I40E_ERR_NO_MEMORY;

index 1217e5a..6b2b40e 100644
@@ -2189,7 +2189,7 @@ i40e_memzone_reserve(const char *name, uint32_t len, int socket_id)
 	if (mz)
 		return mz;
 
-	mz = rte_memzone_reserve_aligned(name, len,
+	mz = rte_memzone_reserve_aligned_contig(name, len,
 					 socket_id, 0, I40E_RING_BASE_ALIGN);
 	return mz;
 }

index fe42f32..707d553 100644
@@ -135,7 +135,7 @@ void *osal_dma_alloc_coherent(struct ecore_dev *p_dev,
 	if (core_id == (unsigned int)LCORE_ID_ANY)
 		core_id = 0;
 	socket_id = rte_lcore_to_socket_id(core_id);
-	mz = rte_memzone_reserve_aligned(mz_name, size,
+	mz = rte_memzone_reserve_aligned_contig(mz_name, size,
 					 socket_id, 0, RTE_CACHE_LINE_SIZE);
 	if (!mz) {
 		DP_ERR(p_dev, "Unable to allocate DMA memory "

@@ -174,7 +174,8 @@ void *osal_dma_alloc_coherent_aligned(struct ecore_dev *p_dev,
 	if (core_id == (unsigned int)LCORE_ID_ANY)
 		core_id = 0;
 	socket_id = rte_lcore_to_socket_id(core_id);
-	mz = rte_memzone_reserve_aligned(mz_name, size, socket_id, 0, align);
+	mz = rte_memzone_reserve_aligned_contig(mz_name, size, socket_id, 0,
+			align);
 	if (!mz) {
 		DP_ERR(p_dev, "Unable to allocate DMA memory "
 		       "of size %zu bytes - %s\n",

index 706b6d5..7e2475f 100644
@@ -15,6 +15,7 @@
 #include <unistd.h>
 #include <errno.h>
 #include <sys/queue.h>
+#include <sys/stat.h>
 
 #include <rte_memory.h>
 #include <rte_eal.h>

@@ -200,6 +201,18 @@ get_hugepage_dir(uint64_t hugepage_sz)
 }
 
 /*
+ * uses fstat to report the size of a file on disk
+ */
+static off_t
+getFileSize(int fd)
+{
+	struct stat st;
+	if (fstat(fd, &st) < 0)
+		return 0;
+	return st.st_size;
+}
+
+/*
  * Clear the hugepage directory of whatever hugepage files
  * there are. Checks if the file is locked (i.e.
  * if it's in use by another DPDK process).

@@ -229,6 +242,8 @@ clear_hugedir(const char * hugedir)
 	}
 
 	while(dirent != NULL){
+		struct flock lck = {0};
+
 		/* skip files that don't match the hugepage pattern */
 		if (fnmatch(filter, dirent->d_name, 0) > 0) {
 			dirent = readdir(dir);

@@ -245,11 +260,17 @@ clear_hugedir(const char * hugedir)
 		}
 
 		/* non-blocking lock */
-		lck_result = flock(fd, LOCK_EX | LOCK_NB);
+		lck.l_type = F_RDLCK;
+		lck.l_whence = SEEK_SET;
+		lck.l_start = 0;
+		lck.l_len = getFileSize(fd);
+
+		lck_result = fcntl(fd, F_SETLK, &lck);
 
 		/* if lock succeeds, unlock and remove the file */
 		if (lck_result != -1) {
-			flock(fd, LOCK_UN);
+			lck.l_type = F_UNLCK;
+			fcntl(fd, F_SETLK, &lck);
 			unlinkat(dir_fd, dirent->d_name, 0);
 		}
 		close (fd);

110 lines not on this page · raw message

index ed36174..718dee8 100644
@@ -103,7 +103,6 @@ memzone_reserve_aligned_thread_unsafe(const char *name, size_t len,
 	struct rte_memzone *mz;
 	struct rte_mem_config *mcfg;
 	size_t requested_len;
-	int socket, i;
 
 	/* get pointer to global configuration */
 	mcfg = rte_eal_get_configuration()->mem_config;

@@ -181,27 +180,9 @@ memzone_reserve_aligned_thread_unsafe(const char *name, size_t len,
 		}
 	}
 
-	if (socket_id == SOCKET_ID_ANY)
-		socket = malloc_get_numa_socket();
-	else
-		socket = socket_id;
-
 	/* allocate memory on heap */
-	void *mz_addr = malloc_heap_alloc(&mcfg->malloc_heaps[socket], NULL,
-			requested_len, flags, align, bound);
-
-	if ((mz_addr == NULL) && (socket_id == SOCKET_ID_ANY)) {
-		/* try other heaps */
-		for (i = 0; i < RTE_MAX_NUMA_NODES; i++) {
-			if (socket == i)
-				continue;
-
-			mz_addr = malloc_heap_alloc(&mcfg->malloc_heaps[i],
-					NULL, requested_len, flags, align, bound);
-			if (mz_addr != NULL)
-				break;
-		}
-	}
+	void *mz_addr = malloc_heap_alloc(NULL, requested_len, socket_id, flags,
+			align, bound);
 
 	if (mz_addr == NULL) {
 		rte_errno = ENOMEM;

91 lines not on this page · raw message

index 388c16f..6d979d2 100644
@@ -152,6 +152,9 @@ int
 malloc_elem_resize(struct malloc_elem *elem, size_t size);
 
 void
+malloc_elem_hide_region(struct malloc_elem *elem, void *start, size_t len);
+
+void
 malloc_elem_free_list_remove(struct malloc_elem *elem);
 
 /*

377 lines not on this page · raw message

index bb28422..292d578 100644
@@ -24,8 +24,8 @@ malloc_get_numa_socket(void)
 }
 
 void *
-malloc_heap_alloc(struct malloc_heap *heap,	const char *type, size_t size,
-		unsigned flags, size_t align, size_t bound);
+malloc_heap_alloc(const char *type, size_t size, int socket, unsigned int flags,
+		size_t align, size_t bound);
 
 int
 malloc_heap_free(struct malloc_elem *elem);

index bd7e757..b0fe11c 100644
@@ -39,10 +39,6 @@ void rte_free(void *addr)
 void *
 rte_malloc_socket(const char *type, size_t size, unsigned align, int socket_arg)
 {
-	struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
-	int socket, i;
-	void *ret;
-
 	/* return NULL if size is 0 or alignment is not power-of-2 */
 	if (size == 0 || (align && !rte_is_power_of_2(align)))
 		return NULL;

@@ -50,33 +46,12 @@ rte_malloc_socket(const char *type, size_t size, unsigned align, int socket_arg)
 	if (!rte_eal_has_hugepages())
 		socket_arg = SOCKET_ID_ANY;
 
-	if (socket_arg == SOCKET_ID_ANY)
-		socket = malloc_get_numa_socket();
-	else
-		socket = socket_arg;
-
 	/* Check socket parameter */
-	if (socket >= RTE_MAX_NUMA_NODES)
+	if (socket_arg >= RTE_MAX_NUMA_NODES)
 		return NULL;
 
-	ret = malloc_heap_alloc(&mcfg->malloc_heaps[socket], type,
-				size, 0, align == 0 ? 1 : align, 0);
-	if (ret != NULL || socket_arg != SOCKET_ID_ANY)
-		return ret;
-
-	/* try other heaps */
-	for (i = 0; i < RTE_MAX_NUMA_NODES; i++) {
-		/* we already tried this one */
-		if (i == socket)
-			continue;
-
-		ret = malloc_heap_alloc(&mcfg->malloc_heaps[i], type,
-					size, 0, align == 0 ? 1 : align, 0);
-		if (ret != NULL)
-			return ret;
-	}
-
-	return NULL;
+	return malloc_heap_alloc(type, size, socket_arg, 0,
+			align == 0 ? 1 : align, 0);
 }
 
 /*

index 8484fb6..2aaf1b8 100644
@@ -22,6 +22,8 @@
 #include <rte_random.h>
 #include <rte_string_fns.h>
 
+#include "../../lib/librte_eal/common/eal_memalloc.h"
+
 #include "test.h"
 
 #define N 10000

@@ -708,22 +710,56 @@ test_malloc_bad_params(void)
 
 /* Check if memory is avilable on a specific socket */
 static int
-is_mem_on_socket(int32_t socket)
+is_mem_on_socket(unsigned int socket)
 {
+	struct rte_malloc_socket_stats stats;
 	const struct rte_mem_config *mcfg =
 			rte_eal_get_configuration()->mem_config;
-	unsigned i;
+	uint64_t prev_pgsz;
+	unsigned int i;
+
+	/* we cannot know if there's memory on a specific socket, since it might
+	 * be available, but not yet allocated. so, in addition to checking
+	 * already mapped memory, we will attempt to allocate a page from that
+	 * socket and see if it works.
+	 */
+	if (socket >= rte_num_sockets())
+		return 0;
 
+	rte_malloc_get_socket_stats(socket, &stats);
+
+	/* if heap has memory allocated, stop */
+	if (stats.heap_totalsz_bytes > 0)
+		return 1;
+
+	/* to allocate a page, we will have to know its size, so go through all
+	 * supported page sizes and try with each one.
+	 */
+	prev_pgsz = 0;
 	for (i = 0; i < RTE_MAX_MEMSEG_LISTS; i++) {
-		const struct rte_memseg_list *msl =
-				&mcfg->memsegs[i];
-		const struct rte_fbarray *arr = &msl->memseg_arr;
+		const struct rte_memseg_list *msl = &mcfg->memsegs[i];
+		uint64_t page_sz;
 
-		if (msl->socket_id != socket)
+		/* skip unused memseg lists */
+		if (msl->memseg_arr.len == 0)
 			continue;
+		page_sz = msl->hugepage_sz;
 
-		if (arr->count)
-			return 1;
+		/* skip page sizes we've tried already */
+		if (prev_pgsz == page_sz)
+			continue;
+
+		prev_pgsz = page_sz;
+
+		struct rte_memseg *ms = eal_memalloc_alloc_page(page_sz,
+				socket);
+
+		if (ms == NULL)
+			continue;
+
+		eal_memalloc_free_page(ms);
+
+		return 1;
 	}
 	return 0;
 }

index 54f7f4b..5c4d3fd 100644
@@ -98,6 +98,27 @@ static unsigned optimize_object_size(unsigned obj_size)
 	return new_obj_size * RTE_MEMPOOL_ALIGN;
 }
 
+static size_t
+get_min_page_size(void)
+{
+	const struct rte_mem_config *mcfg =
+			rte_eal_get_configuration()->mem_config;
+	int i;
+	size_t min_pagesz = SIZE_MAX;
+
+	for (i = 0; i < RTE_MAX_MEMSEG_LISTS; i++) {
+		const struct rte_memseg_list *msl = &mcfg->memsegs[i];
+
+		if (msl->base_va == NULL)
+			continue;
+
+		if (msl->hugepage_sz < min_pagesz)
+			min_pagesz = msl->hugepage_sz;
+	}
+
+	return min_pagesz == SIZE_MAX ? (size_t) getpagesize() : min_pagesz;
+}
+
 static void
 mempool_add_elem(struct rte_mempool *mp, void *obj, rte_iova_t iova)
 {

@@ -549,6 +570,7 @@ rte_mempool_populate_default(struct rte_mempool *mp)
 	unsigned mz_id, n;
 	unsigned int mp_flags;
 	int ret;
+	bool force_contig, no_contig;
 
 	/* mempool must not be populated */
 	if (mp->nb_mem_chunks != 0)

@@ -563,10 +585,46 @@ rte_mempool_populate_default(struct rte_mempool *mp)
 	/* update mempool capabilities */
 	mp->flags |= mp_flags;
 
-	if (rte_eal_has_hugepages()) {
-		pg_shift = 0; /* not needed, zone is physically contiguous */
+	no_contig = mp->flags & MEMPOOL_F_NO_PHYS_CONTIG;
+	force_contig = mp->flags & MEMPOOL_F_CAPA_PHYS_CONTIG;
+
+	/*
+	 * there are several considerations for page size and page shift here.
+	 *
+	 * if we don't need our mempools to have physically contiguous objects,
+	 * then just set page shift and page size to 0, because the user has
+	 * indicated that there's no need to care about anything.
+	 *
+	 * if we do need contiguous objects, there is also an option to reserve
+	 * the entire mempool memory as one contiguous block of memory, in
+	 * which case the page shift and alignment wouldn't matter as well.
+	 *
+	 * if we require contiguous objects, but not necessarily the entire
+	 * mempool reserved space to be contiguous, then there are two options.
+	 *
+	 * if our IO addresses are virtual, not actual physical (IOVA as VA
+	 * case), then no page shift needed - our memory allocation will give us
+	 * contiguous physical memory as far as the hardware is concerned, so
+	 * act as if we're getting contiguous memory.
+	 *
+	 * if our IO addresses are physical, we may get memory from bigger
+	 * pages, or we might get memory from smaller pages, and how much of it
+	 * we require depends on whether we want bigger or smaller pages.
+	 * However, requesting each and every memory size is too much work, so
+	 * what we'll do instead is walk through the page sizes available, pick
+	 * the smallest one and set up page shift to match that one. We will be
+	 * wasting some space this way, but it's much nicer than looping around
+	 * trying to reserve each and every page size.
+	 */
+
+	if (no_contig || force_contig || rte_eal_iova_mode() == RTE_IOVA_VA) {
 		pg_sz = 0;
+		pg_shift = 0;
 		align = RTE_CACHE_LINE_SIZE;
+	} else if (rte_eal_has_hugepages()) {
+		pg_sz = get_min_page_size();
+		pg_shift = rte_bsf32(pg_sz);
+		align = pg_sz;
 	} else {
 		pg_sz = getpagesize();
 		pg_shift = rte_bsf32(pg_sz);

@@ -585,23 +643,34 @@ rte_mempool_populate_default(struct rte_mempool *mp)
 			goto fail;
 		}
 
-		mz = rte_memzone_reserve_aligned(mz_name, size,
-			mp->socket_id, mz_flags, align);
-		/* not enough memory, retry with the biggest zone we have */
-		if (mz == NULL)
-			mz = rte_memzone_reserve_aligned(mz_name, 0,
+		if (force_contig) {
+			/*
+			 * if contiguous memory for entire mempool memory was
+			 * requested, don't try reserving again if we fail.
+			 */
+			mz = rte_memzone_reserve_aligned_contig(mz_name, size,
+				mp->socket_id, mz_flags, align);
+		} else {
+			mz = rte_memzone_reserve_aligned(mz_name, size,
 				mp->socket_id, mz_flags, align);
+			/* not enough memory, retry with the biggest zone we
+			 * have
+			 */
+			if (mz == NULL)
+				mz = rte_memzone_reserve_aligned(mz_name, 0,
+					mp->socket_id, mz_flags, align);
+		}
 		if (mz == NULL) {
 			ret = -rte_errno;
 			goto fail;
 		}
 
-		if (mp->flags & MEMPOOL_F_NO_PHYS_CONTIG)
+		if (no_contig)
 			iova = RTE_BAD_IOVA;
 		else
 			iova = mz->iova;
 
-		if (rte_eal_has_hugepages())
+		if (rte_eal_has_hugepages() && force_contig)
 			ret = rte_mempool_populate_iova(mp, mz->addr,
 				iova, mz->len,
 				rte_mempool_memchunk_mz_free,