Improve running DPDK without hugetlbfs mounpoint

index 43caf3ced..ba6c4ae39 100644
@@ -878,6 +878,10 @@ rte_fbarray_destroy(struct rte_fbarray *arr)
 	if (ret)
 		return ret;
 
+	/* with no shconf, there were never any files to begin with */
+	if (internal_config.no_shconf)
+		return 0;
+
 	/* try deleting the file */
 	eal_get_fbarray_path(path, sizeof(path), arr->name);
 

index dd5f97402..873099acc 100644
@@ -1390,6 +1390,12 @@ eal_check_common_options(struct internal_config *internal_cfg)
 			"--"OPT_HUGE_UNLINK"\n");
 		return -1;
 	}
+	if (internal_cfg->legacy_mem &&
+			internal_cfg->in_memory) {
+		RTE_LOG(ERR, EAL, "Option --"OPT_LEGACY_MEM" is not compatible "
+				"with --"OPT_IN_MEMORY"\n");
+		return -1;
+	}
 
 	return 0;
 }

index dbf19499e..dfb537f59 100644
@@ -17,6 +17,7 @@
 #include <sys/stat.h>
 #include <sys/queue.h>
 #include <sys/file.h>
+#include <sys/resource.h>
 #include <unistd.h>
 #include <limits.h>
 #include <sys/ioctl.h>

@@ -2204,6 +2205,25 @@ memseg_secondary_init(void)
 int
 rte_eal_memseg_init(void)
 {
+	/* increase rlimit to maximum */
+	struct rlimit lim;
+
+	if (getrlimit(RLIMIT_NOFILE, &lim) == 0) {
+		/* set limit to maximum */
+		lim.rlim_cur = lim.rlim_max;
+
+		if (setrlimit(RLIMIT_NOFILE, &lim) < 0) {
+			RTE_LOG(DEBUG, EAL, "Setting maximum number of open files failed: %s\n",
+					strerror(errno));
+		} else {
+			RTE_LOG(DEBUG, EAL, "Setting maximum number of open files to %"
+					PRIu64 "\n",
+					(uint64_t)lim.rlim_cur);
+		}
+	} else {
+		RTE_LOG(ERR, EAL, "Cannot get current resource limits\n");
+	}
+
 	return rte_eal_process_type() == RTE_PROC_PRIMARY ?
 #ifndef RTE_ARCH_64
 			memseg_primary_init_32() :

index aa95551a8..14bc5dce9 100644
@@ -57,25 +57,33 @@ const int anonymous_hugepages_supported =
  */
 static int fallocate_supported = -1; /* unknown */
 
-/* for single-file segments, we need some kind of mechanism to keep track of
+/*
+ * we have two modes - single file segments, and file-per-page mode.
+ *
+ * for single-file segments, we need some kind of mechanism to keep track of
  * which hugepages can be freed back to the system, and which cannot. we cannot
  * use flock() because they don't allow locking parts of a file, and we cannot
  * use fcntl() due to issues with their semantics, so we will have to rely on a
- * bunch of lockfiles for each page.
+ * bunch of lockfiles for each page. so, we will use 'fds' array to keep track
+ * of per-page lockfiles. we will store the actual segment list fd in the
+ * 'memseg_list_fd' field.
+ *
+ * for file-per-page mode, each page will have its own fd, so 'memseg_list_fd'
+ * will be invalid (set to -1), and we'll use 'fds' to keep track of page fd's.
  *
  * we cannot know how many pages a system will have in advance, but we do know
  * that they come in lists, and we know lengths of these lists. so, simply store
  * a malloc'd array of fd's indexed by list and segment index.
  *
  * they will be initialized at startup, and filled as we allocate/deallocate
- * segments. also, use this to track memseg list proper fd.
+ * segments.
  */
 static struct {
 	int *fds; /**< dynamically allocated array of segment lock fd's */
 	int memseg_list_fd; /**< memseg list fd */
 	int len; /**< total length of the array */
 	int count; /**< entries used in an array */
-} lock_fds[RTE_MAX_MEMSEG_LISTS];
+} fd_list[RTE_MAX_MEMSEG_LISTS];
 
 /** local copy of a memory map, used to synchronize memory hotplug in MP */
 static struct rte_memseg_list local_memsegs[RTE_MAX_MEMSEG_LISTS];

@@ -209,12 +217,12 @@ static int get_segment_lock_fd(int list_idx, int seg_idx)
 	char path[PATH_MAX] = {0};
 	int fd;
 
-	if (list_idx < 0 || list_idx >= (int)RTE_DIM(lock_fds))
+	if (list_idx < 0 || list_idx >= (int)RTE_DIM(fd_list))
 		return -1;
-	if (seg_idx < 0 || seg_idx >= lock_fds[list_idx].len)
+	if (seg_idx < 0 || seg_idx >= fd_list[list_idx].len)
 		return -1;
 
-	fd = lock_fds[list_idx].fds[seg_idx];
+	fd = fd_list[list_idx].fds[seg_idx];
 	/* does this lock already exist? */
 	if (fd >= 0)
 		return fd;

@@ -236,8 +244,8 @@ static int get_segment_lock_fd(int list_idx, int seg_idx)
 		return -1;
 	}
 	/* store it for future reference */
-	lock_fds[list_idx].fds[seg_idx] = fd;
-	lock_fds[list_idx].count++;
+	fd_list[list_idx].fds[seg_idx] = fd;
+	fd_list[list_idx].count++;
 	return fd;
 }
 

@@ -245,12 +253,12 @@ static int unlock_segment(int list_idx, int seg_idx)
 {
 	int fd, ret;
 
-	if (list_idx < 0 || list_idx >= (int)RTE_DIM(lock_fds))
+	if (list_idx < 0 || list_idx >= (int)RTE_DIM(fd_list))
 		return -1;
-	if (seg_idx < 0 || seg_idx >= lock_fds[list_idx].len)
+	if (seg_idx < 0 || seg_idx >= fd_list[list_idx].len)
 		return -1;
 
-	fd = lock_fds[list_idx].fds[seg_idx];
+	fd = fd_list[list_idx].fds[seg_idx];
 
 	/* upgrade lock to exclusive to see if we can remove the lockfile */
 	ret = lock(fd, LOCK_EX);

@@ -270,8 +278,8 @@ static int unlock_segment(int list_idx, int seg_idx)
 	 * and remove it from list anyway.
 	 */
 	close(fd);
-	lock_fds[list_idx].fds[seg_idx] = -1;
-	lock_fds[list_idx].count--;
+	fd_list[list_idx].fds[seg_idx] = -1;
+	fd_list[list_idx].count--;
 
 	if (ret < 0)
 		return -1;

@@ -288,7 +296,7 @@ get_seg_fd(char *path, int buflen, struct hugepage_info *hi,
 		/* create a hugepage file path */
 		eal_get_hugefile_path(path, buflen, hi->hugedir, list_idx);
 
-		fd = lock_fds[list_idx].memseg_list_fd;
+		fd = fd_list[list_idx].memseg_list_fd;
 
 		if (fd < 0) {
 			fd = open(path, O_CREAT | O_RDWR, 0600);

@@ -304,7 +312,7 @@ get_seg_fd(char *path, int buflen, struct hugepage_info *hi,
 				close(fd);
 				return -1;
 			}
-			lock_fds[list_idx].memseg_list_fd = fd;
+			fd_list[list_idx].memseg_list_fd = fd;
 		}
 	} else {
 		/* create a hugepage file path */

@@ -410,9 +418,9 @@ resize_hugefile(int fd, char *path, int list_idx, int seg_idx,
 				 * page file fd, so that one of the processes
 				 * could then delete the file after shrinking.
 				 */
-				if (ret < 1 && lock_fds[list_idx].count == 0) {
+				if (ret < 1 && fd_list[list_idx].count == 0) {
 					close(fd);
-					lock_fds[list_idx].memseg_list_fd = -1;
+					fd_list[list_idx].memseg_list_fd = -1;
 				}
 
 				if (ret < 0) {

@@ -448,13 +456,13 @@ resize_hugefile(int fd, char *path, int list_idx, int seg_idx,
 				 * more segments active in this segment list,
 				 * and remove the file if there aren't.
 				 */
-				if (lock_fds[list_idx].count == 0) {
+				if (fd_list[list_idx].count == 0) {
 					if (unlink(path))
 						RTE_LOG(ERR, EAL, "%s(): unlinking '%s' failed: %s\n",
 							__func__, path,
 							strerror(errno));
 					close(fd);
-					lock_fds[list_idx].memseg_list_fd = -1;
+					fd_list[list_idx].memseg_list_fd = -1;
 				}
 			}
 		}

@@ -1319,7 +1327,7 @@ secondary_msl_create_walk(const struct rte_memseg_list *msl,
 }
 
 static int
-secondary_lock_list_create_walk(const struct rte_memseg_list *msl,
+fd_list_create_walk(const struct rte_memseg_list *msl,
 		void *arg __rte_unused)
 {
 	struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;

@@ -1330,20 +1338,20 @@ secondary_lock_list_create_walk(const struct rte_memseg_list *msl,
 	msl_idx = msl - mcfg->memsegs;
 	len = msl->memseg_arr.len;
 
-	/* ensure we have space to store lock fd per each possible segment */
+	/* ensure we have space to store fd per each possible segment */
 	data = malloc(sizeof(int) * len);
 	if (data == NULL) {
-		RTE_LOG(ERR, EAL, "Unable to allocate space for lock descriptors\n");
+		RTE_LOG(ERR, EAL, "Unable to allocate space for file descriptors\n");
 		return -1;
 	}
 	/* set all fd's as invalid */
 	for (i = 0; i < len; i++)
 		data[i] = -1;
 
-	lock_fds[msl_idx].fds = data;
-	lock_fds[msl_idx].len = len;
-	lock_fds[msl_idx].count = 0;
-	lock_fds[msl_idx].memseg_list_fd = -1;
+	fd_list[msl_idx].fds = data;
+	fd_list[msl_idx].len = len;
+	fd_list[msl_idx].count = 0;
+	fd_list[msl_idx].memseg_list_fd = -1;
 
 	return 0;
 }

@@ -1355,9 +1363,9 @@ eal_memalloc_init(void)
 		if (rte_memseg_list_walk(secondary_msl_create_walk, NULL) < 0)
 			return -1;
 
-	/* initialize all of the lock fd lists */
+	/* initialize all of the fd lists */
 	if (internal_config.single_file_segments)
-		if (rte_memseg_list_walk(secondary_lock_list_create_walk, NULL))
+		if (rte_memseg_list_walk(fd_list_create_walk, NULL))
 			return -1;
 	return 0;
 }

index 14bc5dce9..7d536350e 100644
@@ -318,18 +318,24 @@ get_seg_fd(char *path, int buflen, struct hugepage_info *hi,
 		/* create a hugepage file path */
 		eal_get_hugefile_path(path, buflen, hi->hugedir,
 				list_idx * RTE_MAX_MEMSEG_PER_LIST + seg_idx);
-		fd = open(path, O_CREAT | O_RDWR, 0600);
+
+		fd = fd_list[list_idx].fds[seg_idx];
+
 		if (fd < 0) {
-			RTE_LOG(DEBUG, EAL, "%s(): open failed: %s\n", __func__,
-					strerror(errno));
-			return -1;
-		}
-		/* take out a read lock */
-		if (lock(fd, LOCK_SH) < 0) {
-			RTE_LOG(ERR, EAL, "%s(): lock failed: %s\n",
-				__func__, strerror(errno));
-			close(fd);
-			return -1;
+			fd = open(path, O_CREAT | O_RDWR, 0600);
+			if (fd < 0) {
+				RTE_LOG(DEBUG, EAL, "%s(): open failed: %s\n",
+					__func__, strerror(errno));
+				return -1;
+			}
+			/* take out a read lock */
+			if (lock(fd, LOCK_SH) < 0) {
+				RTE_LOG(ERR, EAL, "%s(): lock failed: %s\n",
+					__func__, strerror(errno));
+				close(fd);
+				return -1;
+			}
+			fd_list[list_idx].fds[seg_idx] = fd;
 		}
 	}
 	return fd;

@@ -601,10 +607,6 @@ alloc_seg(struct rte_memseg *ms, void *addr, int socket_id,
 		goto mapped;
 	}
 #endif
-	/* for non-single file segments that aren't in-memory, we can close fd
-	 * here */
-	if (!internal_config.single_file_segments && !internal_config.in_memory)
-		close(fd);
 
 	ms->addr = addr;
 	ms->hugepage_sz = alloc_sz;

@@ -634,7 +636,10 @@ alloc_seg(struct rte_memseg *ms, void *addr, int socket_id,
 		RTE_LOG(CRIT, EAL, "Can't mmap holes in our virtual address space\n");
 	}
 resized:
-	/* in-memory mode will never be single-file-segments mode */
+	/* some codepaths will return negative fd, so exit early */
+	if (fd < 0)
+		return -1;
+
 	if (internal_config.single_file_segments) {
 		resize_hugefile(fd, path, list_idx, seg_idx, map_offset,
 				alloc_sz, false);

@@ -646,6 +651,7 @@ alloc_seg(struct rte_memseg *ms, void *addr, int socket_id,
 				lock(fd, LOCK_EX) == 1)
 			unlink(path);
 		close(fd);
+		fd_list[list_idx].fds[seg_idx] = -1;
 	}
 	return -1;
 }

@@ -700,6 +706,7 @@ free_seg(struct rte_memseg *ms, struct hugepage_info *hi,
 		}
 		/* closing fd will drop the lock */
 		close(fd);
+		fd_list[list_idx].fds[seg_idx] = -1;
 	}
 
 	memset(ms, 0, sizeof(*ms));

@@ -1364,8 +1371,7 @@ eal_memalloc_init(void)
 			return -1;
 
 	/* initialize all of the fd lists */
-	if (internal_config.single_file_segments)
-		if (rte_memseg_list_walk(fd_list_create_walk, NULL))
-			return -1;
+	if (rte_memseg_list_walk(fd_list_create_walk, NULL))
+		return -1;
 	return 0;
 }

index f7f07abd6..a5fb09f71 100644
@@ -47,6 +47,18 @@ eal_memalloc_sync_with_primary(void)
 	return -1;
 }
 
+int
+eal_memalloc_get_seg_fd(int list_idx, int seg_idx)
+{
+	return -1;
+}
+
+int
+eal_memalloc_set_seg_fd(int list_idx, int seg_idx, int fd)
+{
+	return -1;
+}
+
 int
 eal_memalloc_init(void)
 {

index fbfb1b055..034c2026a 100644
@@ -294,7 +294,7 @@ dump_memseg(const struct rte_memseg_list *msl, const struct rte_memseg *ms,
 		void *arg)
 {
 	struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
-	int msl_idx, ms_idx;
+	int msl_idx, ms_idx, fd;
 	FILE *f = arg;
 
 	msl_idx = msl - mcfg->memsegs;

@@ -305,10 +305,11 @@ dump_memseg(const struct rte_memseg_list *msl, const struct rte_memseg *ms,
 	if (ms_idx < 0)
 		return -1;
 
+	fd = eal_memalloc_get_seg_fd(msl_idx, ms_idx);
 	fprintf(f, "Segment %i-%i: IOVA:0x%"PRIx64", len:%zu, "
 			"virt:%p, socket_id:%"PRId32", "
 			"hugepage_sz:%"PRIu64", nchannel:%"PRIx32", "
-			"nrank:%"PRIx32"\n",
+			"nrank:%"PRIx32" fd:%i\n",
 			msl_idx, ms_idx,
 			ms->iova,
 			ms->len,

@@ -316,7 +317,8 @@ dump_memseg(const struct rte_memseg_list *msl, const struct rte_memseg *ms,
 			ms->socket_id,
 			ms->hugepage_sz,
 			ms->nchannel,
-			ms->nrank);
+			ms->nrank,
+			fd);
 
 	return 0;
 }

index 36bb1a027..a46c69c72 100644
@@ -76,6 +76,12 @@ eal_memalloc_mem_alloc_validator_unregister(const char *name, int socket_id);
 int
 eal_memalloc_mem_alloc_validate(int socket_id, size_t new_len);
 
+int
+eal_memalloc_get_seg_fd(int list_idx, int seg_idx);
+
+int
+eal_memalloc_set_seg_fd(int list_idx, int seg_idx, int fd);
+
 int
 eal_memalloc_init(void);
 

78 lines not on this page · raw message

index dfb537f59..ace14c877 100644
@@ -772,7 +772,8 @@ remap_segment(struct hugepage_file *hugepages, int seg_start, int seg_end)
 
 		rte_fbarray_set_used(arr, ms_idx);
 
-		close(fd);
+		/* store segment fd internally */
+		eal_memalloc_set_seg_fd(msl_idx, ms_idx, fd);
 	}
 	RTE_LOG(DEBUG, EAL, "Allocated %" PRIu64 "M on socket %i\n",
 			(seg_len * page_sz) >> 20, socket_id);

@@ -1771,6 +1772,7 @@ getFileSize(int fd)
 static int
 eal_legacy_hugepage_attach(void)
 {
+	struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
 	struct hugepage_file *hp = NULL;
 	unsigned int num_hp = 0;
 	unsigned int i = 0;

@@ -1814,6 +1816,9 @@ eal_legacy_hugepage_attach(void)
 		struct hugepage_file *hf = &hp[i];
 		size_t map_sz = hf->size;
 		void *map_addr = hf->final_va;
+		int msl_idx, ms_idx;
+		struct rte_memseg_list *msl;
+		struct rte_memseg *ms;
 
 		/* if size is zero, no more pages left */
 		if (map_sz == 0)

@@ -1831,25 +1836,48 @@ eal_legacy_hugepage_attach(void)
 		if (map_addr == MAP_FAILED) {
 			RTE_LOG(ERR, EAL, "Could not map %s: %s\n",
 				hf->filepath, strerror(errno));
-			close(fd);
-			goto error;
+			goto fd_error;
 		}
 
 		/* set shared lock on the file. */
 		if (flock(fd, LOCK_SH) < 0) {
 			RTE_LOG(DEBUG, EAL, "%s(): Locking file failed: %s\n",
 				__func__, strerror(errno));
-			close(fd);
-			goto error;
+			goto fd_error;
 		}
 
-		close(fd);
+		/* find segment data */
+		msl = rte_mem_virt2memseg_list(map_addr);
+		if (msl == NULL) {
+			RTE_LOG(DEBUG, EAL, "%s(): Cannot find memseg list\n",
+				__func__);
+			goto fd_error;
+		}
+		ms = rte_mem_virt2memseg(map_addr, msl);
+		if (ms == NULL) {
+			RTE_LOG(DEBUG, EAL, "%s(): Cannot find memseg\n",
+				__func__);
+			goto fd_error;
+		}
+
+		msl_idx = msl - mcfg->memsegs;
+		ms_idx = rte_fbarray_find_idx(&msl->memseg_arr, ms);
+		if (ms_idx < 0) {
+			RTE_LOG(DEBUG, EAL, "%s(): Cannot find memseg idx\n",
+				__func__);
+			goto fd_error;
+		}
+
+		/* store segment fd internally */
+		eal_memalloc_set_seg_fd(msl_idx, ms_idx, fd);
 	}
 	/* unmap the hugepage config file, since we are done using it */
 	munmap(hp, size);
 	close(fd_hugepage);
 	return 0;
 
+fd_error:
+	close(fd);
 error:
 	/* map all segments into memory to make sure we get the addrs */
 	cur_seg = 0;

index 034c2026a..cd3805b46 100644
@@ -550,6 +550,53 @@ rte_memseg_list_walk(rte_memseg_list_walk_t func, void *arg)
 	return ret;
 }
 
+int __rte_experimental
+rte_memseg_get_fd_thread_unsafe(const struct rte_memseg *ms)
+{
+	struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
+	struct rte_memseg_list *msl;
+	struct rte_fbarray *arr;
+	int msl_idx, seg_idx, fd;
+
+	if (ms == NULL) {
+		rte_errno = EINVAL;
+		return -1;
+	}
+
+	msl = rte_mem_virt2memseg_list(ms->addr);
+	if (msl == NULL) {
+		rte_errno = EINVAL;
+		return -1;
+	}
+	arr = &msl->memseg_arr;
+
+	msl_idx = msl - mcfg->memsegs;
+	seg_idx = rte_fbarray_find_idx(arr, ms);
+
+	if (!rte_fbarray_is_used(arr, seg_idx)) {
+		rte_errno = ENOENT;
+		return -1;
+	}
+
+	fd = eal_memalloc_get_seg_fd(msl_idx, seg_idx);
+	if (fd < 0)
+		rte_errno = ENOENT;
+	return fd;
+}
+
+int __rte_experimental
+rte_memseg_get_fd(const struct rte_memseg *ms)
+{
+	struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
+	int ret;
+
+	rte_rwlock_read_lock(&mcfg->memory_hotplug_lock);
+	ret = rte_memseg_get_fd_thread_unsafe(ms);
+	rte_rwlock_read_unlock(&mcfg->memory_hotplug_lock);
+
+	return ret;
+}
+
 /* init memory subsystem */
 int
 rte_eal_memory_init(void)

50 lines not on this page · raw message

index 873099acc..ddd624110 100644
@@ -1384,10 +1384,10 @@ eal_check_common_options(struct internal_config *internal_cfg)
 			" is only supported in non-legacy memory mode\n");
 	}
 	if (internal_cfg->single_file_segments &&
-			internal_cfg->hugepage_unlink) {
+			internal_cfg->hugepage_unlink &&
+			!internal_cfg->in_memory) {
 		RTE_LOG(ERR, EAL, "Option --"OPT_SINGLE_FILE_SEGMENTS" is "
-			"not compatible with neither --"OPT_IN_MEMORY" nor "
-			"--"OPT_HUGE_UNLINK"\n");
+			"not compatible with --"OPT_HUGE_UNLINK"\n");
 		return -1;
 	}
 	if (internal_cfg->legacy_mem &&

349 lines not on this page · raw message

index 43caf3ced..ba6c4ae39 100644
@@ -878,6 +878,10 @@ rte_fbarray_destroy(struct rte_fbarray *arr)
 	if (ret)
 		return ret;
 
+	/* with no shconf, there were never any files to begin with */
+	if (internal_config.no_shconf)
+		return 0;
+
 	/* try deleting the file */
 	eal_get_fbarray_path(path, sizeof(path), arr->name);
 

index dd5f97402..873099acc 100644
@@ -1390,6 +1390,12 @@ eal_check_common_options(struct internal_config *internal_cfg)
 			"--"OPT_HUGE_UNLINK"\n");
 		return -1;
 	}
+	if (internal_cfg->legacy_mem &&
+			internal_cfg->in_memory) {
+		RTE_LOG(ERR, EAL, "Option --"OPT_LEGACY_MEM" is not compatible "
+				"with --"OPT_IN_MEMORY"\n");
+		return -1;
+	}
 
 	return 0;
 }

index dbf19499e..dfb537f59 100644
@@ -17,6 +17,7 @@
 #include <sys/stat.h>
 #include <sys/queue.h>
 #include <sys/file.h>
+#include <sys/resource.h>
 #include <unistd.h>
 #include <limits.h>
 #include <sys/ioctl.h>

@@ -2204,6 +2205,25 @@ memseg_secondary_init(void)
 int
 rte_eal_memseg_init(void)
 {
+	/* increase rlimit to maximum */
+	struct rlimit lim;
+
+	if (getrlimit(RLIMIT_NOFILE, &lim) == 0) {
+		/* set limit to maximum */
+		lim.rlim_cur = lim.rlim_max;
+
+		if (setrlimit(RLIMIT_NOFILE, &lim) < 0) {
+			RTE_LOG(DEBUG, EAL, "Setting maximum number of open files failed: %s\n",
+					strerror(errno));
+		} else {
+			RTE_LOG(DEBUG, EAL, "Setting maximum number of open files to %"
+					PRIu64 "\n",
+					(uint64_t)lim.rlim_cur);
+		}
+	} else {
+		RTE_LOG(ERR, EAL, "Cannot get current resource limits\n");
+	}
+
 	return rte_eal_process_type() == RTE_PROC_PRIMARY ?
 #ifndef RTE_ARCH_64
 			memseg_primary_init_32() :

index aa95551a8..14bc5dce9 100644
@@ -57,25 +57,33 @@ const int anonymous_hugepages_supported =
  */
 static int fallocate_supported = -1; /* unknown */
 
-/* for single-file segments, we need some kind of mechanism to keep track of
+/*
+ * we have two modes - single file segments, and file-per-page mode.
+ *
+ * for single-file segments, we need some kind of mechanism to keep track of
  * which hugepages can be freed back to the system, and which cannot. we cannot
  * use flock() because they don't allow locking parts of a file, and we cannot
  * use fcntl() due to issues with their semantics, so we will have to rely on a
- * bunch of lockfiles for each page.
+ * bunch of lockfiles for each page. so, we will use 'fds' array to keep track
+ * of per-page lockfiles. we will store the actual segment list fd in the
+ * 'memseg_list_fd' field.
+ *
+ * for file-per-page mode, each page will have its own fd, so 'memseg_list_fd'
+ * will be invalid (set to -1), and we'll use 'fds' to keep track of page fd's.
  *
  * we cannot know how many pages a system will have in advance, but we do know
  * that they come in lists, and we know lengths of these lists. so, simply store
  * a malloc'd array of fd's indexed by list and segment index.
  *
  * they will be initialized at startup, and filled as we allocate/deallocate
- * segments. also, use this to track memseg list proper fd.
+ * segments.
  */
 static struct {
 	int *fds; /**< dynamically allocated array of segment lock fd's */
 	int memseg_list_fd; /**< memseg list fd */
 	int len; /**< total length of the array */
 	int count; /**< entries used in an array */
-} lock_fds[RTE_MAX_MEMSEG_LISTS];
+} fd_list[RTE_MAX_MEMSEG_LISTS];
 
 /** local copy of a memory map, used to synchronize memory hotplug in MP */
 static struct rte_memseg_list local_memsegs[RTE_MAX_MEMSEG_LISTS];

@@ -209,12 +217,12 @@ static int get_segment_lock_fd(int list_idx, int seg_idx)
 	char path[PATH_MAX] = {0};
 	int fd;
 
-	if (list_idx < 0 || list_idx >= (int)RTE_DIM(lock_fds))
+	if (list_idx < 0 || list_idx >= (int)RTE_DIM(fd_list))
 		return -1;
-	if (seg_idx < 0 || seg_idx >= lock_fds[list_idx].len)
+	if (seg_idx < 0 || seg_idx >= fd_list[list_idx].len)
 		return -1;
 
-	fd = lock_fds[list_idx].fds[seg_idx];
+	fd = fd_list[list_idx].fds[seg_idx];
 	/* does this lock already exist? */
 	if (fd >= 0)
 		return fd;

@@ -236,8 +244,8 @@ static int get_segment_lock_fd(int list_idx, int seg_idx)
 		return -1;
 	}
 	/* store it for future reference */
-	lock_fds[list_idx].fds[seg_idx] = fd;
-	lock_fds[list_idx].count++;
+	fd_list[list_idx].fds[seg_idx] = fd;
+	fd_list[list_idx].count++;
 	return fd;
 }
 

@@ -245,12 +253,12 @@ static int unlock_segment(int list_idx, int seg_idx)
 {
 	int fd, ret;
 
-	if (list_idx < 0 || list_idx >= (int)RTE_DIM(lock_fds))
+	if (list_idx < 0 || list_idx >= (int)RTE_DIM(fd_list))
 		return -1;
-	if (seg_idx < 0 || seg_idx >= lock_fds[list_idx].len)
+	if (seg_idx < 0 || seg_idx >= fd_list[list_idx].len)
 		return -1;
 
-	fd = lock_fds[list_idx].fds[seg_idx];
+	fd = fd_list[list_idx].fds[seg_idx];
 
 	/* upgrade lock to exclusive to see if we can remove the lockfile */
 	ret = lock(fd, LOCK_EX);

@@ -270,8 +278,8 @@ static int unlock_segment(int list_idx, int seg_idx)
 	 * and remove it from list anyway.
 	 */
 	close(fd);
-	lock_fds[list_idx].fds[seg_idx] = -1;
-	lock_fds[list_idx].count--;
+	fd_list[list_idx].fds[seg_idx] = -1;
+	fd_list[list_idx].count--;
 
 	if (ret < 0)
 		return -1;

@@ -288,7 +296,7 @@ get_seg_fd(char *path, int buflen, struct hugepage_info *hi,
 		/* create a hugepage file path */
 		eal_get_hugefile_path(path, buflen, hi->hugedir, list_idx);
 
-		fd = lock_fds[list_idx].memseg_list_fd;
+		fd = fd_list[list_idx].memseg_list_fd;
 
 		if (fd < 0) {
 			fd = open(path, O_CREAT | O_RDWR, 0600);

@@ -304,7 +312,7 @@ get_seg_fd(char *path, int buflen, struct hugepage_info *hi,
 				close(fd);
 				return -1;
 			}
-			lock_fds[list_idx].memseg_list_fd = fd;
+			fd_list[list_idx].memseg_list_fd = fd;
 		}
 	} else {
 		/* create a hugepage file path */

@@ -410,9 +418,9 @@ resize_hugefile(int fd, char *path, int list_idx, int seg_idx,
 				 * page file fd, so that one of the processes
 				 * could then delete the file after shrinking.
 				 */
-				if (ret < 1 && lock_fds[list_idx].count == 0) {
+				if (ret < 1 && fd_list[list_idx].count == 0) {
 					close(fd);
-					lock_fds[list_idx].memseg_list_fd = -1;
+					fd_list[list_idx].memseg_list_fd = -1;
 				}
 
 				if (ret < 0) {

@@ -448,13 +456,13 @@ resize_hugefile(int fd, char *path, int list_idx, int seg_idx,
 				 * more segments active in this segment list,
 				 * and remove the file if there aren't.
 				 */
-				if (lock_fds[list_idx].count == 0) {
+				if (fd_list[list_idx].count == 0) {
 					if (unlink(path))
 						RTE_LOG(ERR, EAL, "%s(): unlinking '%s' failed: %s\n",
 							__func__, path,
 							strerror(errno));
 					close(fd);
-					lock_fds[list_idx].memseg_list_fd = -1;
+					fd_list[list_idx].memseg_list_fd = -1;
 				}
 			}
 		}

@@ -1319,7 +1327,7 @@ secondary_msl_create_walk(const struct rte_memseg_list *msl,
 }
 
 static int
-secondary_lock_list_create_walk(const struct rte_memseg_list *msl,
+fd_list_create_walk(const struct rte_memseg_list *msl,
 		void *arg __rte_unused)
 {
 	struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;

@@ -1330,20 +1338,20 @@ secondary_lock_list_create_walk(const struct rte_memseg_list *msl,
 	msl_idx = msl - mcfg->memsegs;
 	len = msl->memseg_arr.len;
 
-	/* ensure we have space to store lock fd per each possible segment */
+	/* ensure we have space to store fd per each possible segment */
 	data = malloc(sizeof(int) * len);
 	if (data == NULL) {
-		RTE_LOG(ERR, EAL, "Unable to allocate space for lock descriptors\n");
+		RTE_LOG(ERR, EAL, "Unable to allocate space for file descriptors\n");
 		return -1;
 	}
 	/* set all fd's as invalid */
 	for (i = 0; i < len; i++)
 		data[i] = -1;
 
-	lock_fds[msl_idx].fds = data;
-	lock_fds[msl_idx].len = len;
-	lock_fds[msl_idx].count = 0;
-	lock_fds[msl_idx].memseg_list_fd = -1;
+	fd_list[msl_idx].fds = data;
+	fd_list[msl_idx].len = len;
+	fd_list[msl_idx].count = 0;
+	fd_list[msl_idx].memseg_list_fd = -1;
 
 	return 0;
 }

@@ -1355,9 +1363,9 @@ eal_memalloc_init(void)
 		if (rte_memseg_list_walk(secondary_msl_create_walk, NULL) < 0)
 			return -1;
 
-	/* initialize all of the lock fd lists */
+	/* initialize all of the fd lists */
 	if (internal_config.single_file_segments)
-		if (rte_memseg_list_walk(secondary_lock_list_create_walk, NULL))
+		if (rte_memseg_list_walk(fd_list_create_walk, NULL))
 			return -1;
 	return 0;
 }

index 14bc5dce9..7d536350e 100644
@@ -318,18 +318,24 @@ get_seg_fd(char *path, int buflen, struct hugepage_info *hi,
 		/* create a hugepage file path */
 		eal_get_hugefile_path(path, buflen, hi->hugedir,
 				list_idx * RTE_MAX_MEMSEG_PER_LIST + seg_idx);
-		fd = open(path, O_CREAT | O_RDWR, 0600);
+
+		fd = fd_list[list_idx].fds[seg_idx];
+
 		if (fd < 0) {
-			RTE_LOG(DEBUG, EAL, "%s(): open failed: %s\n", __func__,
-					strerror(errno));
-			return -1;
-		}
-		/* take out a read lock */
-		if (lock(fd, LOCK_SH) < 0) {
-			RTE_LOG(ERR, EAL, "%s(): lock failed: %s\n",
-				__func__, strerror(errno));
-			close(fd);
-			return -1;
+			fd = open(path, O_CREAT | O_RDWR, 0600);
+			if (fd < 0) {
+				RTE_LOG(DEBUG, EAL, "%s(): open failed: %s\n",
+					__func__, strerror(errno));
+				return -1;
+			}
+			/* take out a read lock */
+			if (lock(fd, LOCK_SH) < 0) {
+				RTE_LOG(ERR, EAL, "%s(): lock failed: %s\n",
+					__func__, strerror(errno));
+				close(fd);
+				return -1;
+			}
+			fd_list[list_idx].fds[seg_idx] = fd;
 		}
 	}
 	return fd;

@@ -601,10 +607,6 @@ alloc_seg(struct rte_memseg *ms, void *addr, int socket_id,
 		goto mapped;
 	}
 #endif
-	/* for non-single file segments that aren't in-memory, we can close fd
-	 * here */
-	if (!internal_config.single_file_segments && !internal_config.in_memory)
-		close(fd);
 
 	ms->addr = addr;
 	ms->hugepage_sz = alloc_sz;

@@ -634,7 +636,10 @@ alloc_seg(struct rte_memseg *ms, void *addr, int socket_id,
 		RTE_LOG(CRIT, EAL, "Can't mmap holes in our virtual address space\n");
 	}
 resized:
-	/* in-memory mode will never be single-file-segments mode */
+	/* some codepaths will return negative fd, so exit early */
+	if (fd < 0)
+		return -1;
+
 	if (internal_config.single_file_segments) {
 		resize_hugefile(fd, path, list_idx, seg_idx, map_offset,
 				alloc_sz, false);

@@ -646,6 +651,7 @@ alloc_seg(struct rte_memseg *ms, void *addr, int socket_id,
 				lock(fd, LOCK_EX) == 1)
 			unlink(path);
 		close(fd);
+		fd_list[list_idx].fds[seg_idx] = -1;
 	}
 	return -1;
 }

@@ -700,6 +706,7 @@ free_seg(struct rte_memseg *ms, struct hugepage_info *hi,
 		}
 		/* closing fd will drop the lock */
 		close(fd);
+		fd_list[list_idx].fds[seg_idx] = -1;
 	}
 
 	memset(ms, 0, sizeof(*ms));

@@ -1364,8 +1371,7 @@ eal_memalloc_init(void)
 			return -1;
 
 	/* initialize all of the fd lists */
-	if (internal_config.single_file_segments)
-		if (rte_memseg_list_walk(fd_list_create_walk, NULL))
-			return -1;
+	if (rte_memseg_list_walk(fd_list_create_walk, NULL))
+		return -1;
 	return 0;
 }

index f7f07abd6..a5fb09f71 100644
@@ -47,6 +47,18 @@ eal_memalloc_sync_with_primary(void)
 	return -1;
 }
 
+int
+eal_memalloc_get_seg_fd(int list_idx, int seg_idx)
+{
+	return -1;
+}
+
+int
+eal_memalloc_set_seg_fd(int list_idx, int seg_idx, int fd)
+{
+	return -1;
+}
+
 int
 eal_memalloc_init(void)
 {

index fbfb1b055..034c2026a 100644
@@ -294,7 +294,7 @@ dump_memseg(const struct rte_memseg_list *msl, const struct rte_memseg *ms,
 		void *arg)
 {
 	struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
-	int msl_idx, ms_idx;
+	int msl_idx, ms_idx, fd;
 	FILE *f = arg;
 
 	msl_idx = msl - mcfg->memsegs;

@@ -305,10 +305,11 @@ dump_memseg(const struct rte_memseg_list *msl, const struct rte_memseg *ms,
 	if (ms_idx < 0)
 		return -1;
 
+	fd = eal_memalloc_get_seg_fd(msl_idx, ms_idx);
 	fprintf(f, "Segment %i-%i: IOVA:0x%"PRIx64", len:%zu, "
 			"virt:%p, socket_id:%"PRId32", "
 			"hugepage_sz:%"PRIu64", nchannel:%"PRIx32", "
-			"nrank:%"PRIx32"\n",
+			"nrank:%"PRIx32" fd:%i\n",
 			msl_idx, ms_idx,
 			ms->iova,
 			ms->len,

@@ -316,7 +317,8 @@ dump_memseg(const struct rte_memseg_list *msl, const struct rte_memseg *ms,
 			ms->socket_id,
 			ms->hugepage_sz,
 			ms->nchannel,
-			ms->nrank);
+			ms->nrank,
+			fd);
 
 	return 0;
 }

index 36bb1a027..a46c69c72 100644
@@ -76,6 +76,12 @@ eal_memalloc_mem_alloc_validator_unregister(const char *name, int socket_id);
 int
 eal_memalloc_mem_alloc_validate(int socket_id, size_t new_len);
 
+int
+eal_memalloc_get_seg_fd(int list_idx, int seg_idx);
+
+int
+eal_memalloc_set_seg_fd(int list_idx, int seg_idx, int fd);
+
 int
 eal_memalloc_init(void);
 

78 lines not on this page · raw message

82 lines not on this page · raw message

index a5fb09f71..80e4c3d4f 100644
@@ -4,6 +4,7 @@
 
 #include <inttypes.h>
 
+#include <rte_errno.h>
 #include <rte_log.h>
 #include <rte_memory.h>
 

@@ -50,13 +51,13 @@ eal_memalloc_sync_with_primary(void)
 int
 eal_memalloc_get_seg_fd(int list_idx, int seg_idx)
 {
-	return -1;
+	return -ENOTSUP;
 }
 
 int
 eal_memalloc_set_seg_fd(int list_idx, int seg_idx, int fd)
 {
-	return -1;
+	return -ENOTSUP;
 }
 
 int

55 lines not on this page · raw message

index a46c69c72..70a214de4 100644
@@ -76,9 +76,11 @@ eal_memalloc_mem_alloc_validator_unregister(const char *name, int socket_id);
 int
 eal_memalloc_mem_alloc_validate(int socket_id, size_t new_len);
 
+/* returns fd or -errno */
 int
 eal_memalloc_get_seg_fd(int list_idx, int seg_idx);
 
+/* returns 0 or -errno */
 int
 eal_memalloc_set_seg_fd(int list_idx, int seg_idx, int fd);
 

54 lines not on this page · raw message

index b820989e9..21f842753 100644
@@ -34,6 +34,7 @@
 #include <rte_log.h>
 #include <rte_eal_memconfig.h>
 #include <rte_eal.h>
+#include <rte_errno.h>
 #include <rte_memory.h>
 #include <rte_spinlock.h>
 

@@ -1381,7 +1382,7 @@ eal_memalloc_set_seg_fd(int list_idx, int seg_idx, int fd)
 		int len = mcfg->memsegs[list_idx].memseg_arr.len;
 
 		if (alloc_list(list_idx, len) < 0)
-			return -1;
+			return -ENOMEM;
 	}
 	fd_list[list_idx].fds[seg_idx] = fd;
 

@@ -1391,12 +1392,18 @@ eal_memalloc_set_seg_fd(int list_idx, int seg_idx, int fd)
 int
 eal_memalloc_get_seg_fd(int list_idx, int seg_idx)
 {
-	if (internal_config.single_file_segments)
-		return fd_list[list_idx].memseg_list_fd;
-	/* list not initialized */
-	if (fd_list[list_idx].len == 0)
-		return -1;
-	return fd_list[list_idx].fds[seg_idx];
+	int fd;
+	if (internal_config.single_file_segments) {
+		fd = fd_list[list_idx].memseg_list_fd;
+	} else if (fd_list[list_idx].len == 0) {
+		/* list not initialized */
+		fd = -1;
+	} else {
+		fd = fd_list[list_idx].fds[seg_idx];
+	}
+	if (fd < 0)
+		return -ENODEV;
+	return fd;
 }
 
 int

index 80e4c3d4f..06afbcc99 100644
@@ -60,6 +60,12 @@ eal_memalloc_set_seg_fd(int list_idx, int seg_idx, int fd)
 	return -ENOTSUP;
 }
 
+int
+eal_memalloc_get_seg_fd_offset(int list_idx, int seg_idx, size_t *offset)
+{
+	return -ENOTSUP;
+}
+
 int
 eal_memalloc_init(void)
 {

56 lines not on this page · raw message

index 70a214de4..af917c2f9 100644
@@ -84,6 +84,9 @@ eal_memalloc_get_seg_fd(int list_idx, int seg_idx);
 int
 eal_memalloc_set_seg_fd(int list_idx, int seg_idx, int fd);
 
+int
+eal_memalloc_get_seg_fd_offset(int list_idx, int seg_idx, size_t *offset);
+
 int
 eal_memalloc_init(void);
 

55 lines not on this page · raw message

index 21f842753..66e1d87b6 100644
@@ -1406,6 +1406,27 @@ eal_memalloc_get_seg_fd(int list_idx, int seg_idx)
 	return fd;
 }
 
+int
+eal_memalloc_get_seg_fd_offset(int list_idx, int seg_idx, size_t *offset)
+{
+	struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
+
+	/* fd_list not initialized? */
+	if (fd_list[list_idx].len == 0)
+		return -ENODEV;
+	/* segment not active? */
+	if (fd_list[list_idx].fds[seg_idx] < 0)
+		return -ENOENT;
+	/* for single-file segments mode, offset is seg idx times page size */
+	if (internal_config.single_file_segments) {
+		size_t pgsz = mcfg->memsegs[list_idx].page_sz;
+		*offset = pgsz * seg_idx;
+	} else {
+		*offset = 0;
+	}
+	return 0;
+}
+
 int
 eal_memalloc_init(void)
 {

index 873099acc..ddd624110 100644
@@ -1384,10 +1384,10 @@ eal_check_common_options(struct internal_config *internal_cfg)
 			" is only supported in non-legacy memory mode\n");
 	}
 	if (internal_cfg->single_file_segments &&
-			internal_cfg->hugepage_unlink) {
+			internal_cfg->hugepage_unlink &&
+			!internal_cfg->in_memory) {
 		RTE_LOG(ERR, EAL, "Option --"OPT_SINGLE_FILE_SEGMENTS" is "
-			"not compatible with neither --"OPT_IN_MEMORY" nor "
-			"--"OPT_HUGE_UNLINK"\n");
+			"not compatible with --"OPT_HUGE_UNLINK"\n");
 		return -1;
 	}
 	if (internal_cfg->legacy_mem &&

349 lines not on this page · raw message

index 43caf3ced..ba6c4ae39 100644
@@ -878,6 +878,10 @@ rte_fbarray_destroy(struct rte_fbarray *arr)
 	if (ret)
 		return ret;
 
+	/* with no shconf, there were never any files to begin with */
+	if (internal_config.no_shconf)
+		return 0;
+
 	/* try deleting the file */
 	eal_get_fbarray_path(path, sizeof(path), arr->name);
 

index dd5f97402..873099acc 100644
@@ -1390,6 +1390,12 @@ eal_check_common_options(struct internal_config *internal_cfg)
 			"--"OPT_HUGE_UNLINK"\n");
 		return -1;
 	}
+	if (internal_cfg->legacy_mem &&
+			internal_cfg->in_memory) {
+		RTE_LOG(ERR, EAL, "Option --"OPT_LEGACY_MEM" is not compatible "
+				"with --"OPT_IN_MEMORY"\n");
+		return -1;
+	}
 
 	return 0;
 }

index dbf19499e..dfb537f59 100644
@@ -17,6 +17,7 @@
 #include <sys/stat.h>
 #include <sys/queue.h>
 #include <sys/file.h>
+#include <sys/resource.h>
 #include <unistd.h>
 #include <limits.h>
 #include <sys/ioctl.h>

@@ -2204,6 +2205,25 @@ memseg_secondary_init(void)
 int
 rte_eal_memseg_init(void)
 {
+	/* increase rlimit to maximum */
+	struct rlimit lim;
+
+	if (getrlimit(RLIMIT_NOFILE, &lim) == 0) {
+		/* set limit to maximum */
+		lim.rlim_cur = lim.rlim_max;
+
+		if (setrlimit(RLIMIT_NOFILE, &lim) < 0) {
+			RTE_LOG(DEBUG, EAL, "Setting maximum number of open files failed: %s\n",
+					strerror(errno));
+		} else {
+			RTE_LOG(DEBUG, EAL, "Setting maximum number of open files to %"
+					PRIu64 "\n",
+					(uint64_t)lim.rlim_cur);
+		}
+	} else {
+		RTE_LOG(ERR, EAL, "Cannot get current resource limits\n");
+	}
+
 	return rte_eal_process_type() == RTE_PROC_PRIMARY ?
 #ifndef RTE_ARCH_64
 			memseg_primary_init_32() :

index aa95551a8..14bc5dce9 100644
@@ -57,25 +57,33 @@ const int anonymous_hugepages_supported =
  */
 static int fallocate_supported = -1; /* unknown */
 
-/* for single-file segments, we need some kind of mechanism to keep track of
+/*
+ * we have two modes - single file segments, and file-per-page mode.
+ *
+ * for single-file segments, we need some kind of mechanism to keep track of
  * which hugepages can be freed back to the system, and which cannot. we cannot
  * use flock() because they don't allow locking parts of a file, and we cannot
  * use fcntl() due to issues with their semantics, so we will have to rely on a
- * bunch of lockfiles for each page.
+ * bunch of lockfiles for each page. so, we will use 'fds' array to keep track
+ * of per-page lockfiles. we will store the actual segment list fd in the
+ * 'memseg_list_fd' field.
+ *
+ * for file-per-page mode, each page will have its own fd, so 'memseg_list_fd'
+ * will be invalid (set to -1), and we'll use 'fds' to keep track of page fd's.
  *
  * we cannot know how many pages a system will have in advance, but we do know
  * that they come in lists, and we know lengths of these lists. so, simply store
  * a malloc'd array of fd's indexed by list and segment index.
  *
  * they will be initialized at startup, and filled as we allocate/deallocate
- * segments. also, use this to track memseg list proper fd.
+ * segments.
  */
 static struct {
 	int *fds; /**< dynamically allocated array of segment lock fd's */
 	int memseg_list_fd; /**< memseg list fd */
 	int len; /**< total length of the array */
 	int count; /**< entries used in an array */
-} lock_fds[RTE_MAX_MEMSEG_LISTS];
+} fd_list[RTE_MAX_MEMSEG_LISTS];
 
 /** local copy of a memory map, used to synchronize memory hotplug in MP */
 static struct rte_memseg_list local_memsegs[RTE_MAX_MEMSEG_LISTS];

@@ -209,12 +217,12 @@ static int get_segment_lock_fd(int list_idx, int seg_idx)
 	char path[PATH_MAX] = {0};
 	int fd;
 
-	if (list_idx < 0 || list_idx >= (int)RTE_DIM(lock_fds))
+	if (list_idx < 0 || list_idx >= (int)RTE_DIM(fd_list))
 		return -1;
-	if (seg_idx < 0 || seg_idx >= lock_fds[list_idx].len)
+	if (seg_idx < 0 || seg_idx >= fd_list[list_idx].len)
 		return -1;
 
-	fd = lock_fds[list_idx].fds[seg_idx];
+	fd = fd_list[list_idx].fds[seg_idx];
 	/* does this lock already exist? */
 	if (fd >= 0)
 		return fd;

@@ -236,8 +244,8 @@ static int get_segment_lock_fd(int list_idx, int seg_idx)
 		return -1;
 	}
 	/* store it for future reference */
-	lock_fds[list_idx].fds[seg_idx] = fd;
-	lock_fds[list_idx].count++;
+	fd_list[list_idx].fds[seg_idx] = fd;
+	fd_list[list_idx].count++;
 	return fd;
 }
 

@@ -245,12 +253,12 @@ static int unlock_segment(int list_idx, int seg_idx)
 {
 	int fd, ret;
 
-	if (list_idx < 0 || list_idx >= (int)RTE_DIM(lock_fds))
+	if (list_idx < 0 || list_idx >= (int)RTE_DIM(fd_list))
 		return -1;
-	if (seg_idx < 0 || seg_idx >= lock_fds[list_idx].len)
+	if (seg_idx < 0 || seg_idx >= fd_list[list_idx].len)
 		return -1;
 
-	fd = lock_fds[list_idx].fds[seg_idx];
+	fd = fd_list[list_idx].fds[seg_idx];
 
 	/* upgrade lock to exclusive to see if we can remove the lockfile */
 	ret = lock(fd, LOCK_EX);

@@ -270,8 +278,8 @@ static int unlock_segment(int list_idx, int seg_idx)
 	 * and remove it from list anyway.
 	 */
 	close(fd);
-	lock_fds[list_idx].fds[seg_idx] = -1;
-	lock_fds[list_idx].count--;
+	fd_list[list_idx].fds[seg_idx] = -1;
+	fd_list[list_idx].count--;
 
 	if (ret < 0)
 		return -1;

@@ -288,7 +296,7 @@ get_seg_fd(char *path, int buflen, struct hugepage_info *hi,
 		/* create a hugepage file path */
 		eal_get_hugefile_path(path, buflen, hi->hugedir, list_idx);
 
-		fd = lock_fds[list_idx].memseg_list_fd;
+		fd = fd_list[list_idx].memseg_list_fd;
 
 		if (fd < 0) {
 			fd = open(path, O_CREAT | O_RDWR, 0600);

@@ -304,7 +312,7 @@ get_seg_fd(char *path, int buflen, struct hugepage_info *hi,
 				close(fd);
 				return -1;
 			}
-			lock_fds[list_idx].memseg_list_fd = fd;
+			fd_list[list_idx].memseg_list_fd = fd;
 		}
 	} else {
 		/* create a hugepage file path */

@@ -410,9 +418,9 @@ resize_hugefile(int fd, char *path, int list_idx, int seg_idx,
 				 * page file fd, so that one of the processes
 				 * could then delete the file after shrinking.
 				 */
-				if (ret < 1 && lock_fds[list_idx].count == 0) {
+				if (ret < 1 && fd_list[list_idx].count == 0) {
 					close(fd);
-					lock_fds[list_idx].memseg_list_fd = -1;
+					fd_list[list_idx].memseg_list_fd = -1;
 				}
 
 				if (ret < 0) {

@@ -448,13 +456,13 @@ resize_hugefile(int fd, char *path, int list_idx, int seg_idx,
 				 * more segments active in this segment list,
 				 * and remove the file if there aren't.
 				 */
-				if (lock_fds[list_idx].count == 0) {
+				if (fd_list[list_idx].count == 0) {
 					if (unlink(path))
 						RTE_LOG(ERR, EAL, "%s(): unlinking '%s' failed: %s\n",
 							__func__, path,
 							strerror(errno));
 					close(fd);
-					lock_fds[list_idx].memseg_list_fd = -1;
+					fd_list[list_idx].memseg_list_fd = -1;
 				}
 			}
 		}

@@ -1319,7 +1327,7 @@ secondary_msl_create_walk(const struct rte_memseg_list *msl,
 }
 
 static int
-secondary_lock_list_create_walk(const struct rte_memseg_list *msl,
+fd_list_create_walk(const struct rte_memseg_list *msl,
 		void *arg __rte_unused)
 {
 	struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;

@@ -1330,20 +1338,20 @@ secondary_lock_list_create_walk(const struct rte_memseg_list *msl,
 	msl_idx = msl - mcfg->memsegs;
 	len = msl->memseg_arr.len;
 
-	/* ensure we have space to store lock fd per each possible segment */
+	/* ensure we have space to store fd per each possible segment */
 	data = malloc(sizeof(int) * len);
 	if (data == NULL) {
-		RTE_LOG(ERR, EAL, "Unable to allocate space for lock descriptors\n");
+		RTE_LOG(ERR, EAL, "Unable to allocate space for file descriptors\n");
 		return -1;
 	}
 	/* set all fd's as invalid */
 	for (i = 0; i < len; i++)
 		data[i] = -1;
 
-	lock_fds[msl_idx].fds = data;
-	lock_fds[msl_idx].len = len;
-	lock_fds[msl_idx].count = 0;
-	lock_fds[msl_idx].memseg_list_fd = -1;
+	fd_list[msl_idx].fds = data;
+	fd_list[msl_idx].len = len;
+	fd_list[msl_idx].count = 0;
+	fd_list[msl_idx].memseg_list_fd = -1;
 
 	return 0;
 }

@@ -1355,9 +1363,9 @@ eal_memalloc_init(void)
 		if (rte_memseg_list_walk(secondary_msl_create_walk, NULL) < 0)
 			return -1;
 
-	/* initialize all of the lock fd lists */
+	/* initialize all of the fd lists */
 	if (internal_config.single_file_segments)
-		if (rte_memseg_list_walk(secondary_lock_list_create_walk, NULL))
+		if (rte_memseg_list_walk(fd_list_create_walk, NULL))
 			return -1;
 	return 0;
 }

index 14bc5dce9..7d536350e 100644
@@ -318,18 +318,24 @@ get_seg_fd(char *path, int buflen, struct hugepage_info *hi,
 		/* create a hugepage file path */
 		eal_get_hugefile_path(path, buflen, hi->hugedir,
 				list_idx * RTE_MAX_MEMSEG_PER_LIST + seg_idx);
-		fd = open(path, O_CREAT | O_RDWR, 0600);
+
+		fd = fd_list[list_idx].fds[seg_idx];
+
 		if (fd < 0) {
-			RTE_LOG(DEBUG, EAL, "%s(): open failed: %s\n", __func__,
-					strerror(errno));
-			return -1;
-		}
-		/* take out a read lock */
-		if (lock(fd, LOCK_SH) < 0) {
-			RTE_LOG(ERR, EAL, "%s(): lock failed: %s\n",
-				__func__, strerror(errno));
-			close(fd);
-			return -1;
+			fd = open(path, O_CREAT | O_RDWR, 0600);
+			if (fd < 0) {
+				RTE_LOG(DEBUG, EAL, "%s(): open failed: %s\n",
+					__func__, strerror(errno));
+				return -1;
+			}
+			/* take out a read lock */
+			if (lock(fd, LOCK_SH) < 0) {
+				RTE_LOG(ERR, EAL, "%s(): lock failed: %s\n",
+					__func__, strerror(errno));
+				close(fd);
+				return -1;
+			}
+			fd_list[list_idx].fds[seg_idx] = fd;
 		}
 	}
 	return fd;

@@ -601,10 +607,6 @@ alloc_seg(struct rte_memseg *ms, void *addr, int socket_id,
 		goto mapped;
 	}
 #endif
-	/* for non-single file segments that aren't in-memory, we can close fd
-	 * here */
-	if (!internal_config.single_file_segments && !internal_config.in_memory)
-		close(fd);
 
 	ms->addr = addr;
 	ms->hugepage_sz = alloc_sz;

@@ -634,7 +636,10 @@ alloc_seg(struct rte_memseg *ms, void *addr, int socket_id,
 		RTE_LOG(CRIT, EAL, "Can't mmap holes in our virtual address space\n");
 	}
 resized:
-	/* in-memory mode will never be single-file-segments mode */
+	/* some codepaths will return negative fd, so exit early */
+	if (fd < 0)
+		return -1;
+
 	if (internal_config.single_file_segments) {
 		resize_hugefile(fd, path, list_idx, seg_idx, map_offset,
 				alloc_sz, false);

@@ -646,6 +651,7 @@ alloc_seg(struct rte_memseg *ms, void *addr, int socket_id,
 				lock(fd, LOCK_EX) == 1)
 			unlink(path);
 		close(fd);
+		fd_list[list_idx].fds[seg_idx] = -1;
 	}
 	return -1;
 }

@@ -700,6 +706,7 @@ free_seg(struct rte_memseg *ms, struct hugepage_info *hi,
 		}
 		/* closing fd will drop the lock */
 		close(fd);
+		fd_list[list_idx].fds[seg_idx] = -1;
 	}
 
 	memset(ms, 0, sizeof(*ms));

@@ -1364,8 +1371,7 @@ eal_memalloc_init(void)
 			return -1;
 
 	/* initialize all of the fd lists */
-	if (internal_config.single_file_segments)
-		if (rte_memseg_list_walk(fd_list_create_walk, NULL))
-			return -1;
+	if (rte_memseg_list_walk(fd_list_create_walk, NULL))
+		return -1;
 	return 0;
 }

index f7f07abd6..a5fb09f71 100644
@@ -47,6 +47,18 @@ eal_memalloc_sync_with_primary(void)
 	return -1;
 }
 
+int
+eal_memalloc_get_seg_fd(int list_idx, int seg_idx)
+{
+	return -1;
+}
+
+int
+eal_memalloc_set_seg_fd(int list_idx, int seg_idx, int fd)
+{
+	return -1;
+}
+
 int
 eal_memalloc_init(void)
 {

index fbfb1b055..034c2026a 100644
@@ -294,7 +294,7 @@ dump_memseg(const struct rte_memseg_list *msl, const struct rte_memseg *ms,
 		void *arg)
 {
 	struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
-	int msl_idx, ms_idx;
+	int msl_idx, ms_idx, fd;
 	FILE *f = arg;
 
 	msl_idx = msl - mcfg->memsegs;

@@ -305,10 +305,11 @@ dump_memseg(const struct rte_memseg_list *msl, const struct rte_memseg *ms,
 	if (ms_idx < 0)
 		return -1;
 
+	fd = eal_memalloc_get_seg_fd(msl_idx, ms_idx);
 	fprintf(f, "Segment %i-%i: IOVA:0x%"PRIx64", len:%zu, "
 			"virt:%p, socket_id:%"PRId32", "
 			"hugepage_sz:%"PRIu64", nchannel:%"PRIx32", "
-			"nrank:%"PRIx32"\n",
+			"nrank:%"PRIx32" fd:%i\n",
 			msl_idx, ms_idx,
 			ms->iova,
 			ms->len,

@@ -316,7 +317,8 @@ dump_memseg(const struct rte_memseg_list *msl, const struct rte_memseg *ms,
 			ms->socket_id,
 			ms->hugepage_sz,
 			ms->nchannel,
-			ms->nrank);
+			ms->nrank,
+			fd);
 
 	return 0;
 }

index 36bb1a027..a46c69c72 100644
@@ -76,6 +76,12 @@ eal_memalloc_mem_alloc_validator_unregister(const char *name, int socket_id);
 int
 eal_memalloc_mem_alloc_validate(int socket_id, size_t new_len);
 
+int
+eal_memalloc_get_seg_fd(int list_idx, int seg_idx);
+
+int
+eal_memalloc_set_seg_fd(int list_idx, int seg_idx, int fd);
+
 int
 eal_memalloc_init(void);
 

78 lines not on this page · raw message

82 lines not on this page · raw message

index a5fb09f71..80e4c3d4f 100644
@@ -4,6 +4,7 @@
 
 #include <inttypes.h>
 
+#include <rte_errno.h>
 #include <rte_log.h>
 #include <rte_memory.h>
 

@@ -50,13 +51,13 @@ eal_memalloc_sync_with_primary(void)
 int
 eal_memalloc_get_seg_fd(int list_idx, int seg_idx)
 {
-	return -1;
+	return -ENOTSUP;
 }
 
 int
 eal_memalloc_set_seg_fd(int list_idx, int seg_idx, int fd)
 {
-	return -1;
+	return -ENOTSUP;
 }
 
 int

55 lines not on this page · raw message

index a46c69c72..70a214de4 100644
@@ -76,9 +76,11 @@ eal_memalloc_mem_alloc_validator_unregister(const char *name, int socket_id);
 int
 eal_memalloc_mem_alloc_validate(int socket_id, size_t new_len);
 
+/* returns fd or -errno */
 int
 eal_memalloc_get_seg_fd(int list_idx, int seg_idx);
 
+/* returns 0 or -errno */
 int
 eal_memalloc_set_seg_fd(int list_idx, int seg_idx, int fd);
 

54 lines not on this page · raw message

index b820989e9..21f842753 100644
@@ -34,6 +34,7 @@
 #include <rte_log.h>
 #include <rte_eal_memconfig.h>
 #include <rte_eal.h>
+#include <rte_errno.h>
 #include <rte_memory.h>
 #include <rte_spinlock.h>
 

@@ -1381,7 +1382,7 @@ eal_memalloc_set_seg_fd(int list_idx, int seg_idx, int fd)
 		int len = mcfg->memsegs[list_idx].memseg_arr.len;
 
 		if (alloc_list(list_idx, len) < 0)
-			return -1;
+			return -ENOMEM;
 	}
 	fd_list[list_idx].fds[seg_idx] = fd;
 

@@ -1391,12 +1392,18 @@ eal_memalloc_set_seg_fd(int list_idx, int seg_idx, int fd)
 int
 eal_memalloc_get_seg_fd(int list_idx, int seg_idx)
 {
-	if (internal_config.single_file_segments)
-		return fd_list[list_idx].memseg_list_fd;
-	/* list not initialized */
-	if (fd_list[list_idx].len == 0)
-		return -1;
-	return fd_list[list_idx].fds[seg_idx];
+	int fd;
+	if (internal_config.single_file_segments) {
+		fd = fd_list[list_idx].memseg_list_fd;
+	} else if (fd_list[list_idx].len == 0) {
+		/* list not initialized */
+		fd = -1;
+	} else {
+		fd = fd_list[list_idx].fds[seg_idx];
+	}
+	if (fd < 0)
+		return -ENODEV;
+	return fd;
 }
 
 int

index 80e4c3d4f..06afbcc99 100644
@@ -60,6 +60,12 @@ eal_memalloc_set_seg_fd(int list_idx, int seg_idx, int fd)
 	return -ENOTSUP;
 }
 
+int
+eal_memalloc_get_seg_fd_offset(int list_idx, int seg_idx, size_t *offset)
+{
+	return -ENOTSUP;
+}
+
 int
 eal_memalloc_init(void)
 {

56 lines not on this page · raw message

index 70a214de4..af917c2f9 100644
@@ -84,6 +84,9 @@ eal_memalloc_get_seg_fd(int list_idx, int seg_idx);
 int
 eal_memalloc_set_seg_fd(int list_idx, int seg_idx, int fd);
 
+int
+eal_memalloc_get_seg_fd_offset(int list_idx, int seg_idx, size_t *offset);
+
 int
 eal_memalloc_init(void);
 

55 lines not on this page · raw message

index 21f842753..6fc9dc0ca 100644
@@ -1406,6 +1406,30 @@ eal_memalloc_get_seg_fd(int list_idx, int seg_idx)
 	return fd;
 }
 
+int
+eal_memalloc_get_seg_fd_offset(int list_idx, int seg_idx, size_t *offset)
+{
+	struct rte_mem_config *mcfg = rte_eal_get_configuration()->mem_config;
+
+	/* fd_list not initialized? */
+	if (fd_list[list_idx].len == 0)
+		return -ENODEV;
+	if (internal_config.single_file_segments) {
+		size_t pgsz = mcfg->memsegs[list_idx].page_sz;
+
+		/* segment not active? */
+		if (fd_list[list_idx].memseg_list_fd < 0)
+			return -ENOENT;
+		*offset = pgsz * seg_idx;
+	} else {
+		/* segment not active? */
+		if (fd_list[list_idx].fds[seg_idx] < 0)
+			return -ENOENT;
+		*offset = 0;
+	}
+	return 0;
+}
+
 int
 eal_memalloc_init(void)
 {

index 873099acc..ddd624110 100644
@@ -1384,10 +1384,10 @@ eal_check_common_options(struct internal_config *internal_cfg)
 			" is only supported in non-legacy memory mode\n");
 	}
 	if (internal_cfg->single_file_segments &&
-			internal_cfg->hugepage_unlink) {
+			internal_cfg->hugepage_unlink &&
+			!internal_cfg->in_memory) {
 		RTE_LOG(ERR, EAL, "Option --"OPT_SINGLE_FILE_SEGMENTS" is "
-			"not compatible with neither --"OPT_IN_MEMORY" nor "
-			"--"OPT_HUGE_UNLINK"\n");
+			"not compatible with --"OPT_HUGE_UNLINK"\n");
 		return -1;
 	}
 	if (internal_cfg->legacy_mem &&

349 lines not on this page · raw message