Skip to contentSkip to conversation

fstests: test reads/writes from hugepages-backed buffers

PATCH Superseded Joanne Koong 4 participants 8 messages Dec 27, 2024 – Jan 15, 2025 VFS

v2 of 5 Superseded Latest is v5 →

lore
Share & tools
lore
#0 lore raw

v2 cover fstests: test reads/writes from hugepages-backed buffers

There was a recent bug in rc1 [1] that was due to faulty handling for userspace buffers backed by hugepages.

This patchset adds generic tests for reads/writes from buffers backed by hugepages.

[1] https://lore.kernel.org/linux-fsdevel/p3iss6hssbvtdutnwmuddvdadubrhfkdoosgmbewvo674f7f3y@cwnwffjqltzw/

Changelog:
v1: https://lore.kernel.org/linux-fsdevel/[email protected]/
- Refactor out buffer initialization (Brian)
- Update commit messages of 1st patch (Brian)
- Use << 10 instead of * 1024 (Nirjhar)
- Replace CONFIG_TRANSPARENT_HUGEPAGE check with checking 'sys/kernel/mm/transparent_hugepage/' (Darrick)
- Integrate readbdy and writebdy options
- Update options of generic/759 to include psize/bsize

Joanne Koong (2):
fsx: support reads/writes from buffers backed by hugepages
generic: add tests for read/writes from hugepages-backed buffers

 common/rc             |  13 +++++
 ltp/fsx.c             | 108 +++++++++++++++++++++++++++++++++++++-----
 tests/generic/758     |  22 +++++++++
 tests/generic/758.out |   4 ++
 tests/generic/759     |  26 ++++++++++
 tests/generic/759.out |   4 ++
 6 files changed, 166 insertions(+), 11 deletions(-)
 create mode 100755 tests/generic/758
 create mode 100644 tests/generic/758.out
 create mode 100755 tests/generic/759
 create mode 100644 tests/generic/759.out
Signature · 1 line
2.47.1
#1 lore raw

PATCH v2 1/2 fsx: support reads/writes from buffers backed by hugepages

Add support for reads/writes from buffers backed by hugepages. This can be enabled through the '-h' flag. This flag should only be used on systems where THP capabilities are enabled.

This is motivated by a recent bug that was due to faulty handling of userspace buffers backed by hugepages. This patch is a mitigation against problems like this in the future.

---
 ltp/fsx.c | 108 ++++++++++++++++++++++++++++++++++++++++++++++++------
 1 file changed, 97 insertions(+), 11 deletions(-)
ltp/fsx.c +97 -11
index 41933354..fb6a9b31 100644
@@ -190,6 +190,7 @@ int	o_direct;			/* -Z */
 int	aio = 0;
 int	uring = 0;
 int	mark_nr = 0;
+int	hugepages = 0;                  /* -h flag */
 
 int page_size;
 int page_mask;
@@ -2471,7 +2472,7 @@ void
 usage(void)
 {
 	fprintf(stdout, "usage: %s",
-		"fsx [-dfknqxyzBEFHIJKLORWXZ0]\n\
+		"fsx [-dfhknqxyzBEFHIJKLORWXZ0]\n\
 	   [-b opnum] [-c Prob] [-g filldata] [-i logdev] [-j logid]\n\
 	   [-l flen] [-m start:end] [-o oplen] [-p progressinterval]\n\
 	   [-r readbdy] [-s style] [-t truncbdy] [-w writebdy]\n\
@@ -2484,6 +2485,7 @@ usage(void)
 	-e: pollute post-eof on size changes (default 0)\n\
 	-f: flush and invalidate cache after I/O\n\
 	-g X: write character X instead of random generated data\n\
+	-h hugepages: use buffers backed by hugepages for reads/writes\n\
 	-i logdev: do integrity testing, logdev is the dm log writes device\n\
 	-j logid: prefix debug log messsages with this id\n\
 	-k: do not truncate existing file and use its size as upper bound on file size\n\
@@ -2856,6 +2858,95 @@ keep_running(void)
 	return numops-- != 0;
 }
 
+static long
+get_hugepage_size(void)
+{
+	const char str[] = "Hugepagesize:";
+	size_t str_len =  sizeof(str) - 1;
+	unsigned int hugepage_size = 0;
+	char buffer[64];
+	FILE *file;
+
+	file = fopen("/proc/meminfo", "r");
+	if (!file) {
+		prterr("get_hugepage_size: fopen /proc/meminfo");
+		return -1;
+	}
+	while (fgets(buffer, sizeof(buffer), file)) {
+		if (strncmp(buffer, str, str_len) == 0) {
+			sscanf(buffer + str_len, "%u", &hugepage_size);
+			break;
+		}
+	}
+	fclose(file);
+	if (!hugepage_size) {
+		prterr("get_hugepage_size: failed to find "
+			"hugepage size in /proc/meminfo\n");
+		return -1;
+	}
+
+	/* convert from KiB to bytes */
+	return hugepage_size << 10;
+}
+
+static void *
+init_hugepages_buf(unsigned len, int hugepage_size, int alignment)
+{
+	void *buf;
+	long buf_size = roundup(len, hugepage_size) + alignment;
+
+	if (posix_memalign(&buf, hugepage_size, buf_size)) {
+		prterr("posix_memalign for buf");
+		return NULL;
+	}
+	memset(buf, '\0', buf_size);
+	if (madvise(buf, buf_size, MADV_COLLAPSE)) {
+		prterr("madvise collapse for buf");
+		free(buf);
+		return NULL;
+	}
+
+	return buf;
+}
+
+static void
+init_buffers(void)
+{
+	int i;
+
+	original_buf = (char *) malloc(maxfilelen);
+	for (i = 0; i < maxfilelen; i++)
+		original_buf[i] = random() % 256;
+	if (hugepages) {
+		long hugepage_size = get_hugepage_size();
+		if (hugepage_size == -1) {
+			prterr("get_hugepage_size()");
+			exit(100);
+		}
+		good_buf = init_hugepages_buf(maxfilelen, hugepage_size, writebdy);
+		if (!good_buf) {
+			prterr("init_hugepages_buf failed for good_buf");
+			exit(101);
+		}
+
+		temp_buf = init_hugepages_buf(maxoplen, hugepage_size, readbdy);
+		if (!temp_buf) {
+			prterr("init_hugepages_buf failed for temp_buf");
+			exit(101);
+		}
+	} else {
+		unsigned long good_buf_len = maxfilelen + writebdy;
+		unsigned long temp_buf_len = maxoplen + readbdy;
+
+		good_buf = (char *) malloc(good_buf_len);
+		memset(good_buf, '\0', good_buf_len);
+		temp_buf = (char *) malloc(temp_buf_len);
+		memset(temp_buf, '\0', temp_buf_len);
+	}
+	good_buf = round_ptr_up(good_buf, writebdy, 0);
+	temp_buf = round_ptr_up(temp_buf, readbdy, 0);
+}
+
 static struct option longopts[] = {
 	{"replay-ops", required_argument, 0, 256},
 	{"record-ops", optional_argument, 0, 255},
@@ -2883,7 +2974,7 @@ main(int argc, char **argv)
 	setvbuf(stdout, (char *)0, _IOLBF, 0); /* line buffered stdout */
 
 	while ((ch = getopt_long(argc, argv,
-				 "0b:c:de:fg:i:j:kl:m:no:p:qr:s:t:uw:xyABD:EFJKHzCILN:OP:RS:UWXZ",
+				 "0b:c:de:fg:hi:j:kl:m:no:p:qr:s:t:uw:xyABD:EFJKHzCILN:OP:RS:UWXZ",
 				 longopts, NULL)) != EOF)
 		switch (ch) {
 		case 'b':
@@ -2916,6 +3007,9 @@ main(int argc, char **argv)
 		case 'g':
 			filldata = *optarg;
 			break;
+		case 'h':
+			hugepages = 1;
+			break;
 		case 'i':
 			integrity = 1;
 			logdev = strdup(optarg);
@@ -3229,15 +3323,7 @@ main(int argc, char **argv)
 			exit(95);
 		}
 	}
-	original_buf = (char *) malloc(maxfilelen);
-	for (i = 0; i < maxfilelen; i++)
-		original_buf[i] = random() % 256;
-	good_buf = (char *) malloc(maxfilelen + writebdy);
-	good_buf = round_ptr_up(good_buf, writebdy, 0);
-	memset(good_buf, '\0', maxfilelen);
-	temp_buf = (char *) malloc(maxoplen + readbdy);
-	temp_buf = round_ptr_up(temp_buf, readbdy, 0);
-	memset(temp_buf, '\0', maxoplen);
+	init_buffers();
 	if (lite) {	/* zero entire existing file */
 		ssize_t written;
 
Signature · 1 line
2.47.1
#2 lore raw

Re: [PATCH v2 1/2] fsx: support reads/writes from buffers backed by hugepages

Quoted 65 lines · Joanne Koong

Add support for reads/writes from buffers backed by hugepages. This can be enabled through the '-h' flag. This flag should only be used on systems where THP capabilities are enabled.

This is motivated by a recent bug that was due to faulty handling of userspace buffers backed by hugepages. This patch is a mitigation against problems like this in the future.

---
  ltp/fsx.c | 108 ++++++++++++++++++++++++++++++++++++++++++++++++------
  1 file changed, 97 insertions(+), 11 deletions(-)
ltp/fsx.c +22 -1
index 41933354..fb6a9b31 100644
@@ -190,6 +190,7 @@ int	o_direct;			/* -Z */
  int	aio = 0;
  int	uring = 0;
  int	mark_nr = 0;
+int	hugepages = 0;                  /* -h flag */
 
  int page_size;
  int page_mask;
@@ -2471,7 +2472,7 @@ void
  usage(void)
  {
  	fprintf(stdout, "usage: %s",
-		"fsx [-dfknqxyzBEFHIJKLORWXZ0]\n\
+		"fsx [-dfhknqxyzBEFHIJKLORWXZ0]\n\
  	   [-b opnum] [-c Prob] [-g filldata] [-i logdev] [-j logid]\n\
  	   [-l flen] [-m start:end] [-o oplen] [-p progressinterval]\n\
  	   [-r readbdy] [-s style] [-t truncbdy] [-w writebdy]\n\
@@ -2484,6 +2485,7 @@ usage(void)
  	-e: pollute post-eof on size changes (default 0)\n\
  	-f: flush and invalidate cache after I/O\n\
  	-g X: write character X instead of random generated data\n\
+	-h hugepages: use buffers backed by hugepages for reads/writes\n\
  	-i logdev: do integrity testing, logdev is the dm log writes device\n\
  	-j logid: prefix debug log messsages with this id\n\
  	-k: do not truncate existing file and use its size as upper bound on file size\n\
@@ -2856,6 +2858,95 @@ keep_running(void)
  	return numops-- != 0;
  }
 
+static long
+get_hugepage_size(void)
+{
+	const char str[] = "Hugepagesize:";
+	size_t str_len =  sizeof(str) - 1;
+	unsigned int hugepage_size = 0;
+	char buffer[64];
+	FILE *file;
+
+	file = fopen("/proc/meminfo", "r");
+	if (!file) {
+		prterr("get_hugepage_size: fopen /proc/meminfo");
+		return -1;
+	}
+	while (fgets(buffer, sizeof(buffer), file)) {
+		if (strncmp(buffer, str, str_len) == 0) {
+			sscanf(buffer + str_len, "%u", &hugepage_size);
+			break;
+		}

ltp/fsx.c keep_running(void) · line 2889

Quoted 10 lines
ltp/fsx.c +10 -0
+	}
+	fclose(file);
+	if (!hugepage_size) {
+		prterr("get_hugepage_size: failed to find "
+			"hugepage size in /proc/meminfo\n");
+		return -1;
+	}
+
+	/* convert from KiB to bytes */
+	return hugepage_size << 10;

Thanks for the change.

Quoted 43 lines
ltp/fsx.c +43 -0
+}
+
+static void *
+init_hugepages_buf(unsigned len, int hugepage_size, int alignment)
+{
+	void *buf;
+	long buf_size = roundup(len, hugepage_size) + alignment;
+
+	if (posix_memalign(&buf, hugepage_size, buf_size)) {
+		prterr("posix_memalign for buf");
+		return NULL;
+	}
+	memset(buf, '\0', buf_size);
+	if (madvise(buf, buf_size, MADV_COLLAPSE)) {
+		prterr("madvise collapse for buf");
+		free(buf);
+		return NULL;
+	}
+
+	return buf;
+}
+
+static void
+init_buffers(void)
+{
+	int i;
+
+	original_buf = (char *) malloc(maxfilelen);
+	for (i = 0; i < maxfilelen; i++)
+		original_buf[i] = random() % 256;
+	if (hugepages) {
+		long hugepage_size = get_hugepage_size();
+		if (hugepage_size == -1) {
+			prterr("get_hugepage_size()");
+			exit(100);
+		}
+		good_buf = init_hugepages_buf(maxfilelen, hugepage_size, writebdy);
+		if (!good_buf) {
+			prterr("init_hugepages_buf failed for good_buf");
+			exit(101);
+		}
+
+		temp_buf = init_hugepages_buf(maxoplen, hugepage_size, readbdy);

ltp/fsx.c keep_running(void) · line 2942

Quoted 10 lines
ltp/fsx.c +10 -0
+		if (!temp_buf) {
+			prterr("init_hugepages_buf failed for temp_buf");
+			exit(101);
+		}
+	} else {
+		unsigned long good_buf_len = maxfilelen + writebdy;
+		unsigned long temp_buf_len = maxoplen + readbdy;
+
+		good_buf = (char *) malloc(good_buf_len);
+		memset(good_buf, '\0', good_buf_len);

minor:  maybe good_buf = (char *)calloc(1, good_buf_len); So we don't need to use memset.

Quoted 46 lines
ltp/fsx.c +12 -10
+		temp_buf = (char *) malloc(temp_buf_len);
+		memset(temp_buf, '\0', temp_buf_len);
+	}
+	good_buf = round_ptr_up(good_buf, writebdy, 0);
+	temp_buf = round_ptr_up(temp_buf, readbdy, 0);
+}
+
  static struct option longopts[] = {
  	{"replay-ops", required_argument, 0, 256},
  	{"record-ops", optional_argument, 0, 255},
@@ -2883,7 +2974,7 @@ main(int argc, char **argv)
  	setvbuf(stdout, (char *)0, _IOLBF, 0); /* line buffered stdout */
 
  	while ((ch = getopt_long(argc, argv,
-				 "0b:c:de:fg:i:j:kl:m:no:p:qr:s:t:uw:xyABD:EFJKHzCILN:OP:RS:UWXZ",
+				 "0b:c:de:fg:hi:j:kl:m:no:p:qr:s:t:uw:xyABD:EFJKHzCILN:OP:RS:UWXZ",
  				 longopts, NULL)) != EOF)
  		switch (ch) {
  		case 'b':
@@ -2916,6 +3007,9 @@ main(int argc, char **argv)
  		case 'g':
  			filldata = *optarg;
  			break;
+		case 'h':
+			hugepages = 1;
+			break;
  		case 'i':
  			integrity = 1;
  			logdev = strdup(optarg);
@@ -3229,15 +3323,7 @@ main(int argc, char **argv)
  			exit(95);
  		}
  	}
-	original_buf = (char *) malloc(maxfilelen);
-	for (i = 0; i < maxfilelen; i++)
-		original_buf[i] = random() % 256;
-	good_buf = (char *) malloc(maxfilelen + writebdy);
-	good_buf = round_ptr_up(good_buf, writebdy, 0);
-	memset(good_buf, '\0', maxfilelen);
-	temp_buf = (char *) malloc(maxoplen + readbdy);
-	temp_buf = round_ptr_up(temp_buf, readbdy, 0);
-	memset(temp_buf, '\0', maxoplen);
+	init_buffers();
  	if (lite) {	/* zero entire existing file */
  		ssize_t written;
 
Signature · 3 lines
Nirjhar Roy
Linux Kernel Developer
IBM, Bangalore
Reviewed-by on v2 1/2
#3 lore raw

Re: [PATCH v2 1/2] fsx: support reads/writes from buffers backed by hugepages

In reply to Joanne Koong
Quoted 4 lines · Joanne Koong

Add support for reads/writes from buffers backed by hugepages. This can be enabled through the '-h' flag. This flag should only be used on systems where THP capabilities are enabled.

Quoted 8 lines

This is motivated by a recent bug that was due to faulty handling of userspace buffers backed by hugepages. This patch is a mitigation against problems like this in the future.

---
 ltp/fsx.c | 108 ++++++++++++++++++++++++++++++++++++++++++++++++------
 1 file changed, 97 insertions(+), 11 deletions(-)

Thanks for the buffer init code cleanup. This looks much nicer to me now. Modulo comments from others:

Quoted 165 lines
ltp/fsx.c +97 -11
index 41933354..fb6a9b31 100644
@@ -190,6 +190,7 @@ int	o_direct;			/* -Z */
 int	aio = 0;
 int	uring = 0;
 int	mark_nr = 0;
+int	hugepages = 0;                  /* -h flag */
 
 int page_size;
 int page_mask;
@@ -2471,7 +2472,7 @@ void
 usage(void)
 {
 	fprintf(stdout, "usage: %s",
-		"fsx [-dfknqxyzBEFHIJKLORWXZ0]\n\
+		"fsx [-dfhknqxyzBEFHIJKLORWXZ0]\n\
 	   [-b opnum] [-c Prob] [-g filldata] [-i logdev] [-j logid]\n\
 	   [-l flen] [-m start:end] [-o oplen] [-p progressinterval]\n\
 	   [-r readbdy] [-s style] [-t truncbdy] [-w writebdy]\n\
@@ -2484,6 +2485,7 @@ usage(void)
 	-e: pollute post-eof on size changes (default 0)\n\
 	-f: flush and invalidate cache after I/O\n\
 	-g X: write character X instead of random generated data\n\
+	-h hugepages: use buffers backed by hugepages for reads/writes\n\
 	-i logdev: do integrity testing, logdev is the dm log writes device\n\
 	-j logid: prefix debug log messsages with this id\n\
 	-k: do not truncate existing file and use its size as upper bound on file size\n\
@@ -2856,6 +2858,95 @@ keep_running(void)
 	return numops-- != 0;
 }
 
+static long
+get_hugepage_size(void)
+{
+	const char str[] = "Hugepagesize:";
+	size_t str_len =  sizeof(str) - 1;
+	unsigned int hugepage_size = 0;
+	char buffer[64];
+	FILE *file;
+
+	file = fopen("/proc/meminfo", "r");
+	if (!file) {
+		prterr("get_hugepage_size: fopen /proc/meminfo");
+		return -1;
+	}
+	while (fgets(buffer, sizeof(buffer), file)) {
+		if (strncmp(buffer, str, str_len) == 0) {
+			sscanf(buffer + str_len, "%u", &hugepage_size);
+			break;
+		}
+	}
+	fclose(file);
+	if (!hugepage_size) {
+		prterr("get_hugepage_size: failed to find "
+			"hugepage size in /proc/meminfo\n");
+		return -1;
+	}
+
+	/* convert from KiB to bytes */
+	return hugepage_size << 10;
+}
+
+static void *
+init_hugepages_buf(unsigned len, int hugepage_size, int alignment)
+{
+	void *buf;
+	long buf_size = roundup(len, hugepage_size) + alignment;
+
+	if (posix_memalign(&buf, hugepage_size, buf_size)) {
+		prterr("posix_memalign for buf");
+		return NULL;
+	}
+	memset(buf, '\0', buf_size);
+	if (madvise(buf, buf_size, MADV_COLLAPSE)) {
+		prterr("madvise collapse for buf");
+		free(buf);
+		return NULL;
+	}
+
+	return buf;
+}
+
+static void
+init_buffers(void)
+{
+	int i;
+
+	original_buf = (char *) malloc(maxfilelen);
+	for (i = 0; i < maxfilelen; i++)
+		original_buf[i] = random() % 256;
+	if (hugepages) {
+		long hugepage_size = get_hugepage_size();
+		if (hugepage_size == -1) {
+			prterr("get_hugepage_size()");
+			exit(100);
+		}
+		good_buf = init_hugepages_buf(maxfilelen, hugepage_size, writebdy);
+		if (!good_buf) {
+			prterr("init_hugepages_buf failed for good_buf");
+			exit(101);
+		}
+
+		temp_buf = init_hugepages_buf(maxoplen, hugepage_size, readbdy);
+		if (!temp_buf) {
+			prterr("init_hugepages_buf failed for temp_buf");
+			exit(101);
+		}
+	} else {
+		unsigned long good_buf_len = maxfilelen + writebdy;
+		unsigned long temp_buf_len = maxoplen + readbdy;
+
+		good_buf = (char *) malloc(good_buf_len);
+		memset(good_buf, '\0', good_buf_len);
+		temp_buf = (char *) malloc(temp_buf_len);
+		memset(temp_buf, '\0', temp_buf_len);
+	}
+	good_buf = round_ptr_up(good_buf, writebdy, 0);
+	temp_buf = round_ptr_up(temp_buf, readbdy, 0);
+}
+
 static struct option longopts[] = {
 	{"replay-ops", required_argument, 0, 256},
 	{"record-ops", optional_argument, 0, 255},
@@ -2883,7 +2974,7 @@ main(int argc, char **argv)
 	setvbuf(stdout, (char *)0, _IOLBF, 0); /* line buffered stdout */
 
 	while ((ch = getopt_long(argc, argv,
-				 "0b:c:de:fg:i:j:kl:m:no:p:qr:s:t:uw:xyABD:EFJKHzCILN:OP:RS:UWXZ",
+				 "0b:c:de:fg:hi:j:kl:m:no:p:qr:s:t:uw:xyABD:EFJKHzCILN:OP:RS:UWXZ",
 				 longopts, NULL)) != EOF)
 		switch (ch) {
 		case 'b':
@@ -2916,6 +3007,9 @@ main(int argc, char **argv)
 		case 'g':
 			filldata = *optarg;
 			break;
+		case 'h':
+			hugepages = 1;
+			break;
 		case 'i':
 			integrity = 1;
 			logdev = strdup(optarg);
@@ -3229,15 +3323,7 @@ main(int argc, char **argv)
 			exit(95);
 		}
 	}
-	original_buf = (char *) malloc(maxfilelen);
-	for (i = 0; i < maxfilelen; i++)
-		original_buf[i] = random() % 256;
-	good_buf = (char *) malloc(maxfilelen + writebdy);
-	good_buf = round_ptr_up(good_buf, writebdy, 0);
-	memset(good_buf, '\0', maxfilelen);
-	temp_buf = (char *) malloc(maxoplen + readbdy);
-	temp_buf = round_ptr_up(temp_buf, readbdy, 0);
-	memset(temp_buf, '\0', maxoplen);
+	init_buffers();
 	if (lite) {	/* zero entire existing file */
 		ssize_t written;
 
Signature · 1 line
2.47.1
#4 lore raw

Re: [PATCH v2 1/2] fsx: support reads/writes from buffers backed by hugepages

In reply to Joanne Koong
Quoted 13 lines · Joanne Koong

Add support for reads/writes from buffers backed by hugepages. This can be enabled through the '-h' flag. This flag should only be used on systems where THP capabilities are enabled.

This is motivated by a recent bug that was due to faulty handling of userspace buffers backed by hugepages. This patch is a mitigation against problems like this in the future.

---
 ltp/fsx.c | 108 ++++++++++++++++++++++++++++++++++++++++++++++++------
 1 file changed, 97 insertions(+), 11 deletions(-)

[snip]

ltp/fsx.c keep_running(void) · line 2903

Quoted 12 lines
ltp/fsx.c +12 -0
+static void *
+init_hugepages_buf(unsigned len, int hugepage_size, int alignment)
+{
+	void *buf;
+	long buf_size = roundup(len, hugepage_size) + alignment;
+
+	if (posix_memalign(&buf, hugepage_size, buf_size)) {
+		prterr("posix_memalign for buf");
+		return NULL;
+	}
+	memset(buf, '\0', buf_size);
+	if (madvise(buf, buf_size, MADV_COLLAPSE)) {

Hi Joanne,

Sorry I have to drop this patchset from the "upcoming" release v2025.01.12. Due to it cause a regression build error on older system, e.g. RHEL-9:

[CC] fsx
fsx.c: In function 'init_hugepages_buf':
fsx.c:2935:36: error: 'MADV_COLLAPSE' undeclared (first use in this function); did you mean 'MADV_COLD'?

  2935 |         if (madvise(buf, buf_size, MADV_COLLAPSE)) {
       |                                    ^~~~~~~~~~~~~
       |                                    MADV_COLD

fsx.c:2935:36: note: each undeclared identifier is reported only once for each function it appears in gmake[4]: *** [Makefile:51: fsx] Error 1
gmake[4]: *** Waiting for unfinished jobs....
gmake[3]: *** [include/buildrules:30: ltp] Error 2

It might cause xfstests totally can't be used on downstream systems, so it can't catch up the release of this weekend. Sorry about that, let's try to have it in next release :)

Thanks,
Zorro

Quoted 89 lines
ltp/fsx.c +51 -10
+		prterr("madvise collapse for buf");
+		free(buf);
+		return NULL;
+	}
+
+	return buf;
+}
+
+static void
+init_buffers(void)
+{
+	int i;
+
+	original_buf = (char *) malloc(maxfilelen);
+	for (i = 0; i < maxfilelen; i++)
+		original_buf[i] = random() % 256;
+	if (hugepages) {
+		long hugepage_size = get_hugepage_size();
+		if (hugepage_size == -1) {
+			prterr("get_hugepage_size()");
+			exit(100);
+		}
+		good_buf = init_hugepages_buf(maxfilelen, hugepage_size, writebdy);
+		if (!good_buf) {
+			prterr("init_hugepages_buf failed for good_buf");
+			exit(101);
+		}
+
+		temp_buf = init_hugepages_buf(maxoplen, hugepage_size, readbdy);
+		if (!temp_buf) {
+			prterr("init_hugepages_buf failed for temp_buf");
+			exit(101);
+		}
+	} else {
+		unsigned long good_buf_len = maxfilelen + writebdy;
+		unsigned long temp_buf_len = maxoplen + readbdy;
+
+		good_buf = (char *) malloc(good_buf_len);
+		memset(good_buf, '\0', good_buf_len);
+		temp_buf = (char *) malloc(temp_buf_len);
+		memset(temp_buf, '\0', temp_buf_len);
+	}
+	good_buf = round_ptr_up(good_buf, writebdy, 0);
+	temp_buf = round_ptr_up(temp_buf, readbdy, 0);
+}
+
 static struct option longopts[] = {
 	{"replay-ops", required_argument, 0, 256},
 	{"record-ops", optional_argument, 0, 255},
@@ -2883,7 +2974,7 @@ main(int argc, char **argv)
 	setvbuf(stdout, (char *)0, _IOLBF, 0); /* line buffered stdout */
 
 	while ((ch = getopt_long(argc, argv,
-				 "0b:c:de:fg:i:j:kl:m:no:p:qr:s:t:uw:xyABD:EFJKHzCILN:OP:RS:UWXZ",
+				 "0b:c:de:fg:hi:j:kl:m:no:p:qr:s:t:uw:xyABD:EFJKHzCILN:OP:RS:UWXZ",
 				 longopts, NULL)) != EOF)
 		switch (ch) {
 		case 'b':
@@ -2916,6 +3007,9 @@ main(int argc, char **argv)
 		case 'g':
 			filldata = *optarg;
 			break;
+		case 'h':
+			hugepages = 1;
+			break;
 		case 'i':
 			integrity = 1;
 			logdev = strdup(optarg);
@@ -3229,15 +3323,7 @@ main(int argc, char **argv)
 			exit(95);
 		}
 	}
-	original_buf = (char *) malloc(maxfilelen);
-	for (i = 0; i < maxfilelen; i++)
-		original_buf[i] = random() % 256;
-	good_buf = (char *) malloc(maxfilelen + writebdy);
-	good_buf = round_ptr_up(good_buf, writebdy, 0);
-	memset(good_buf, '\0', maxfilelen);
-	temp_buf = (char *) malloc(maxoplen + readbdy);
-	temp_buf = round_ptr_up(temp_buf, readbdy, 0);
-	memset(temp_buf, '\0', maxoplen);
+	init_buffers();
 	if (lite) {	/* zero entire existing file */
 		ssize_t written;
 
Signature · 1 line
2.47.1
#5 lore raw

Re: [PATCH v2 1/2] fsx: support reads/writes from buffers backed by hugepages

Quoted 42 lines · Zorro Lang
Quoted 13 lines · Joanne Koong

Add support for reads/writes from buffers backed by hugepages. This can be enabled through the '-h' flag. This flag should only be used on systems where THP capabilities are enabled.

This is motivated by a recent bug that was due to faulty handling of userspace buffers backed by hugepages. This patch is a mitigation against problems like this in the future.

---
 ltp/fsx.c | 108 ++++++++++++++++++++++++++++++++++++++++++++++++------
 1 file changed, 97 insertions(+), 11 deletions(-)

[snip]

ltp/fsx.c keep_running(void) · line 2903

Quoted 12 lines
ltp/fsx.c +12 -0
+static void *
+init_hugepages_buf(unsigned len, int hugepage_size, int alignment)
+{
+     void *buf;
+     long buf_size = roundup(len, hugepage_size) + alignment;
+
+     if (posix_memalign(&buf, hugepage_size, buf_size)) {
+             prterr("posix_memalign for buf");
+             return NULL;
+     }
+     memset(buf, '\0', buf_size);
+     if (madvise(buf, buf_size, MADV_COLLAPSE)) {

Hi Joanne,

Sorry I have to drop this patchset from the "upcoming" release v2025.01.12. Due to it cause a regression build error on older system, e.g. RHEL-9:

[CC] fsx
fsx.c: In function 'init_hugepages_buf':
fsx.c:2935:36: error: 'MADV_COLLAPSE' undeclared (first use in this function); did you mean 'MADV_COLD'?

  2935 |         if (madvise(buf, buf_size, MADV_COLLAPSE)) {
       |                                    ^~~~~~~~~~~~~
       |                                    MADV_COLD
Quoted 8 lines

fsx.c:2935:36: note: each undeclared identifier is reported only once for each function it appears in gmake[4]: *** [Makefile:51: fsx] Error 1
gmake[4]: *** Waiting for unfinished jobs....
gmake[3]: *** [include/buildrules:30: ltp] Error 2

It might cause xfstests totally can't be used on downstream systems, so it can't catch up the release of this weekend. Sorry about that, let's try to have it in next release :)

Hi Zorro,

Thanks for the update. I'll submit a v3 of this patch that gates this function behind #ifdef MADV_COLLAPSE, and hopefully that should fix this issue.

Thanks,
Joanne

Quoted 95 lines

Thanks,
Zorro

Quoted 89 lines
ltp/fsx.c +51 -11
+             prterr("madvise collapse for buf");
+             free(buf);
+             return NULL;
+     }
+
+     return buf;
+}
+
+static void
+init_buffers(void)
+{
+     int i;
+
+     original_buf = (char *) malloc(maxfilelen);
+     for (i = 0; i < maxfilelen; i++)
+             original_buf[i] = random() % 256;
+     if (hugepages) {
+             long hugepage_size = get_hugepage_size();
+             if (hugepage_size == -1) {
+                     prterr("get_hugepage_size()");
+                     exit(100);
+             }
+             good_buf = init_hugepages_buf(maxfilelen, hugepage_size, writebdy);
+             if (!good_buf) {
+                     prterr("init_hugepages_buf failed for good_buf");
+                     exit(101);
+             }
+
+             temp_buf = init_hugepages_buf(maxoplen, hugepage_size, readbdy);
+             if (!temp_buf) {
+                     prterr("init_hugepages_buf failed for temp_buf");
+                     exit(101);
+             }
+     } else {
+             unsigned long good_buf_len = maxfilelen + writebdy;
+             unsigned long temp_buf_len = maxoplen + readbdy;
+
+             good_buf = (char *) malloc(good_buf_len);
+             memset(good_buf, '\0', good_buf_len);
+             temp_buf = (char *) malloc(temp_buf_len);
+             memset(temp_buf, '\0', temp_buf_len);
+     }
+     good_buf = round_ptr_up(good_buf, writebdy, 0);
+     temp_buf = round_ptr_up(temp_buf, readbdy, 0);
+}
+
 static struct option longopts[] = {
      {"replay-ops", required_argument, 0, 256},
      {"record-ops", optional_argument, 0, 255},
@@ -2883,7 +2974,7 @@ main(int argc, char **argv)
      setvbuf(stdout, (char *)0, _IOLBF, 0); /* line buffered stdout */
 
      while ((ch = getopt_long(argc, argv,
-                              "0b:c:de:fg:i:j:kl:m:no:p:qr:s:t:uw:xyABD:EFJKHzCILN:OP:RS:UWXZ",
+                              "0b:c:de:fg:hi:j:kl:m:no:p:qr:s:t:uw:xyABD:EFJKHzCILN:OP:RS:UWXZ",
                               longopts, NULL)) != EOF)
              switch (ch) {
              case 'b':
@@ -2916,6 +3007,9 @@ main(int argc, char **argv)
              case 'g':
                      filldata = *optarg;
                      break;
+             case 'h':
+                     hugepages = 1;
+                     break;
              case 'i':
                      integrity = 1;
                      logdev = strdup(optarg);
@@ -3229,15 +3323,7 @@ main(int argc, char **argv)
                      exit(95);
              }
      }
-     original_buf = (char *) malloc(maxfilelen);
-     for (i = 0; i < maxfilelen; i++)
-             original_buf[i] = random() % 256;
-     good_buf = (char *) malloc(maxfilelen + writebdy);
-     good_buf = round_ptr_up(good_buf, writebdy, 0);
-     memset(good_buf, '\0', maxfilelen);
-     temp_buf = (char *) malloc(maxoplen + readbdy);
-     temp_buf = round_ptr_up(temp_buf, readbdy, 0);
-     memset(temp_buf, '\0', maxoplen);
+     init_buffers();
      if (lite) {     /* zero entire existing file */
              ssize_t written;
 
--

2.47.1

#6 lore raw

PATCH v2 2/2 generic: add tests for read/writes from hugepages-backed buffers

In reply to Joanne Koong

Add generic tests 758 and 759 for testing reads/writes from buffers backed by hugepages.

---
 common/rc             | 13 +++++++++++++
 tests/generic/758     | 22 ++++++++++++++++++++++
 tests/generic/758.out |  4 ++++
 tests/generic/759     | 26 ++++++++++++++++++++++++++
 tests/generic/759.out |  4 ++++
 5 files changed, 69 insertions(+)
 create mode 100755 tests/generic/758
 create mode 100644 tests/generic/758.out
 create mode 100755 tests/generic/759
 create mode 100644 tests/generic/759.out
common/rc +13 -0
index 1b2e4508..825e350e 100644
@@ -3016,6 +3016,19 @@ _require_xfs_io_command()
 	fi
 }
 
+# check that the system supports transparent hugepages
+_require_thp()
+{
+    if [ ! -e /sys/kernel/mm/transparent_hugepage/enabled ]; then
+	    _notrun "system doesn't support transparent hugepages"
+    fi
+
+    thp_status=$(cat /sys/kernel/mm/transparent_hugepage/enabled)
+    if [[ $thp_status == *"[never]"* ]]; then
+	    _notrun "system doesn't have transparent hugepages enabled"
+    fi
+}
+
 # check that kernel and filesystem support direct I/O, and check if "$1" size
 # aligned (optional) is supported
 _require_odirect()
tests/generic/758 new file +22 -0
new file mode 100755
index 00000000..e7cd8cdc
@@ -0,0 +1,22 @@
+#! /bin/bash
+# SPDX-License-Identifier: GPL-2.0
+#
+# FS QA Test No. 758
+#
+# fsx exercising reads/writes from userspace buffers
+# backed by hugepages
+#
+. ./common/preamble
+_begin_fstest rw auto quick
+
+. ./common/filter
+
+_require_test
+_require_thp
+
+run_fsx -N 10000            -l 500000 -h
+run_fsx -N 10000  -o 8192   -l 500000 -h
+run_fsx -N 10000  -o 128000 -l 500000 -h
+
+status=0
+exit
tests/generic/758.out new file +4 -0
new file mode 100644
index 00000000..af04bb14
@@ -0,0 +1,4 @@
+QA output created by 758
+fsx -N 10000 -l 500000 -h
+fsx -N 10000 -o 8192 -l 500000 -h
+fsx -N 10000 -o 128000 -l 500000 -h
tests/generic/759 new file +26 -0
new file mode 100755
index 00000000..514e7603
@@ -0,0 +1,26 @@
+#! /bin/bash
+# SPDX-License-Identifier: GPL-2.0
+#
+# FS QA Test No. 759
+#
+# fsx exercising direct IO reads/writes from userspace buffers
+# backed by hugepages
+#
+. ./common/preamble
+_begin_fstest rw auto quick
+
+. ./common/filter
+
+_require_test
+_require_odirect
+_require_thp
+
+psize=`$here/src/feature -s`
+bsize=`$here/src/min_dio_alignment $TEST_DIR $TEST_DEV`
+
+run_fsx -N 10000            -l 500000 -r PSIZE -t BSIZE -w BSIZE -Z -R -W -h
+run_fsx -N 10000  -o 8192   -l 500000 -r PSIZE -t BSIZE -w BSIZE -Z -R -W -h
+run_fsx -N 10000  -o 128000 -l 500000 -r PSIZE -t BSIZE -w BSIZE -Z -R -W -h
+
+status=0
+exit
tests/generic/759.out new file +4 -0
new file mode 100644
index 00000000..86bb66ef
@@ -0,0 +1,4 @@
+QA output created by 759
+fsx -N 10000 -l 500000 -r PSIZE -t BSIZE -w BSIZE -Z -R -W -h
+fsx -N 10000 -o 8192 -l 500000 -r PSIZE -t BSIZE -w BSIZE -Z -R -W -h
+fsx -N 10000 -o 128000 -l 500000 -r PSIZE -t BSIZE -w BSIZE -Z -R -W -h
Signature · 1 line
2.47.1
Reviewed-by on v2 2/2
#7 lore raw

Re: [PATCH v2 2/2] generic: add tests for read/writes from hugepages-backed buffers

Quoted 5 lines · Joanne Koong

Add generic tests 758 and 759 for testing reads/writes from buffers backed by hugepages.

---

Quoted 118 lines
 common/rc             | 13 +++++++++++++
 tests/generic/758     | 22 ++++++++++++++++++++++
 tests/generic/758.out |  4 ++++
 tests/generic/759     | 26 ++++++++++++++++++++++++++
 tests/generic/759.out |  4 ++++
 5 files changed, 69 insertions(+)
 create mode 100755 tests/generic/758
 create mode 100644 tests/generic/758.out
 create mode 100755 tests/generic/759
 create mode 100644 tests/generic/759.out
common/rc +13 -0
index 1b2e4508..825e350e 100644
@@ -3016,6 +3016,19 @@ _require_xfs_io_command()
 	fi
 }
 
+# check that the system supports transparent hugepages
+_require_thp()
+{
+    if [ ! -e /sys/kernel/mm/transparent_hugepage/enabled ]; then
+	    _notrun "system doesn't support transparent hugepages"
+    fi
+
+    thp_status=$(cat /sys/kernel/mm/transparent_hugepage/enabled)
+    if [[ $thp_status == *"[never]"* ]]; then
+	    _notrun "system doesn't have transparent hugepages enabled"
+    fi
+}
+
 # check that kernel and filesystem support direct I/O, and check if "$1" size
 # aligned (optional) is supported
 _require_odirect()
tests/generic/758 new file +22 -0
new file mode 100755
index 00000000..e7cd8cdc
@@ -0,0 +1,22 @@
+#! /bin/bash
+# SPDX-License-Identifier: GPL-2.0
+#
+# FS QA Test No. 758
+#
+# fsx exercising reads/writes from userspace buffers
+# backed by hugepages
+#
+. ./common/preamble
+_begin_fstest rw auto quick
+
+. ./common/filter
+
+_require_test
+_require_thp
+
+run_fsx -N 10000            -l 500000 -h
+run_fsx -N 10000  -o 8192   -l 500000 -h
+run_fsx -N 10000  -o 128000 -l 500000 -h
+
+status=0
+exit
tests/generic/758.out new file +4 -0
new file mode 100644
index 00000000..af04bb14
@@ -0,0 +1,4 @@
+QA output created by 758
+fsx -N 10000 -l 500000 -h
+fsx -N 10000 -o 8192 -l 500000 -h
+fsx -N 10000 -o 128000 -l 500000 -h
tests/generic/759 new file +26 -0
new file mode 100755
index 00000000..514e7603
@@ -0,0 +1,26 @@
+#! /bin/bash
+# SPDX-License-Identifier: GPL-2.0
+#
+# FS QA Test No. 759
+#
+# fsx exercising direct IO reads/writes from userspace buffers
+# backed by hugepages
+#
+. ./common/preamble
+_begin_fstest rw auto quick
+
+. ./common/filter
+
+_require_test
+_require_odirect
+_require_thp
+
+psize=`$here/src/feature -s`
+bsize=`$here/src/min_dio_alignment $TEST_DIR $TEST_DEV`
+
+run_fsx -N 10000            -l 500000 -r PSIZE -t BSIZE -w BSIZE -Z -R -W -h
+run_fsx -N 10000  -o 8192   -l 500000 -r PSIZE -t BSIZE -w BSIZE -Z -R -W -h
+run_fsx -N 10000  -o 128000 -l 500000 -r PSIZE -t BSIZE -w BSIZE -Z -R -W -h
+
+status=0
+exit
tests/generic/759.out new file +4 -0
new file mode 100644
index 00000000..86bb66ef
@@ -0,0 +1,4 @@
+QA output created by 759
+fsx -N 10000 -l 500000 -r PSIZE -t BSIZE -w BSIZE -Z -R -W -h
+fsx -N 10000 -o 8192 -l 500000 -r PSIZE -t BSIZE -w BSIZE -Z -R -W -h
+fsx -N 10000 -o 128000 -l 500000 -r PSIZE -t BSIZE -w BSIZE -Z -R -W -h
Signature · 1 line
2.47.1