fstests: test reads/writes from hugepages-backed buffers
v2 cover fstests: test reads/writes from hugepages-backed buffers
There was a recent bug in rc1 [1] that was due to faulty handling for userspace buffers backed by hugepages.
This patchset adds generic tests for reads/writes from buffers backed by hugepages.
Changelog:
v1: https://lore.kernel.org/linux-fsdevel/[email protected]/
- Refactor out buffer initialization (Brian)
- Update commit messages of 1st patch (Brian)
- Use << 10 instead of * 1024 (Nirjhar)
- Replace CONFIG_TRANSPARENT_HUGEPAGE check with checking 'sys/kernel/mm/transparent_hugepage/' (Darrick)
- Integrate readbdy and writebdy options
- Update options of generic/759 to include psize/bsize
Joanne Koong (2):
fsx: support reads/writes from buffers backed by hugepages
generic: add tests for read/writes from hugepages-backed buffers
common/rc | 13 +++++ ltp/fsx.c | 108 +++++++++++++++++++++++++++++++++++++----- tests/generic/758 | 22 +++++++++ tests/generic/758.out | 4 ++ tests/generic/759 | 26 ++++++++++ tests/generic/759.out | 4 ++ 6 files changed, 166 insertions(+), 11 deletions(-) create mode 100755 tests/generic/758 create mode 100644 tests/generic/758.out create mode 100755 tests/generic/759 create mode 100644 tests/generic/759.out
Signature · 1 line
2.47.1
PATCH v2 1/2 fsx: support reads/writes from buffers backed by hugepages
Add support for reads/writes from buffers backed by hugepages. This can be enabled through the '-h' flag. This flag should only be used on systems where THP capabilities are enabled.
This is motivated by a recent bug that was due to faulty handling of userspace buffers backed by hugepages. This patch is a mitigation against problems like this in the future.
--- ltp/fsx.c | 108 ++++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 97 insertions(+), 11 deletions(-)
ltp/fsx.c +97 -11
@@ -190,6 +190,7 @@ int o_direct; /* -Z */int aio =0 ;int uring =0 ;int mark_nr =0 ; +int hugepages =0 ;/* -h flag */ int page_size;int page_mask;
@@ -2471,7 +2472,7 @@ voidusage (void) { fprintf(stdout, "usage: %s", - "fsx [-dfknqxyzBEFHIJKLORWXZ0]\n\ + "fsx [-dfhknqxyzBEFHIJKLORWXZ0]\n\ [-b opnum] [-c Prob] [-g filldata] [-i logdev] [-j logid]\n\ [-l flen] [-m start:end] [-o oplen] [-p progressinterval]\n\ [-r readbdy] [-s style] [-t truncbdy] [-w writebdy]\n\
@@ -2484,6 +2485,7 @@ usage(void) -e: pollute post-eof on size changes (default 0)\n\ -f: flush and invalidate cache after I/O\n\ -g X: write character X instead of random generated data\n\ + -h hugepages: use buffers backed by hugepages for reads/writes\n\ -i logdev: do integrity testing, logdev is the dm log writes device\n\ -j logid: prefix debug log messsages with this id\n\ -k: do not truncate existing file and use its size as upper bound on file size\n\
@@ -2856,6 +2858,95 @@ keep_running(void)return numops-- !=0 ; } +static long +get_hugepage_size (void ) +{ +const char str[] ="Hugepagesize:" ; +size_t str_len =sizeof (str) -1 ; +unsigned int hugepage_size =0 ; +char buffer[64 ]; +FILE *file; + + file =fopen ("/proc/meminfo" ,"r" ); +if (!file) { +prterr ("get_hugepage_size: fopen /proc/meminfo" ); +return -1 ; + } +while (fgets (buffer,sizeof (buffer), file)) { +if (strncmp (buffer, str, str_len) ==0 ) { +sscanf (buffer + str_len,"%u" , &hugepage_size); +break ; + } + } +fclose (file); +if (!hugepage_size) { +prterr ("get_hugepage_size: failed to find " +"hugepage size in /proc/meminfo\n" ); +return -1 ; + } + +/* convert from KiB to bytes */ +return hugepage_size <<10 ; +} + +static void * +init_hugepages_buf (unsigned len,int hugepage_size,int alignment) +{ +void *buf; +long buf_size =roundup (len, hugepage_size) + alignment; + +if (posix_memalign (&buf, hugepage_size, buf_size)) { +prterr ("posix_memalign for buf" ); +return NULL ; + } +memset (buf,'\0' , buf_size); +if (madvise (buf, buf_size,MADV_COLLAPSE )) { +prterr ("madvise collapse for buf" ); +free (buf); +return NULL ; + } + +return buf; +} + +static void +init_buffers (void ) +{ +int i; + + original_buf = (char *)malloc (maxfilelen); +for (i =0 ; i < maxfilelen; i++) + original_buf[i] =random () %256 ; +if (hugepages) { +long hugepage_size =get_hugepage_size (); +if (hugepage_size ==-1 ) { +prterr ("get_hugepage_size()" ); +exit (100 ); + } + good_buf =init_hugepages_buf (maxfilelen, hugepage_size, writebdy); +if (!good_buf) { +prterr ("init_hugepages_buf failed for good_buf" ); +exit (101 ); + } + + temp_buf =init_hugepages_buf (maxoplen, hugepage_size, readbdy); +if (!temp_buf) { +prterr ("init_hugepages_buf failed for temp_buf" ); +exit (101 ); + } + }else { +unsigned long good_buf_len = maxfilelen + writebdy; +unsigned long temp_buf_len = maxoplen + readbdy; + + good_buf = (char *)malloc (good_buf_len); +memset (good_buf,'\0' , good_buf_len); + temp_buf = (char *)malloc (temp_buf_len); +memset (temp_buf,'\0' , temp_buf_len); + } + good_buf =round_ptr_up (good_buf, writebdy,0 ); + temp_buf =round_ptr_up (temp_buf, readbdy,0 ); +} +static struct option longopts[] = { {"replay-ops" , required_argument,0 ,256 }, {"record-ops" , optional_argument,0 ,255 },
@@ -2883,7 +2974,7 @@ main(int argc, char **argv)setvbuf (stdout, (char *)0 , _IOLBF,0 );/* line buffered stdout */ while ((ch =getopt_long (argc, argv, -"0b:c:de:fg:i:j:kl:m:no:p:qr:s:t:uw:xyABD:EFJKHzCILN:OP:RS:UWXZ" , +"0b:c:de:fg:hi:j:kl:m:no:p:qr:s:t:uw:xyABD:EFJKHzCILN:OP:RS:UWXZ" , longopts,NULL )) !=EOF )switch (ch) {case 'b' :
@@ -2916,6 +3007,9 @@ main(int argc, char **argv)case 'g' : filldata = *optarg;break ; +case 'h' : + hugepages =1 ; +break ;case 'i' : integrity =1 ; logdev =strdup (optarg);
@@ -3229,15 +3323,7 @@ main(int argc, char **argv)exit (95 ); } } - original_buf = (char *)malloc (maxfilelen); -for (i =0 ; i < maxfilelen; i++) - original_buf[i] =random () %256 ; - good_buf = (char *)malloc (maxfilelen + writebdy); - good_buf =round_ptr_up (good_buf, writebdy,0 ); -memset (good_buf,'\0' , maxfilelen); - temp_buf = (char *)malloc (maxoplen + readbdy); - temp_buf =round_ptr_up (temp_buf, readbdy,0 ); -memset (temp_buf,'\0' , maxoplen); +init_buffers ();if (lite) {/* zero entire existing file */ ssize_t written;
Signature · 1 line
2.47.1
Re: [PATCH v2 1/2] fsx: support reads/writes from buffers backed by hugepages
Quoted 65 lines · Joanne Koong
Add support for reads/writes from buffers backed by hugepages. This can be enabled through the '-h' flag. This flag should only be used on systems where THP capabilities are enabled.
This is motivated by a recent bug that was due to faulty handling of userspace buffers backed by hugepages. This patch is a mitigation against problems like this in the future.
Signed-off-by Joanne Koong <[email protected]>--- ltp/fsx.c | 108 ++++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 97 insertions(+), 11 deletions(-)ltp/fsx.c +22
-1@@ -190,6 +190,7 @@ int o_direct; /* -Z */int aio =0 ;int uring =0 ;int mark_nr =0 ; +int hugepages =0 ;/* -h flag */ int page_size;int page_mask;@@ -2471,7 +2472,7 @@ voidusage (void) { fprintf(stdout, "usage: %s", - "fsx [-dfknqxyzBEFHIJKLORWXZ0]\n\ + "fsx [-dfhknqxyzBEFHIJKLORWXZ0]\n\ [-b opnum] [-c Prob] [-g filldata] [-i logdev] [-j logid]\n\ [-l flen] [-m start:end] [-o oplen] [-p progressinterval]\n\ [-r readbdy] [-s style] [-t truncbdy] [-w writebdy]\n\@@ -2484,6 +2485,7 @@ usage(void) -e: pollute post-eof on size changes (default 0)\n\ -f: flush and invalidate cache after I/O\n\ -g X: write character X instead of random generated data\n\ + -h hugepages: use buffers backed by hugepages for reads/writes\n\ -i logdev: do integrity testing, logdev is the dm log writes device\n\ -j logid: prefix debug log messsages with this id\n\ -k: do not truncate existing file and use its size as upper bound on file size\n\@@ -2856,6 +2858,95 @@ keep_running(void)return numops-- !=0 ; } +static long +get_hugepage_size (void ) +{ +const char str[] ="Hugepagesize:" ; +size_t str_len =sizeof (str) -1 ; +unsigned int hugepage_size =0 ; +char buffer[64 ]; +FILE *file; + + file =fopen ("/proc/meminfo" ,"r" ); +if (!file) { +prterr ("get_hugepage_size: fopen /proc/meminfo" ); +return -1 ; + } +while (fgets (buffer,sizeof (buffer), file)) { +if (strncmp (buffer, str, str_len) ==0 ) { +sscanf (buffer + str_len,"%u" , &hugepage_size); +break ; + }
ltp/fsx.c keep_running(void) · line 2889
Quoted 10 lines
ltp/fsx.c +10
-0+ } +fclose (file); +if (!hugepage_size) { +prterr ("get_hugepage_size: failed to find " +"hugepage size in /proc/meminfo\n" ); +return -1 ; + } + +/* convert from KiB to bytes */ +return hugepage_size <<10 ;
Thanks for the change.
Quoted 43 lines
ltp/fsx.c +43
-0+} + +static void * +init_hugepages_buf (unsigned len,int hugepage_size,int alignment) +{ +void *buf; +long buf_size =roundup (len, hugepage_size) + alignment; + +if (posix_memalign (&buf, hugepage_size, buf_size)) { +prterr ("posix_memalign for buf" ); +return NULL ; + } +memset (buf,'\0' , buf_size); +if (madvise (buf, buf_size,MADV_COLLAPSE )) { +prterr ("madvise collapse for buf" ); +free (buf); +return NULL ; + } + +return buf; +} + +static void +init_buffers (void ) +{ +int i; + + original_buf = (char *)malloc (maxfilelen); +for (i =0 ; i < maxfilelen; i++) + original_buf[i] =random () %256 ; +if (hugepages) { +long hugepage_size =get_hugepage_size (); +if (hugepage_size ==-1 ) { +prterr ("get_hugepage_size()" ); +exit (100 ); + } + good_buf =init_hugepages_buf (maxfilelen, hugepage_size, writebdy); +if (!good_buf) { +prterr ("init_hugepages_buf failed for good_buf" ); +exit (101 ); + } + + temp_buf =init_hugepages_buf (maxoplen, hugepage_size, readbdy);
ltp/fsx.c keep_running(void) · line 2942
Quoted 10 lines
ltp/fsx.c +10
-0+if (!temp_buf) { +prterr ("init_hugepages_buf failed for temp_buf" ); +exit (101 ); + } + }else { +unsigned long good_buf_len = maxfilelen + writebdy; +unsigned long temp_buf_len = maxoplen + readbdy; + + good_buf = (char *)malloc (good_buf_len); +memset (good_buf,'\0' , good_buf_len);
minor: maybe good_buf = (char *)calloc(1, good_buf_len); So we don't need to use memset.
Quoted 46 lines
ltp/fsx.c +12
-10+ temp_buf = (char *)malloc (temp_buf_len); +memset (temp_buf,'\0' , temp_buf_len); + } + good_buf =round_ptr_up (good_buf, writebdy,0 ); + temp_buf =round_ptr_up (temp_buf, readbdy,0 ); +} +static struct option longopts[] = { {"replay-ops" , required_argument,0 ,256 }, {"record-ops" , optional_argument,0 ,255 },@@ -2883,7 +2974,7 @@ main(int argc, char **argv)setvbuf (stdout, (char *)0 , _IOLBF,0 );/* line buffered stdout */ while ((ch =getopt_long (argc, argv, -"0b:c:de:fg:i:j:kl:m:no:p:qr:s:t:uw:xyABD:EFJKHzCILN:OP:RS:UWXZ" , +"0b:c:de:fg:hi:j:kl:m:no:p:qr:s:t:uw:xyABD:EFJKHzCILN:OP:RS:UWXZ" , longopts,NULL )) !=EOF )switch (ch) {case 'b' :@@ -2916,6 +3007,9 @@ main(int argc, char **argv)case 'g' : filldata = *optarg;break ; +case 'h' : + hugepages =1 ; +break ;case 'i' : integrity =1 ; logdev =strdup (optarg);@@ -3229,15 +3323,7 @@ main(int argc, char **argv)exit (95 ); } } - original_buf = (char *)malloc (maxfilelen); -for (i =0 ; i < maxfilelen; i++) - original_buf[i] =random () %256 ; - good_buf = (char *)malloc (maxfilelen + writebdy); - good_buf =round_ptr_up (good_buf, writebdy,0 ); -memset (good_buf,'\0' , maxfilelen); - temp_buf = (char *)malloc (maxoplen + readbdy); - temp_buf =round_ptr_up (temp_buf, readbdy,0 ); -memset (temp_buf,'\0' , maxoplen); +init_buffers ();if (lite) {/* zero entire existing file */ ssize_t written;
Signature · 3 lines
Nirjhar Roy Linux Kernel Developer IBM, Bangalore
Re: [PATCH v2 1/2] fsx: support reads/writes from buffers backed by hugepages
In reply to Joanne KoongQuoted 4 lines · Joanne Koong
Add support for reads/writes from buffers backed by hugepages. This can be enabled through the '-h' flag. This flag should only be used on systems where THP capabilities are enabled.
Quoted 8 lines
This is motivated by a recent bug that was due to faulty handling of userspace buffers backed by hugepages. This patch is a mitigation against problems like this in the future.
Signed-off-by Joanne Koong <[email protected]>--- ltp/fsx.c | 108 ++++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 97 insertions(+), 11 deletions(-)
Thanks for the buffer init code cleanup. This looks much nicer to me now. Modulo comments from others:
Quoted 165 lines
ltp/fsx.c +97
-11@@ -190,6 +190,7 @@ int o_direct; /* -Z */int aio =0 ;int uring =0 ;int mark_nr =0 ; +int hugepages =0 ;/* -h flag */ int page_size;int page_mask;@@ -2471,7 +2472,7 @@ voidusage (void) { fprintf(stdout, "usage: %s", - "fsx [-dfknqxyzBEFHIJKLORWXZ0]\n\ + "fsx [-dfhknqxyzBEFHIJKLORWXZ0]\n\ [-b opnum] [-c Prob] [-g filldata] [-i logdev] [-j logid]\n\ [-l flen] [-m start:end] [-o oplen] [-p progressinterval]\n\ [-r readbdy] [-s style] [-t truncbdy] [-w writebdy]\n\@@ -2484,6 +2485,7 @@ usage(void) -e: pollute post-eof on size changes (default 0)\n\ -f: flush and invalidate cache after I/O\n\ -g X: write character X instead of random generated data\n\ + -h hugepages: use buffers backed by hugepages for reads/writes\n\ -i logdev: do integrity testing, logdev is the dm log writes device\n\ -j logid: prefix debug log messsages with this id\n\ -k: do not truncate existing file and use its size as upper bound on file size\n\@@ -2856,6 +2858,95 @@ keep_running(void)return numops-- !=0 ; } +static long +get_hugepage_size (void ) +{ +const char str[] ="Hugepagesize:" ; +size_t str_len =sizeof (str) -1 ; +unsigned int hugepage_size =0 ; +char buffer[64 ]; +FILE *file; + + file =fopen ("/proc/meminfo" ,"r" ); +if (!file) { +prterr ("get_hugepage_size: fopen /proc/meminfo" ); +return -1 ; + } +while (fgets (buffer,sizeof (buffer), file)) { +if (strncmp (buffer, str, str_len) ==0 ) { +sscanf (buffer + str_len,"%u" , &hugepage_size); +break ; + } + } +fclose (file); +if (!hugepage_size) { +prterr ("get_hugepage_size: failed to find " +"hugepage size in /proc/meminfo\n" ); +return -1 ; + } + +/* convert from KiB to bytes */ +return hugepage_size <<10 ; +} + +static void * +init_hugepages_buf (unsigned len,int hugepage_size,int alignment) +{ +void *buf; +long buf_size =roundup (len, hugepage_size) + alignment; + +if (posix_memalign (&buf, hugepage_size, buf_size)) { +prterr ("posix_memalign for buf" ); +return NULL ; + } +memset (buf,'\0' , buf_size); +if (madvise (buf, buf_size,MADV_COLLAPSE )) { +prterr ("madvise collapse for buf" ); +free (buf); +return NULL ; + } + +return buf; +} + +static void +init_buffers (void ) +{ +int i; + + original_buf = (char *)malloc (maxfilelen); +for (i =0 ; i < maxfilelen; i++) + original_buf[i] =random () %256 ; +if (hugepages) { +long hugepage_size =get_hugepage_size (); +if (hugepage_size ==-1 ) { +prterr ("get_hugepage_size()" ); +exit (100 ); + } + good_buf =init_hugepages_buf (maxfilelen, hugepage_size, writebdy); +if (!good_buf) { +prterr ("init_hugepages_buf failed for good_buf" ); +exit (101 ); + } + + temp_buf =init_hugepages_buf (maxoplen, hugepage_size, readbdy); +if (!temp_buf) { +prterr ("init_hugepages_buf failed for temp_buf" ); +exit (101 ); + } + }else { +unsigned long good_buf_len = maxfilelen + writebdy; +unsigned long temp_buf_len = maxoplen + readbdy; + + good_buf = (char *)malloc (good_buf_len); +memset (good_buf,'\0' , good_buf_len); + temp_buf = (char *)malloc (temp_buf_len); +memset (temp_buf,'\0' , temp_buf_len); + } + good_buf =round_ptr_up (good_buf, writebdy,0 ); + temp_buf =round_ptr_up (temp_buf, readbdy,0 ); +} +static struct option longopts[] = { {"replay-ops" , required_argument,0 ,256 }, {"record-ops" , optional_argument,0 ,255 },@@ -2883,7 +2974,7 @@ main(int argc, char **argv)setvbuf (stdout, (char *)0 , _IOLBF,0 );/* line buffered stdout */ while ((ch =getopt_long (argc, argv, -"0b:c:de:fg:i:j:kl:m:no:p:qr:s:t:uw:xyABD:EFJKHzCILN:OP:RS:UWXZ" , +"0b:c:de:fg:hi:j:kl:m:no:p:qr:s:t:uw:xyABD:EFJKHzCILN:OP:RS:UWXZ" , longopts,NULL )) !=EOF )switch (ch) {case 'b' :@@ -2916,6 +3007,9 @@ main(int argc, char **argv)case 'g' : filldata = *optarg;break ; +case 'h' : + hugepages =1 ; +break ;case 'i' : integrity =1 ; logdev =strdup (optarg);@@ -3229,15 +3323,7 @@ main(int argc, char **argv)exit (95 ); } } - original_buf = (char *)malloc (maxfilelen); -for (i =0 ; i < maxfilelen; i++) - original_buf[i] =random () %256 ; - good_buf = (char *)malloc (maxfilelen + writebdy); - good_buf =round_ptr_up (good_buf, writebdy,0 ); -memset (good_buf,'\0' , maxfilelen); - temp_buf = (char *)malloc (maxoplen + readbdy); - temp_buf =round_ptr_up (temp_buf, readbdy,0 ); -memset (temp_buf,'\0' , maxoplen); +init_buffers ();if (lite) {/* zero entire existing file */ ssize_t written;Signature · 1 line
2.47.1
Re: [PATCH v2 1/2] fsx: support reads/writes from buffers backed by hugepages
In reply to Joanne KoongQuoted 13 lines · Joanne Koong
Add support for reads/writes from buffers backed by hugepages. This can be enabled through the '-h' flag. This flag should only be used on systems where THP capabilities are enabled.
This is motivated by a recent bug that was due to faulty handling of userspace buffers backed by hugepages. This patch is a mitigation against problems like this in the future.
Signed-off-by Joanne Koong <[email protected]>--- ltp/fsx.c | 108 ++++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 97 insertions(+), 11 deletions(-)
[snip]
ltp/fsx.c keep_running(void) · line 2903
Quoted 12 lines
ltp/fsx.c +12
-0+static void * +init_hugepages_buf (unsigned len,int hugepage_size,int alignment) +{ +void *buf; +long buf_size =roundup (len, hugepage_size) + alignment; + +if (posix_memalign (&buf, hugepage_size, buf_size)) { +prterr ("posix_memalign for buf" ); +return NULL ; + } +memset (buf,'\0' , buf_size); +if (madvise (buf, buf_size,MADV_COLLAPSE )) {
Hi Joanne,
Sorry I have to drop this patchset from the "upcoming" release v2025.01.12. Due to it cause a regression build error on older system, e.g. RHEL-9:
[CC] fsx
fsx.c: In function 'init_hugepages_buf':
fsx.c:2935:36: error: 'MADV_COLLAPSE' undeclared (first use in this function); did you mean 'MADV_COLD'?
2935 | if (madvise(buf, buf_size, MADV_COLLAPSE)) {
| ^~~~~~~~~~~~~
| MADV_COLD fsx.c:2935:36: note: each undeclared identifier is reported only once for each function it appears in gmake[4]: *** [Makefile:51: fsx] Error 1
gmake[4]: *** Waiting for unfinished jobs....
gmake[3]: *** [include/buildrules:30: ltp] Error 2
It might cause xfstests totally can't be used on downstream systems, so it can't catch up the release of this weekend. Sorry about that, let's try to have it in next release :)
Thanks,
Zorro
Quoted 89 lines
ltp/fsx.c +51
-10+prterr ("madvise collapse for buf" ); +free (buf); +return NULL ; + } + +return buf; +} + +static void +init_buffers (void ) +{ +int i; + + original_buf = (char *)malloc (maxfilelen); +for (i =0 ; i < maxfilelen; i++) + original_buf[i] =random () %256 ; +if (hugepages) { +long hugepage_size =get_hugepage_size (); +if (hugepage_size ==-1 ) { +prterr ("get_hugepage_size()" ); +exit (100 ); + } + good_buf =init_hugepages_buf (maxfilelen, hugepage_size, writebdy); +if (!good_buf) { +prterr ("init_hugepages_buf failed for good_buf" ); +exit (101 ); + } + + temp_buf =init_hugepages_buf (maxoplen, hugepage_size, readbdy); +if (!temp_buf) { +prterr ("init_hugepages_buf failed for temp_buf" ); +exit (101 ); + } + }else { +unsigned long good_buf_len = maxfilelen + writebdy; +unsigned long temp_buf_len = maxoplen + readbdy; + + good_buf = (char *)malloc (good_buf_len); +memset (good_buf,'\0' , good_buf_len); + temp_buf = (char *)malloc (temp_buf_len); +memset (temp_buf,'\0' , temp_buf_len); + } + good_buf =round_ptr_up (good_buf, writebdy,0 ); + temp_buf =round_ptr_up (temp_buf, readbdy,0 ); +} +static struct option longopts[] = { {"replay-ops" , required_argument,0 ,256 }, {"record-ops" , optional_argument,0 ,255 },@@ -2883,7 +2974,7 @@ main(int argc, char **argv)setvbuf (stdout, (char *)0 , _IOLBF,0 );/* line buffered stdout */ while ((ch =getopt_long (argc, argv, -"0b:c:de:fg:i:j:kl:m:no:p:qr:s:t:uw:xyABD:EFJKHzCILN:OP:RS:UWXZ" , +"0b:c:de:fg:hi:j:kl:m:no:p:qr:s:t:uw:xyABD:EFJKHzCILN:OP:RS:UWXZ" , longopts,NULL )) !=EOF )switch (ch) {case 'b' :@@ -2916,6 +3007,9 @@ main(int argc, char **argv)case 'g' : filldata = *optarg;break ; +case 'h' : + hugepages =1 ; +break ;case 'i' : integrity =1 ; logdev =strdup (optarg);@@ -3229,15 +3323,7 @@ main(int argc, char **argv)exit (95 ); } } - original_buf = (char *)malloc (maxfilelen); -for (i =0 ; i < maxfilelen; i++) - original_buf[i] =random () %256 ; - good_buf = (char *)malloc (maxfilelen + writebdy); - good_buf =round_ptr_up (good_buf, writebdy,0 ); -memset (good_buf,'\0' , maxfilelen); - temp_buf = (char *)malloc (maxoplen + readbdy); - temp_buf =round_ptr_up (temp_buf, readbdy,0 ); -memset (temp_buf,'\0' , maxoplen); +init_buffers ();if (lite) {/* zero entire existing file */ ssize_t written;Signature · 1 line
2.47.1
Re: [PATCH v2 1/2] fsx: support reads/writes from buffers backed by hugepages
Quoted 42 lines · Zorro Lang
Quoted 13 lines · Joanne Koong
Add support for reads/writes from buffers backed by hugepages. This can be enabled through the '-h' flag. This flag should only be used on systems where THP capabilities are enabled.
This is motivated by a recent bug that was due to faulty handling of userspace buffers backed by hugepages. This patch is a mitigation against problems like this in the future.
Signed-off-by Joanne Koong <[email protected]>--- ltp/fsx.c | 108 ++++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 97 insertions(+), 11 deletions(-)[snip]
ltp/fsx.c keep_running(void) · line 2903
Quoted 12 lines
ltp/fsx.c +12
-0+static void * +init_hugepages_buf (unsigned len,int hugepage_size,int alignment) +{ +void *buf; +long buf_size =roundup (len, hugepage_size) + alignment; + +if (posix_memalign (&buf, hugepage_size, buf_size)) { +prterr ("posix_memalign for buf" ); +return NULL ; + } +memset (buf,'\0' , buf_size); +if (madvise (buf, buf_size,MADV_COLLAPSE )) {Hi Joanne,
Sorry I have to drop this patchset from the "upcoming" release v2025.01.12. Due to it cause a regression build error on older system, e.g. RHEL-9:
[CC] fsx
fsx.c: In function 'init_hugepages_buf':
fsx.c:2935:36: error: 'MADV_COLLAPSE' undeclared (first use in this function); did you mean 'MADV_COLD'?2935 | if (madvise(buf, buf_size, MADV_COLLAPSE)) { | ^~~~~~~~~~~~~ | MADV_COLD
Quoted 8 lines
fsx.c:2935:36: note: each undeclared identifier is reported only once for each function it appears in gmake[4]: *** [Makefile:51: fsx] Error 1
gmake[4]: *** Waiting for unfinished jobs....
gmake[3]: *** [include/buildrules:30: ltp] Error 2It might cause xfstests totally can't be used on downstream systems, so it can't catch up the release of this weekend. Sorry about that, let's try to have it in next release :)
Hi Zorro,
Thanks for the update. I'll submit a v3 of this patch that gates this function behind #ifdef MADV_COLLAPSE, and hopefully that should fix this issue.
Thanks,
Joanne
Quoted 95 lines
Thanks,
ZorroQuoted 89 lines
ltp/fsx.c +51
-11+prterr ("madvise collapse for buf" ); +free (buf); +return NULL ; + } + +return buf; +} + +static void +init_buffers (void ) +{ +int i; + + original_buf = (char *)malloc (maxfilelen); +for (i =0 ; i < maxfilelen; i++) + original_buf[i] =random () %256 ; +if (hugepages) { +long hugepage_size =get_hugepage_size (); +if (hugepage_size ==-1 ) { +prterr ("get_hugepage_size()" ); +exit (100 ); + } + good_buf =init_hugepages_buf (maxfilelen, hugepage_size, writebdy); +if (!good_buf) { +prterr ("init_hugepages_buf failed for good_buf" ); +exit (101 ); + } + + temp_buf =init_hugepages_buf (maxoplen, hugepage_size, readbdy); +if (!temp_buf) { +prterr ("init_hugepages_buf failed for temp_buf" ); +exit (101 ); + } + }else { +unsigned long good_buf_len = maxfilelen + writebdy; +unsigned long temp_buf_len = maxoplen + readbdy; + + good_buf = (char *)malloc (good_buf_len); +memset (good_buf,'\0' , good_buf_len); + temp_buf = (char *)malloc (temp_buf_len); +memset (temp_buf,'\0' , temp_buf_len); + } + good_buf =round_ptr_up (good_buf, writebdy,0 ); + temp_buf =round_ptr_up (temp_buf, readbdy,0 ); +} +static struct option longopts[] = { {"replay-ops" , required_argument,0 ,256 }, {"record-ops" , optional_argument,0 ,255 },@@ -2883,7 +2974,7 @@ main(int argc, char **argv)setvbuf (stdout, (char *)0 , _IOLBF,0 );/* line buffered stdout */ while ((ch =getopt_long (argc, argv, -"0b:c:de:fg:i:j:kl:m:no:p:qr:s:t:uw:xyABD:EFJKHzCILN:OP:RS:UWXZ" , +"0b:c:de:fg:hi:j:kl:m:no:p:qr:s:t:uw:xyABD:EFJKHzCILN:OP:RS:UWXZ" , longopts,NULL )) !=EOF )switch (ch) {case 'b' :@@ -2916,6 +3007,9 @@ main(int argc, char **argv)case 'g' : filldata = *optarg;break ; +case 'h' : + hugepages =1 ; +break ;case 'i' : integrity =1 ; logdev =strdup (optarg);@@ -3229,15 +3323,7 @@ main(int argc, char **argv)exit (95 ); } } - original_buf = (char *)malloc (maxfilelen); -for (i =0 ; i < maxfilelen; i++) - original_buf[i] =random () %256 ; - good_buf = (char *)malloc (maxfilelen + writebdy); - good_buf =round_ptr_up (good_buf, writebdy,0 ); -memset (good_buf,'\0' , maxfilelen); - temp_buf = (char *)malloc (maxoplen + readbdy); - temp_buf =round_ptr_up (temp_buf, readbdy,0 ); -memset (temp_buf,'\0' , maxoplen); +init_buffers ();if (lite) {/* zero entire existing file */ ssize_t written; --2.47.1
PATCH v2 2/2 generic: add tests for read/writes from hugepages-backed buffers
In reply to Joanne KoongAdd generic tests 758 and 759 for testing reads/writes from buffers backed by hugepages.
--- common/rc | 13 +++++++++++++ tests/generic/758 | 22 ++++++++++++++++++++++ tests/generic/758.out | 4 ++++ tests/generic/759 | 26 ++++++++++++++++++++++++++ tests/generic/759.out | 4 ++++ 5 files changed, 69 insertions(+) create mode 100755 tests/generic/758 create mode 100644 tests/generic/758.out create mode 100755 tests/generic/759 create mode 100644 tests/generic/759.out
common/rc +13 -0
@@ -3016,6 +3016,19 @@ _require_xfs_io_command() fi } +# check that the system supports transparent hugepages +_require_thp() +{ + if [ ! -e /sys/kernel/mm/transparent_hugepage/enabled ]; then + _notrun "system doesn't support transparent hugepages" + fi + + thp_status=$(cat /sys/kernel/mm/transparent_hugepage/enabled) + if [[ $thp_status == *"[never]"* ]]; then + _notrun "system doesn't have transparent hugepages enabled" + fi +} + # check that kernel and filesystem support direct I/O, and check if "$1" size # aligned (optional) is supported _require_odirect()
tests/generic/758 new file +22 -0
@@ -0,0 +1,22 @@ +#! /bin/bash +# SPDX-License-Identifier: GPL-2.0 +# +# FS QA Test No. 758 +# +# fsx exercising reads/writes from userspace buffers +# backed by hugepages +# +. ./common/preamble +_begin_fstest rw auto quick + +. ./common/filter + +_require_test +_require_thp + +run_fsx -N 10000 -l 500000 -h +run_fsx -N 10000 -o 8192 -l 500000 -h +run_fsx -N 10000 -o 128000 -l 500000 -h + +status=0 +exit
tests/generic/758.out new file +4 -0
@@ -0,0 +1,4 @@ +QA output created by 758 +fsx -N 10000 -l 500000 -h +fsx -N 10000 -o 8192 -l 500000 -h +fsx -N 10000 -o 128000 -l 500000 -h
tests/generic/759 new file +26 -0
@@ -0,0 +1,26 @@ +#! /bin/bash +# SPDX-License-Identifier: GPL-2.0 +# +# FS QA Test No. 759 +# +# fsx exercising direct IO reads/writes from userspace buffers +# backed by hugepages +# +. ./common/preamble +_begin_fstest rw auto quick + +. ./common/filter + +_require_test +_require_odirect +_require_thp + +psize=`$here/src/feature -s` +bsize=`$here/src/min_dio_alignment $TEST_DIR $TEST_DEV` + +run_fsx -N 10000 -l 500000 -r PSIZE -t BSIZE -w BSIZE -Z -R -W -h +run_fsx -N 10000 -o 8192 -l 500000 -r PSIZE -t BSIZE -w BSIZE -Z -R -W -h +run_fsx -N 10000 -o 128000 -l 500000 -r PSIZE -t BSIZE -w BSIZE -Z -R -W -h + +status=0 +exit
tests/generic/759.out new file +4 -0
@@ -0,0 +1,4 @@ +QA output created by 759 +fsx -N 10000 -l 500000 -r PSIZE -t BSIZE -w BSIZE -Z -R -W -h +fsx -N 10000 -o 8192 -l 500000 -r PSIZE -t BSIZE -w BSIZE -Z -R -W -h +fsx -N 10000 -o 128000 -l 500000 -r PSIZE -t BSIZE -w BSIZE -Z -R -W -h
Signature · 1 line
2.47.1
Re: [PATCH v2 2/2] generic: add tests for read/writes from hugepages-backed buffers
Quoted 5 lines · Joanne Koong
Add generic tests 758 and 759 for testing reads/writes from buffers backed by hugepages.
Signed-off-by Joanne Koong <[email protected]>---
Quoted 118 lines
common/rc | 13 +++++++++++++ tests/generic/758 | 22 ++++++++++++++++++++++ tests/generic/758.out | 4 ++++ tests/generic/759 | 26 ++++++++++++++++++++++++++ tests/generic/759.out | 4 ++++ 5 files changed, 69 insertions(+) create mode 100755 tests/generic/758 create mode 100644 tests/generic/758.out create mode 100755 tests/generic/759 create mode 100644 tests/generic/759.outcommon/rc +13
-0@@ -3016,6 +3016,19 @@ _require_xfs_io_command() fi } +# check that the system supports transparent hugepages +_require_thp() +{ + if [ ! -e /sys/kernel/mm/transparent_hugepage/enabled ]; then + _notrun "system doesn't support transparent hugepages" + fi + + thp_status=$(cat /sys/kernel/mm/transparent_hugepage/enabled) + if [[ $thp_status == *"[never]"* ]]; then + _notrun "system doesn't have transparent hugepages enabled" + fi +} + # check that kernel and filesystem support direct I/O, and check if "$1" size # aligned (optional) is supported _require_odirect()tests/generic/758 new file +22
-0@@ -0,0 +1,22 @@ +#! /bin/bash +# SPDX-License-Identifier: GPL-2.0 +# +# FS QA Test No. 758 +# +# fsx exercising reads/writes from userspace buffers +# backed by hugepages +# +. ./common/preamble +_begin_fstest rw auto quick + +. ./common/filter + +_require_test +_require_thp + +run_fsx -N 10000 -l 500000 -h +run_fsx -N 10000 -o 8192 -l 500000 -h +run_fsx -N 10000 -o 128000 -l 500000 -h + +status=0 +exittests/generic/758.out new file +4
-0@@ -0,0 +1,4 @@ +QA output created by 758 +fsx -N 10000 -l 500000 -h +fsx -N 10000 -o 8192 -l 500000 -h +fsx -N 10000 -o 128000 -l 500000 -htests/generic/759 new file +26
-0@@ -0,0 +1,26 @@ +#! /bin/bash +# SPDX-License-Identifier: GPL-2.0 +# +# FS QA Test No. 759 +# +# fsx exercising direct IO reads/writes from userspace buffers +# backed by hugepages +# +. ./common/preamble +_begin_fstest rw auto quick + +. ./common/filter + +_require_test +_require_odirect +_require_thp + +psize=`$here/src/feature -s` +bsize=`$here/src/min_dio_alignment $TEST_DIR $TEST_DEV` + +run_fsx -N 10000 -l 500000 -r PSIZE -t BSIZE -w BSIZE -Z -R -W -h +run_fsx -N 10000 -o 8192 -l 500000 -r PSIZE -t BSIZE -w BSIZE -Z -R -W -h +run_fsx -N 10000 -o 128000 -l 500000 -r PSIZE -t BSIZE -w BSIZE -Z -R -W -h + +status=0 +exittests/generic/759.out new file +4
-0@@ -0,0 +1,4 @@ +QA output created by 759 +fsx -N 10000 -l 500000 -r PSIZE -t BSIZE -w BSIZE -Z -R -W -h +fsx -N 10000 -o 8192 -l 500000 -r PSIZE -t BSIZE -w BSIZE -Z -R -W -h +fsx -N 10000 -o 128000 -l 500000 -r PSIZE -t BSIZE -w BSIZE -Z -R -W -hSignature · 1 line
2.47.1