Recent changes (master)
The following changes since commit fd80924b22fef6ce0d5580724d91490347445f90:
Fio 3.25 (2020-12-04 11:47:42 -0700)
are available in the Git repository at:
git://git.kernel.dk/fio.git master
for you to fetch changes up to a210654b03dbd04f17984d1cf791b1fd56862f1b:
Merge branch 'cufile' of https://github.com/SystemFabricWorks/fio (2020-12-05 14:45:16 -0700)
----------------------------------------------------------------
Brian T. Smith (1):
ioengine: Add libcufile I/O engine
Jens Axboe (1):
Merge branch 'cufile' of https://github.com/SystemFabricWorks/fio
HOWTO | 30 ++ Makefile | 3 + configure | 30 ++ engines/libcufile.c | 627 ++++++++++++++++++++++++++++++++++++++++++ examples/libcufile-cufile.fio | 42 +++ examples/libcufile-posix.fio | 41 +++ fio.1 | 38 ++- optgroup.c | 4 + optgroup.h | 2 + 9 files changed, 816 insertions(+), 1 deletion(-) create mode 100644 engines/libcufile.c create mode 100644 examples/libcufile-cufile.fio create mode 100644 examples/libcufile-posix.fio
---
Diff of recent changes:
HOWTO +30 -0
@@ -2048,6 +2048,14 @@ I/O engine **nbd** Read and write a Network Block Device (NBD). + **libcufile** + I/O engine supporting libcufile synchronous access to nvidia-fs and a + GPUDirect Storage-supported filesystem. This engine performs + I/O without transferring buffers between user-space and the kernel, + unless :option:`verify` is set or :option:`cuda_io` is `posix`. + :option:`iomem` must not be `cudamalloc`. This ioengine defines + engine specific options. + I/O engine specific parameters ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -2398,6 +2406,28 @@ with the caveat that when used on the command line, they must come after the nbd+unix:///?socket=/tmp/socket nbds://tlshost/exportname +.. option:: gpu_dev_ids=str : [libcufile] + + Specify the GPU IDs to use with CUDA. This is a colon-separated list of + int. GPUs are assigned to workers roundrobin. Default is 0. + +.. option:: cuda_io=str : [libcufile] + + Specify the type of I/O to use with CUDA. Default is **cufile**. + + **cufile** + Use libcufile and nvidia-fs. This option performs I/O directly + between a GPUDirect Storage filesystem and GPU buffers, + avoiding use of a bounce buffer. If :option:`verify` is set, + cudaMemcpy is used to copy verificaton data between RAM and GPU. + Verification data is copied from RAM to GPU before a write + and from GPU to RAM after a read. :option:`direct` must be 1. + **posix** + Use POSIX to perform I/O with a RAM buffer, and use cudaMemcpy + to transfer data between RAM and the GPUs. Data is copied from + GPU to RAM before a write and copied from RAM to GPU after a + read. :option:`verify` does not affect use of cudaMemcpy. + I/O depth ~~~~~~~~~
Makefile +3 -0
@@ -103,6 +103,9 @@ endif ifdef CONFIG_LINUX_EXT4_MOVE_EXTENT SOURCE += engines/e4defrag.c endif +ifdef CONFIG_LIBCUFILE + SOURCE += engines/libcufile.c +endif ifdef CONFIG_LINUX_SPLICE SOURCE += engines/splice.c endif
configure +30 -0
@@ -162,6 +162,7 @@ pmemblk="no" devdax="no" pmem="no" cuda="no" +libcufile="no" disable_lex="" disable_pmem="no" disable_native="no"
@@ -224,6 +225,8 @@ for opt do ;; --enable-cuda) cuda="yes" ;; + --enable-libcufile) libcufile="yes" + ;; --disable-native) disable_native="yes" ;; --with-ime=*) ime_path="$optarg"
@@ -272,6 +275,7 @@ if test "$show_help" = "yes" ; then echo "--disable-shm Disable SHM support" echo "--disable-optimizations Don't enable compiler optimizations" echo "--enable-cuda Enable GPUDirect RDMA support" + echo "--enable-libcufile Enable GPUDirect Storage cuFile support" echo "--disable-native Don't build for native host" echo "--with-ime= Install path for DDN's Infinite Memory Engine" echo "--enable-libiscsi Enable iscsi support"
@@ -2495,6 +2499,29 @@ EOF fi print_config "cuda" "$cuda" +########################################## +# libcufile probe +if test "$libcufile" != "no" ; then +cat > $TMPC << EOF +#include <cufile.h> + +int main(int argc, char* argv[]) { + cuFileDriverOpen(); + return 0; +} +EOF + if compile_prog "" "-lcuda -lcudart -lcufile" "libcufile"; then + libcufile="yes" + LIBS="-lcuda -lcudart -lcufile $LIBS" + else + if test "$libcufile" = "yes" ; then + feature_not_found "libcufile" "" + fi + libcufile="no" + fi +fi +print_config "libcufile" "$libcufile" + ########################################## # check for cc -march=native build_native="no"
@@ -2966,6 +2993,9 @@ fi if test "$cuda" = "yes" ; then output_sym "CONFIG_CUDA" fi +if test "$libcufile" = "yes" ; then + output_sym "CONFIG_LIBCUFILE" +fi if test "$march_set" = "no" && test "$build_native" = "yes" ; then output_sym "CONFIG_BUILD_NATIVE" fi
engines/libcufile.c new file +627 -0
627 lines not on this page · raw message
examples/libcufile-cufile.fio new file +42 -0
42 lines not on this page · raw message
examples/libcufile-posix.fio new file +41 -0
41 lines not on this page · raw message
fio.1 +37 -1
@@ -1826,6 +1826,13 @@ Read and write iscsi lun with libiscsi. .TP .B nbd Synchronous read and write a Network Block Device (NBD). +.TP +.B libcufile +I/O engine supporting libcufile synchronous access to nvidia-fs and a +GPUDirect Storage-supported filesystem. This engine performs +I/O without transferring buffers between user-space and the kernel, +unless \fBverify\fR is set or \fBcuda_io\fR is \fBposix\fR. \fBiomem\fR must +not be \fBcudamalloc\fR. This ioengine defines engine specific options. .SS "I/O engine specific parameters" In addition, there are some parameters which are only valid when a specific \fBioengine\fR is in use. These are used identically to normal parameters,
@@ -2139,7 +2146,36 @@ Example URIs: \fInbd+unix:///?socket=/tmp/socket\fR .TP \fInbds://tlshost/exportname\fR - +.RE +.RE +.TP +.BI (libcufile)gpu_dev_ids\fR=\fPstr +Specify the GPU IDs to use with CUDA. This is a colon-separated list of int. +GPUs are assigned to workers roundrobin. Default is 0. +.TP +.BI (libcufile)cuda_io\fR=\fPstr +Specify the type of I/O to use with CUDA. This option +takes the following values: +.RS +.RS +.TP +.B cufile (default) +Use libcufile and nvidia-fs. This option performs I/O directly +between a GPUDirect Storage filesystem and GPU buffers, +avoiding use of a bounce buffer. If \fBverify\fR is set, +cudaMemcpy is used to copy verification data between RAM and GPU(s). +Verification data is copied from RAM to GPU before a write +and from GPU to RAM after a read. +\fBdirect\fR must be 1. +.TP +.BI posix +Use POSIX to perform I/O with a RAM buffer, and use +cudaMemcpy to transfer data between RAM and the GPU(s). +Data is copied from GPU to RAM before a write and copied +from RAM to GPU after a read. \fBverify\fR does not affect +the use of cudaMemcpy. +.RE +.RE .SS "I/O depth" .TP .BI iodepth \fR=\fPint
optgroup.c +4 -0
@@ -173,6 +173,10 @@ static const struct opt_group fio_opt_cat_groups[] = { .name = "NBD I/O engine", /* NBD */ .mask = FIO_OPT_G_NBD, }, + { + .name = "libcufile I/O engine", /* libcufile */ + .mask = FIO_OPT_G_LIBCUFILE, + }, { .name = NULL, },
optgroup.h +2 -0
@@ -67,6 +67,7 @@ enum opt_category_group { __FIO_OPT_G_IOURING, __FIO_OPT_G_FILESTAT, __FIO_OPT_G_NR, + __FIO_OPT_G_LIBCUFILE,FIO_OPT_G_RATE = (1ULL << __FIO_OPT_G_RATE), FIO_OPT_G_ZONE = (1ULL << __FIO_OPT_G_ZONE),
@@ -108,6 +109,7 @@ enum opt_category_group {FIO_OPT_G_NBD = (1ULL << __FIO_OPT_G_NBD),FIO_OPT_G_IOURING = (1ULL << __FIO_OPT_G_IOURING),FIO_OPT_G_FILESTAT = (1ULL << __FIO_OPT_G_FILESTAT), + FIO_OPT_G_LIBCUFILE = (1ULL << __FIO_OPT_G_LIBCUFILE), };extern const struct opt_group *opt_group_from_mask (uint64_t *mask);