Recent changes (master)

index ece92563..162115d0 100644
@@ -2366,6 +2366,17 @@ I/O engine
 			:option:`iomem` must not be `cudamalloc`. This ioengine defines
 			engine specific options.
 
+		**libhipfile**
+			I/O engine supporting synchronous access to a GPUDirect Storage-supported
+			filesystem from AMD GPUs via the ROCm hipFile API. This engine performs
+			I/O without transferring buffers between user-space and the kernel,
+			unless :option:`verify` is set or :option:`rocm_io` is `posix`.
+			:option:`iomem` must not be `cudamalloc`. This ioengine defines
+			engine specific options. See `ROCm hipFile API`_ for information on
+			ROCm hipFile and how to install the ROCm stack.
+
+			.. _ROCm hipFile API: https://rocm.docs.amd.com/projects/hip/en/latest/reference/hip_runtime_api/modules/file_apis.html
+
 		**dfs**
 			I/O engine supporting asynchronous read and write operations to the
 			DAOS File System (DFS) via libdfs.

@@ -3264,6 +3275,28 @@ with the caveat that when used on the command line, they must come after the
 		GPU to RAM before a write and copied from RAM to GPU after a
 		read. :option:`verify` does not affect use of cudaMemcpy.
 
+.. option:: gpu_dev_ids=str : [libhipfile]
+
+	Specify the GPU IDs to use with ROCm. This is a colon-separated list of
+	int. GPUs are assigned to workers roundrobin. Default is 0.
+
+.. option:: rocm_io=str : [libhipfile]
+
+	Specify the type of I/O to use with ROCm. Default is **hipfile**.
+
+	**hipfile**
+		Use the ROCm hipFile API. This option performs I/O directly
+		between a GPUDirect Storage filesystem and GPU buffers,
+		avoiding use of a bounce buffer. If :option:`verify` is set,
+		hipMemcpy is used to copy verification data between RAM and GPU.
+		Verification data is copied from RAM to GPU before a write
+		and from GPU to RAM after a read. :option:`direct` must be 1.
+	**posix**
+		Use POSIX to perform I/O with a RAM buffer, and use hipMemcpy
+		to transfer data between RAM and the GPUs. Data is copied from
+		GPU to RAM before a write and copied from RAM to GPU after a
+		read. :option:`verify` does not affect use of hipMemcpy.
+
 .. option:: nfs_url=str : [nfs]
 
 	URL in libnfs format, eg nfs://<server|ipv4|ipv6>/path[?arg=val[&arg=val]*]

index 4f87adba..20db1a7d 100644
@@ -120,6 +120,14 @@ ifdef CONFIG_LINUX_EXT4_MOVE_EXTENT
 endif
 ifdef CONFIG_LIBCUFILE
   SOURCE += engines/libcufile.c
+  SHARED_GPUACCEL_SOURCE = 1
+endif
+ifdef CONFIG_LIBHIPFILE
+  SOURCE += engines/libhipfile.c
+  SHARED_GPUACCEL_SOURCE = 1
+endif
+ifdef SHARED_GPUACCEL_SOURCE
+  SOURCE += engines/gpuaccel.c
 endif
 ifdef CONFIG_LINUX_SPLICE
   SOURCE += engines/splice.c

index cdfefa59..804db167 100755
@@ -182,6 +182,7 @@ pmem="no"
 cuda="no"
 cuda13="no"
 libcufile="no"
+libhipfile="no"
 disable_lex=""
 disable_pmem="no"
 disable_native="no"

@@ -255,6 +256,8 @@ for opt do
   ;;
   --enable-libcufile) libcufile="yes"
   ;;
+  --enable-libhipfile) libhipfile="yes"
+  ;;
   --disable-native) disable_native="yes"
   ;;
   --with-ime=*) ime_path="$optarg"

@@ -326,6 +329,7 @@ if test "$show_help" = "yes" ; then
   echo "--disable-optimizations Don't enable compiler optimizations"
   echo "--enable-cuda           Enable GPUDirect RDMA support"
   echo "--enable-libcufile      Enable GPUDirect Storage cuFile support"
+  echo "--enable-libhipfile     Enable ROCm hipFile support"
   echo "--disable-native        Don't build for native host"
   echo "--with-ime=             Install path for DDN's Infinite Memory Engine"
   echo "--enable-libiscsi       Enable iscsi support"

@@ -2872,6 +2876,37 @@ EOF
 fi
 print_config "libcufile" "$libcufile"
 
+
+##########################################
+# libhipfile probe
+if test "$libhipfile" != "no" ; then
+cat > $TMPC << EOF
+#include <hipfile.h>
+#include <hip/hip_runtime.h>
+
+int main(int argc, char* argv[]) {
+  void *buf = NULL;
+  hipMalloc(&buf, 0);
+  hipFileBufRegister(NULL, 0, 0);
+  return 0;
+}
+EOF
+  ROCM_PATH="${ROCM_PATH:-/opt/rocm}"
+  HIPFILE_CFLAGS="-D__HIP_PLATFORM_AMD__ -I${ROCM_PATH}/include"
+  HIPFILE_LIBS="-L${ROCM_PATH}/lib -Wl,-rpath,${ROCM_PATH}/lib -lamdhip64 -lhipfile"
+  if compile_prog "$HIPFILE_CFLAGS" "$HIPFILE_LIBS" "libhipfile"; then
+    libhipfile="yes"
+    CFLAGS="$HIPFILE_CFLAGS $CFLAGS"
+    LIBS="$HIPFILE_LIBS $LIBS"
+  else
+    if test "$libhipfile" = "yes" ; then
+      feature_not_found "libhipfile" ""
+    fi
+    libhipfile="no"
+  fi
+fi
+print_config "libhipfile" "$libhipfile"
+
 ##########################################
 # cuda 13 probe
 if test "$cuda" != "no" || test "$libcufile" != "no"; then

@@ -3390,6 +3425,9 @@ fi
 if test "$libcufile" = "yes" ; then
   output_sym "CONFIG_LIBCUFILE"
 fi
+if test "$libhipfile" = "yes" ; then
+  output_sym "CONFIG_LIBHIPFILE"
+fi
 if test "$cuda13" = "yes" ; then
   output_sym "CONFIG_CUDA13"
 fi

similarity index 100%
rename from example_latency_steadystate.fio
rename to examples/example_latency_steadystate.fio

44 lines not on this page · raw message

41 lines not on this page · raw message

index fa281caa..993762f2 100644
@@ -2163,6 +2163,15 @@ I/O without transferring buffers between user-space and the kernel,
 unless \fBverify\fR is set or \fBcuda_io\fR is \fBposix\fR. \fBiomem\fR must
 not be \fBcudamalloc\fR. This ioengine defines engine specific options.
 .TP
+.B libhipfile
+I/O engine supporting synchronous access to a GPUDirect Storage-supported
+filesystem from AMD GPUs via the ROCm hipFile API. This engine performs
+I/O without transferring buffers between user-space and the kernel,
+unless \fBverify\fR is set or \fBrocm_io\fR is \fBposix\fR. \fBiomem\fR must
+not be \fBcudamalloc\fR. This ioengine defines engine specific options.
+(See \fIhttps://rocm.docs.amd.com\fR for information on ROCm hipFile
+and how to install the ROCm stack.)
+.TP
 .B dfs
 I/O engine supporting asynchronous read and write operations to the DAOS File
 System (DFS) via libdfs.

@@ -2993,6 +3002,34 @@ the use of cudaMemcpy.
 .RE
 .RE
 .TP
+.BI (libhipfile)gpu_dev_ids\fR=\fPstr
+Specify the GPU IDs to use with ROCm. This is a colon-separated list of int.
+GPUs are assigned to workers roundrobin. Default is 0.
+.TP
+.BI (libhipfile)rocm_io\fR=\fPstr
+Specify the type of I/O to use with ROCm. This option
+takes the following values:
+.RS
+.RS
+.TP
+.B hipfile (default)
+Use the ROCm hipFile API. This option performs I/O directly
+between a GPUDirect Storage filesystem and GPU buffers,
+avoiding use of a bounce buffer. If \fBverify\fR is set,
+hipMemcpy is used to copy verification data between RAM and GPU(s).
+Verification data is copied from RAM to GPU before a write
+and from GPU to RAM after a read.
+\fBdirect\fR must be 1.
+.TP
+.BI posix
+Use POSIX to perform I/O with a RAM buffer, and use
+hipMemcpy to transfer data between RAM and the GPU(s).
+Data is copied from GPU to RAM before a write and copied
+from RAM to GPU after a read. \fBverify\fR does not affect
+the use of hipMemcpy.
+.RE
+.RE
+.TP
 .BI (dfs)pool
 Specify the label or UUID of the DAOS pool to connect to.
 .TP