Skip to contentSkip to conversation

Add KFD available memory ioctl thunk support

RFC In review Daniel Phillips 2 participants 4 messages Mar 22, 2022 – Mar 23, 2022

lore
Share & tools
lore
#0 lore raw

RFC 1/1 Add KFD available memory ioctl thunk support

Hi all,

This patch adds thunk support for the new KFD memory availability ioctl.

I am posting this patch inline with Thunderbird just for now, to establish the principle that I can post patches/code old school style without mangling whitespace. Please bear with me while I get that sorted out. I will switch to posting with git am pretty soon.

 include/hsakmt.h          |   11 +++++++++++
 include/linux/kfd_ioctl.h |   18 ++++++++++++++++--
 src/memory.c              |   23 +++++++++++++++++++++++
 3 files changed, 50 insertions(+), 2 deletions(-)

Regards,
Daniel

include/hsakmt.h +11 -0
index ff2d023..abc617f 100644
@@ -374,6 +374,17 @@ hsaKmtFreeMemory(
     HSAuint64   SizeInBytes         //IN
     );
 
+/**
+  Inquires memory available for allocation as a memory buffer
+*/
+
+HSAKMT_STATUS
+HSAKMTAPI
+hsaKmtAvailableMemory(
+    HSAuint32 Node,
+    HSAuint64 *AvailableBytes
+    );
+
 /**
   Registers with KFD a memory buffer that may be accessed by the GPU
 */
include/linux/kfd_ioctl.h +13 -2
index 8a0ed49..abfa948 100644
@@ -34,9 +34,10 @@
  * - 1.6 - Query clear flags in SVM get_attr API
  * - 1.7 - Checkpoint Restore (CRIU) API
  * - 1.8 - CRIU - Support for SDMA transfers with GTT BOs
+ * - 1.9 - Add available_memory ioctl
  */
 #define KFD_IOCTL_MAJOR_VERSION 1
-#define KFD_IOCTL_MINOR_VERSION 8
+#define KFD_IOCTL_MINOR_VERSION 9
 
 /*
  * Debug revision change log
@@ -769,6 +770,16 @@ struct kfd_ioctl_free_memory_of_gpu_args {
 	__u64 handle;		/* to KFD */
 };
 
+/* Inquire available memory with kfd_ioctl_get_available_memory
+ *
+ * @available: memory available for alloc
+ */
+struct kfd_ioctl_get_available_memory_args {
+	__u64 available;	/* from KFD */
+	__u32 gpu_id;		/* to KFD */
+	__u32 pad;
+};
+
 /* Map memory to one or more GPUs
  *
  * @handle:                memory handle returned by alloc
@@ -1328,7 +1339,7 @@ struct kfd_ioctl_set_xnack_mode_args {
 		AMDKFD_IOWR(0x22, struct kfd_ioctl_criu_args)
 
 #define AMDKFD_COMMAND_START		0x01
-#define AMDKFD_COMMAND_END		0x23
+#define AMDKFD_COMMAND_END		0x24
 
 /* non-upstream ioctls */
 #define AMDKFD_IOC_IPC_IMPORT_HANDLE                                    \
src/memory.c +23 -0
index 6d2a4f4..b2cd759 100644
@@ -199,6 +199,29 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtFreeMemory(void *MemoryAddress,
 	return fmm_release(MemoryAddress);
 }
 
+HSAKMT_STATUS HSAKMTAPI hsaKmtAvailableMemory(HSAuint32 Node, HSAuint64 *AvailableBytes)
+{
+	struct kfd_ioctl_get_available_memory_args args = {};
+	HSAKMT_STATUS result;
+
+	CHECK_KFD_OPEN();
+	CHECK_KFD_MINOR_VERSION(7);
+
+	pr_debug("[%s] node %d\n", __func__, Node);
+
+	result = validate_nodeid(Node, &args.gpu_id);
+	if (result != HSAKMT_STATUS_SUCCESS) {
+		pr_err("[%s] invalid node ID: %d\n", __func__, Node);
+		return result;
+	}
+
+	if (kmtIoctl(kfd_fd, AMDKFD_IOC_AVAILABLE_MEMORY, &args))
+		return HSAKMT_STATUS_ERROR;
+
+	*AvailableBytes = args.available;
+	return HSAKMT_STATUS_SUCCESS;
+}
+
 HSAKMT_STATUS HSAKMTAPI hsaKmtRegisterMemory(void *MemoryAddress,
 					     HSAuint64 MemorySizeInBytes)
 {
#1 lore raw
Quoted 98 lines

-----Original Message-----
From: amd-gfx <[email protected]> On Behalf Of Daniel Phillips
Sent: Tuesday, March 22, 2022 1:29 PM
To: [email protected]
Cc: Kuehling, Felix <[email protected]>
Subject: [RFC] Add KFD available memory ioctl thunk support

Hi all,

This patch adds thunk support for the new KFD memory availability ioctl.

I am posting this patch inline with Thunderbird just for now, to establish the principle that I can post patches/code old school style without mangling whitespace. Please bear with me while I get that sorted out. I will switch to posting with git am pretty soon.

 include/hsakmt.h          |   11 +++++++++++
 include/linux/kfd_ioctl.h |   18 ++++++++++++++++--
 src/memory.c              |   23 +++++++++++++++++++++++
 3 files changed, 50 insertions(+), 2 deletions(-)

Regards,
Daniel

include/hsakmt.h index ff2d023..abc617f renamed from include/hsakmt.h +0 -0

100644

include/hsakmt.h +10 -0
@@ -374,6 +374,17 @@ hsaKmtFreeMemory(
     HSAuint64   SizeInBytes         //IN
     );
 
+/**
+  Inquires memory available for allocation as a memory buffer */
+
+HSAKMT_STATUS
+HSAKMTAPI
+hsaKmtAvailableMemory(
+    HSAuint32 Node,
+    HSAuint64 *AvailableBytes
+    );
+
 /**
   Registers with KFD a memory buffer that may be accessed by the GPU  */
include/linux/kfd_ioctl.h index renamed from include/linux/kfd_ioctl.h +0 -0

8a0ed49..abfa948 100644

include/linux/kfd_ioctl.h +12 -2
@@ -34,9 +34,10 @@
  * - 1.6 - Query clear flags in SVM get_attr API
  * - 1.7 - Checkpoint Restore (CRIU) API
  * - 1.8 - CRIU - Support for SDMA transfers with GTT BOs
+ * - 1.9 - Add available_memory ioctl
  */
 #define KFD_IOCTL_MAJOR_VERSION 1
-#define KFD_IOCTL_MINOR_VERSION 8
+#define KFD_IOCTL_MINOR_VERSION 9
 
 /*
  * Debug revision change log
@@ -769,6 +770,16 @@ struct kfd_ioctl_free_memory_of_gpu_args {
 	__u64 handle;		/* to KFD */
 };
 
+/* Inquire available memory with kfd_ioctl_get_available_memory
+ *
+ * @available: memory available for alloc  */ struct
+kfd_ioctl_get_available_memory_args {
+	__u64 available;	/* from KFD */
+	__u32 gpu_id;		/* to KFD */
+	__u32 pad;
+};
+
 /* Map memory to one or more GPUs
  *
  * @handle:                memory handle returned by alloc
@@ -1328,7 +1339,7 @@ struct kfd_ioctl_set_xnack_mode_args {
 		AMDKFD_IOWR(0x22, struct kfd_ioctl_criu_args)
 
 #define AMDKFD_COMMAND_START		0x01
-#define AMDKFD_COMMAND_END		0x23
+#define AMDKFD_COMMAND_END		0x24
 
 /* non-upstream ioctls */
 #define AMDKFD_IOC_IPC_IMPORT_HANDLE                                    \
src/memory.c +1 -0
@@ -199,6 +199,29 @@ HSAKMT_STATUS HSAKMTAPI
 hsaKmtFreeMemory(void *MemoryAddress,
 	return fmm_release(MemoryAddress);
 }
 
+HSAKMT_STATUS HSAKMTAPI hsaKmtAvailableMemory(HSAuint32 Node,

HSAuint64

src/memory.c HSAKMT_STATUS HSAKMTAPI hsaKmtFreeMemory(void *MemoryAddress, · line 208

Quoted 6 lines
src/memory.c +6 -0
+*AvailableBytes) {
+	struct kfd_ioctl_get_available_memory_args args = {};
+	HSAKMT_STATUS result;
+
+	CHECK_KFD_OPEN();
+	CHECK_KFD_MINOR_VERSION(7);

I think this should check for minor version 9 to match #define KFD_IOCTL_MINOR_VERSION 9 in kfd_ioctl.h

Regards,
David

Quoted 20 lines
src/memory.c +16 -0
+
+	pr_debug("[%s] node %d\n", __func__, Node);
+
+	result = validate_nodeid(Node, &args.gpu_id);
+	if (result != HSAKMT_STATUS_SUCCESS) {
+		pr_err("[%s] invalid node ID: %d\n", __func__, Node);
+		return result;
+	}
+
+	if (kmtIoctl(kfd_fd, AMDKFD_IOC_AVAILABLE_MEMORY, &args))
+		return HSAKMT_STATUS_ERROR;
+
+	*AvailableBytes = args.available;
+	return HSAKMT_STATUS_SUCCESS;
+}
+
 HSAKMT_STATUS HSAKMTAPI hsaKmtRegisterMemory(void

*MemoryAddress,
HSAuint64 MemorySizeInBytes)
{

#2 lore raw
Quoted 3 lines · you

src/memory.c HSAKMT_STATUS HSAKMTAPI hsaKmtFreeMemory(void *MemoryAddress, · line 208

Quoted 1 line
src/memory.c +1 -0
+	CHECK_KFD_MINOR_VERSION(7);

I think this should check for minor version 9 to match #define KFD_IOCTL_MINOR_VERSION 9 in kfd_ioctl.h

Right you are.

Regards,
Daniel

#3 lore raw

Re: [RFC] Add KFD available memory ioctl thunk support (rev 1)

In reply to Yat Sin, David
Quoted 1 line · David

I think this should check for minor version 9 to match #define KFD_IOCTL_MINOR_VERSION 9 in kfd_ioctl.h

Fixed.

include/hsakmt.h          |   11 +++++++++++
include/linux/kfd_ioctl.h |   18 ++++++++++++++++--
src/memory.c              |   23 +++++++++++++++++++++++
3 files changed, 50 insertions(+), 2 deletions(-)
include/hsakmt.h +11 -0
index ff2d023..abc617f 100644
@@ -374,6 +374,17 @@ hsaKmtFreeMemory(
     HSAuint64   SizeInBytes         //IN
     );
 
+/**
+  Inquires memory available for allocation as a memory buffer
+*/
+
+HSAKMT_STATUS
+HSAKMTAPI
+hsaKmtAvailableMemory(
+    HSAuint32 Node,
+    HSAuint64 *AvailableBytes
+    );
+
 /**
   Registers with KFD a memory buffer that may be accessed by the GPU
 */
include/linux/kfd_ioctl.h +16 -2
index 8a0ed49..96f432c 100644
@@ -34,9 +34,10 @@
  * - 1.6 - Query clear flags in SVM get_attr API
  * - 1.7 - Checkpoint Restore (CRIU) API
  * - 1.8 - CRIU - Support for SDMA transfers with GTT BOs
+ * - 1.9 - Add available_memory ioctl
  */
 #define KFD_IOCTL_MAJOR_VERSION 1
-#define KFD_IOCTL_MINOR_VERSION 8
+#define KFD_IOCTL_MINOR_VERSION 9
 
 /*
  * Debug revision change log
@@ -769,6 +770,16 @@ struct kfd_ioctl_free_memory_of_gpu_args {
 	__u64 handle;		/* to KFD */
 };
 
+/* Inquire available memory with kfd_ioctl_get_available_memory
+ *
+ * @available: memory available for alloc
+ */
+struct kfd_ioctl_get_available_memory_args {
+	__u64 available;	/* from KFD */
+	__u32 gpu_id;		/* to KFD */
+	__u32 pad;
+};
+
 /* Map memory to one or more GPUs
  *
  * @handle:                memory handle returned by alloc
@@ -1327,8 +1338,11 @@ struct kfd_ioctl_set_xnack_mode_args {
 #define AMDKFD_IOC_CRIU_OP			\
 		AMDKFD_IOWR(0x22, struct kfd_ioctl_criu_args)
 
+#define AMDKFD_IOC_AVAILABLE_MEMORY            \
+		AMDKFD_IOR(0x23, struct kfd_ioctl_get_available_memory_args)
+
 #define AMDKFD_COMMAND_START		0x01
-#define AMDKFD_COMMAND_END		0x23
+#define AMDKFD_COMMAND_END		0x24
 
 /* non-upstream ioctls */
 #define AMDKFD_IOC_IPC_IMPORT_HANDLE                                    \
src/memory.c +23 -0
index 6d2a4f4..8f99589 100644
@@ -199,6 +199,29 @@ HSAKMT_STATUS HSAKMTAPI hsaKmtFreeMemory(void *MemoryAddress,
 	return fmm_release(MemoryAddress);
 }
 
+HSAKMT_STATUS HSAKMTAPI hsaKmtAvailableMemory(HSAuint32 Node, HSAuint64 *AvailableBytes)
+{
+	struct kfd_ioctl_get_available_memory_args args = {};
+	HSAKMT_STATUS result;
+
+	CHECK_KFD_OPEN();
+	CHECK_KFD_MINOR_VERSION(9);
+
+	pr_debug("[%s] node %d\n", __func__, Node);
+
+	result = validate_nodeid(Node, &args.gpu_id);
+	if (result != HSAKMT_STATUS_SUCCESS) {
+		pr_err("[%s] invalid node ID: %d\n", __func__, Node);
+		return result;
+	}
+
+	if (kmtIoctl(kfd_fd, AMDKFD_IOC_AVAILABLE_MEMORY, &args))
+		return HSAKMT_STATUS_ERROR;
+
+	*AvailableBytes = args.available;
+	return HSAKMT_STATUS_SUCCESS;
+}
+
 HSAKMT_STATUS HSAKMTAPI hsaKmtRegisterMemory(void *MemoryAddress,
 					     HSAuint64 MemorySizeInBytes)
 {