Skip to contentSkip to conversation

Add available memory ioctl for libhsakmt

PATCH In review Daniel Phillips 4 participants 6 messages Jan 10, 2022 – Jan 17, 2022

Subsystems (3)
lore
Share & tools
lore
#0 lore raw

PATCH 1/1 Add available memory ioctl for libhsakmt

From: Daniel Phillips <[email protected]>

Add an ioctl to inquire memory available for allocation by libhsakmt per node, allowing for space consumed by page translation tables.

---
 drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h      |  1 +
 .../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c    | 14 ++++++++++++++
 drivers/gpu/drm/amd/amdkfd/kfd_chardev.c        | 17 +++++++++++++++++
 include/uapi/linux/kfd_ioctl.h                  | 14 ++++++++++++--
 4 files changed, 44 insertions(+), 2 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h +1 -0
index fcbc8a9c9e06..64c6c36685d3 100644
@@ -266,6 +266,7 @@ int amdgpu_amdkfd_gpuvm_acquire_process_vm(struct amdgpu_device *adev,
 void amdgpu_amdkfd_gpuvm_release_process_vm(struct amdgpu_device *adev,
 					void *drm_priv);
 uint64_t amdgpu_amdkfd_gpuvm_get_process_page_dir(void *drm_priv);
+size_t amdgpu_amdkfd_get_available_memory(struct amdgpu_device *adev);
 int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
 		struct amdgpu_device *adev, uint64_t va, uint64_t size,
 		void *drm_priv, struct kgd_mem **mem,
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c +14 -0
index 86a1a6c109d9..b7490a659173 100644
@@ -190,6 +190,20 @@ static int amdgpu_amdkfd_reserve_mem_limit(struct amdgpu_device *adev,
 	return ret;
 }
 
+size_t amdgpu_amdkfd_get_available_memory(struct amdgpu_device *adev)
+{
+	uint64_t reserved_for_pt =
+		ESTIMATE_PT_SIZE(amdgpu_amdkfd_total_mem_size);
+	size_t available_memory;
+
+	spin_lock(&kfd_mem_limit.mem_limit_lock);
+	available_memory =
+		adev->gmc.real_vram_size -
+		adev->kfd.vram_used - reserved_for_pt;
+	spin_unlock(&kfd_mem_limit.mem_limit_lock);
+	return available_memory;
+}
+
 static void unreserve_mem_limit(struct amdgpu_device *adev,
 		uint64_t size, u32 alloc_flag)
 {
drivers/gpu/drm/amd/amdkfd/kfd_chardev.c +17 -0
index 4bfc0c8ab764..5c2f6d97ff1c 100644
@@ -486,6 +486,20 @@ static int kfd_ioctl_get_queue_wave_state(struct file *filep,
 	return r;
 }
 
+static int kfd_ioctl_get_available_memory(struct file *filep,
+			         struct kfd_process *p, void *data)
+{
+	struct kfd_ioctl_get_available_memory_args *args = data;
+	struct kfd_dev *dev;
+
+	dev = kfd_device_by_id(args->gpu_id);
+	if (!dev)
+		return -EINVAL;
+
+	args->available = amdgpu_amdkfd_get_available_memory(dev->adev);
+	return 0;
+}
+
 static int kfd_ioctl_set_memory_policy(struct file *filep,
 					struct kfd_process *p, void *data)
 {
@@ -1959,6 +1973,9 @@ static const struct amdkfd_ioctl_desc amdkfd_ioctls[] = {
 
 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_XNACK_MODE,
 			kfd_ioctl_set_xnack_mode, 0),
+
+	AMDKFD_IOCTL_DEF(AMDKFD_IOC_AVAILABLE_MEMORY,
+			kfd_ioctl_get_available_memory, 0),
 };
 
 #define AMDKFD_CORE_IOCTL_COUNT	ARRAY_SIZE(amdkfd_ioctls)
include/uapi/linux/kfd_ioctl.h +12 -2
index af96af174dc4..94a99add2432 100644
@@ -32,9 +32,10 @@
  * - 1.4 - Indicate new SRAM EDC bit in device properties
  * - 1.5 - Add SVM API
  * - 1.6 - Query clear flags in SVM get_attr API
+ * - 1.7 - Add available_memory ioctl
  */
 #define KFD_IOCTL_MAJOR_VERSION 1
-#define KFD_IOCTL_MINOR_VERSION 6
+#define KFD_IOCTL_MINOR_VERSION 7
 
 struct kfd_ioctl_get_version_args {
 	__u32 major_version;	/* from KFD */
@@ -98,6 +99,12 @@ struct kfd_ioctl_get_queue_wave_state_args {
 	__u32 pad;
 };
 
+struct kfd_ioctl_get_available_memory_args {
+	__u64 available;	/* from KFD */
+	__u32 gpu_id;		/* to KFD */
+	__u32 pad;
+};
+
 /* For kfd_ioctl_set_memory_policy_args.default_policy and alternate_policy */
 #define KFD_IOC_CACHE_POLICY_COHERENT 0
 #define KFD_IOC_CACHE_POLICY_NONCOHERENT 1
@@ -742,7 +749,10 @@ struct kfd_ioctl_set_xnack_mode_args {
 #define AMDKFD_IOC_SET_XNACK_MODE		\
 		AMDKFD_IOWR(0x21, struct kfd_ioctl_set_xnack_mode_args)
 
+#define AMDKFD_IOC_AVAILABLE_MEMORY		\
+		AMDKFD_IOR(0x22, struct kfd_ioctl_get_available_memory_args)
+
 #define AMDKFD_COMMAND_START		0x01
-#define AMDKFD_COMMAND_END		0x22
+#define AMDKFD_COMMAND_END		0x23
 
 #endif
Signature · 1 line
2.34.1
#1 lore raw

[Public]

This is missing your signed-off-by. Additionally, for UAPI changes, we need a link the patches for the userspace component that will make use of it.

Alex

________________________________
From: amd-gfx <[email protected]> on behalf of Daniel Phillips <[email protected]>
Sent: Monday, January 10, 2022 3:54 PM
To: [email protected] <[email protected]>; [email protected] <[email protected]>
Cc: Phillips, Daniel <[email protected]>
Subject: [PATCH 1/1] Add available memory ioctl for libhsakmt

From: Daniel Phillips <[email protected]>

Add an ioctl to inquire memory available for allocation by libhsakmt per node, allowing for space consumed by page translation tables.

---
 drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h      |  1 +
 .../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c    | 14 ++++++++++++++
 drivers/gpu/drm/amd/amdkfd/kfd_chardev.c        | 17 +++++++++++++++++
 include/uapi/linux/kfd_ioctl.h                  | 14 ++++++++++++--
 4 files changed, 44 insertions(+), 2 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h +1 -0
index fcbc8a9c9e06..64c6c36685d3 100644
@@ -266,6 +266,7 @@ int amdgpu_amdkfd_gpuvm_acquire_process_vm(struct amdgpu_device *adev,
 void amdgpu_amdkfd_gpuvm_release_process_vm(struct amdgpu_device *adev,
                                         void *drm_priv);
 uint64_t amdgpu_amdkfd_gpuvm_get_process_page_dir(void *drm_priv);
+size_t amdgpu_amdkfd_get_available_memory(struct amdgpu_device *adev);
 int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
                 struct amdgpu_device *adev, uint64_t va, uint64_t size,
                 void *drm_priv, struct kgd_mem **mem,
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c +14 -0
index 86a1a6c109d9..b7490a659173 100644
@@ -190,6 +190,20 @@ static int amdgpu_amdkfd_reserve_mem_limit(struct amdgpu_device *adev,
         return ret;
 }
 
+size_t amdgpu_amdkfd_get_available_memory(struct amdgpu_device *adev)
+{
+       uint64_t reserved_for_pt =
+               ESTIMATE_PT_SIZE(amdgpu_amdkfd_total_mem_size);
+       size_t available_memory;
+
+       spin_lock(&kfd_mem_limit.mem_limit_lock);
+       available_memory =
+               adev->gmc.real_vram_size -
+               adev->kfd.vram_used - reserved_for_pt;
+       spin_unlock(&kfd_mem_limit.mem_limit_lock);
+       return available_memory;
+}
+
 static void unreserve_mem_limit(struct amdgpu_device *adev,
                 uint64_t size, u32 alloc_flag)
 {
drivers/gpu/drm/amd/amdkfd/kfd_chardev.c +17 -0
index 4bfc0c8ab764..5c2f6d97ff1c 100644
@@ -486,6 +486,20 @@ static int kfd_ioctl_get_queue_wave_state(struct file *filep,
         return r;
 }
 
+static int kfd_ioctl_get_available_memory(struct file *filep,
+                                struct kfd_process *p, void *data)
+{
+       struct kfd_ioctl_get_available_memory_args *args = data;
+       struct kfd_dev *dev;
+
+       dev = kfd_device_by_id(args->gpu_id);
+       if (!dev)
+               return -EINVAL;
+
+       args->available = amdgpu_amdkfd_get_available_memory(dev->adev);
+       return 0;
+}
+
 static int kfd_ioctl_set_memory_policy(struct file *filep,
                                         struct kfd_process *p, void *data)
 {
@@ -1959,6 +1973,9 @@ static const struct amdkfd_ioctl_desc amdkfd_ioctls[] = {
 
         AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_XNACK_MODE,
                         kfd_ioctl_set_xnack_mode, 0),
+
+       AMDKFD_IOCTL_DEF(AMDKFD_IOC_AVAILABLE_MEMORY,
+                       kfd_ioctl_get_available_memory, 0),
 };
 
 #define AMDKFD_CORE_IOCTL_COUNT ARRAY_SIZE(amdkfd_ioctls)
include/uapi/linux/kfd_ioctl.h +12 -2
index af96af174dc4..94a99add2432 100644
@@ -32,9 +32,10 @@
  * - 1.4 - Indicate new SRAM EDC bit in device properties
  * - 1.5 - Add SVM API
  * - 1.6 - Query clear flags in SVM get_attr API
+ * - 1.7 - Add available_memory ioctl
  */
 #define KFD_IOCTL_MAJOR_VERSION 1
-#define KFD_IOCTL_MINOR_VERSION 6
+#define KFD_IOCTL_MINOR_VERSION 7
 
 struct kfd_ioctl_get_version_args {
         __u32 major_version;    /* from KFD */
@@ -98,6 +99,12 @@ struct kfd_ioctl_get_queue_wave_state_args {
         __u32 pad;
 };
 
+struct kfd_ioctl_get_available_memory_args {
+       __u64 available;        /* from KFD */
+       __u32 gpu_id;           /* to KFD */
+       __u32 pad;
+};
+
 /* For kfd_ioctl_set_memory_policy_args.default_policy and alternate_policy */
 #define KFD_IOC_CACHE_POLICY_COHERENT 0
 #define KFD_IOC_CACHE_POLICY_NONCOHERENT 1
@@ -742,7 +749,10 @@ struct kfd_ioctl_set_xnack_mode_args {
 #define AMDKFD_IOC_SET_XNACK_MODE               \
                 AMDKFD_IOWR(0x21, struct kfd_ioctl_set_xnack_mode_args)
 
+#define AMDKFD_IOC_AVAILABLE_MEMORY            \
+               AMDKFD_IOR(0x22, struct kfd_ioctl_get_available_memory_args)
+
 #define AMDKFD_COMMAND_START            0x01
-#define AMDKFD_COMMAND_END             0x22
+#define AMDKFD_COMMAND_END             0x23
 
 #endif

--
2.34.1

maintainer
#2 lore raw
Quoted 8 lines · Alexander

[Public]

[Public]

This is missing your signed-off-by.  Additionally, for UAPI changes, we need a link the patches for the userspace component that will make use of it.

I've asked Daniel to send the Thunk patch to the same mailing list. Should show up any minute ...

Regards,
  Felix

Quoted 141 lines

Alex

------------------------------------------------------------------------
*From:* amd-gfx <[email protected]> on behalf of Daniel Phillips <[email protected]> *Sent:* Monday, January 10, 2022 3:54 PM
*To:* [email protected] <[email protected]>; [email protected] <[email protected]> *Cc:* Phillips, Daniel <[email protected]>
*Subject:* [PATCH 1/1] Add available memory ioctl for libhsakmt
From: Daniel Phillips <[email protected]>

Add an ioctl to inquire memory available for allocation by libhsakmt per node, allowing for space consumed by page translation tables.
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h      |  1 + .../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c    | 14 ++++++++++++++ drivers/gpu/drm/amd/amdkfd/kfd_chardev.c        | 17 +++++++++++++++++ include/uapi/linux/kfd_ioctl.h                  | 14 ++++++++++++--

 4 files changed, 44 insertions(+), 2 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h +1 -0
index fcbc8a9c9e06..64c6c36685d3 100644
@@ -266,6 +266,7 @@ int amdgpu_amdkfd_gpuvm_acquire_process_vm(struct 
 amdgpu_device *adev,
  void amdgpu_amdkfd_gpuvm_release_process_vm(struct amdgpu_device *adev,
                                          void *drm_priv);
  uint64_t amdgpu_amdkfd_gpuvm_get_process_page_dir(void *drm_priv);
+size_t amdgpu_amdkfd_get_available_memory(struct amdgpu_device *adev);

 int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
                 struct amdgpu_device *adev, uint64_t va, uint64_t size, void *drm_priv, struct kgd_mem **mem,

drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c +14 -0
index 86a1a6c109d9..b7490a659173 100644
@@ -190,6 +190,20 @@ static int amdgpu_amdkfd_reserve_mem_limit(struct 
 amdgpu_device *adev,
          return ret;
  }
 
+size_t amdgpu_amdkfd_get_available_memory(struct amdgpu_device *adev)
+{
+       uint64_t reserved_for_pt =
+ ESTIMATE_PT_SIZE(amdgpu_amdkfd_total_mem_size);
+       size_t available_memory;
+
+       spin_lock(&kfd_mem_limit.mem_limit_lock);
+       available_memory =
+               adev->gmc.real_vram_size -
+               adev->kfd.vram_used - reserved_for_pt;
+ spin_unlock(&kfd_mem_limit.mem_limit_lock);
+       return available_memory;
+}
+
  static void unreserve_mem_limit(struct amdgpu_device *adev,
                  uint64_t size, u32 alloc_flag)

 {

drivers/gpu/drm/amd/amdkfd/kfd_chardev.c +14 -0
index 4bfc0c8ab764..5c2f6d97ff1c 100644
@@ -486,6 +486,20 @@ static int kfd_ioctl_get_queue_wave_state(struct 
 file *filep,
          return r;
  }
 
+static int kfd_ioctl_get_available_memory(struct file *filep,
+                                struct kfd_process *p, void *data)
+{
+       struct kfd_ioctl_get_available_memory_args *args = data;
+       struct kfd_dev *dev;
+
+       dev = kfd_device_by_id(args->gpu_id);
+       if (!dev)
+               return -EINVAL;
+
+       args->available = amdgpu_amdkfd_get_available_memory(dev->adev);
+       return 0;
+}
+
  static int kfd_ioctl_set_memory_policy(struct file *filep,
                                          struct kfd_process *p, void *data)

 {

drivers/gpu/drm/amd/amdkfd/kfd_chardev.c +3 -0
@@ -1959,6 +1973,9 @@ static const struct amdkfd_ioctl_desc 
 amdkfd_ioctls[] = {
 
          AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_XNACK_MODE,
                          kfd_ioctl_set_xnack_mode, 0),
+
+       AMDKFD_IOCTL_DEF(AMDKFD_IOC_AVAILABLE_MEMORY,
+ kfd_ioctl_get_available_memory, 0),
  };
 

 #define AMDKFD_CORE_IOCTL_COUNT ARRAY_SIZE(amdkfd_ioctls)

include/uapi/linux/kfd_ioctl.h +2 -1
index af96af174dc4..94a99add2432 100644
@@ -32,9 +32,10 @@
   * - 1.4 - Indicate new SRAM EDC bit in device properties
   * - 1.5 - Add SVM API
   * - 1.6 - Query clear flags in SVM get_attr API
+ * - 1.7 - Add available_memory ioctl
   */
  #define KFD_IOCTL_MAJOR_VERSION 1
-#define KFD_IOCTL_MINOR_VERSION 6
+#define KFD_IOCTL_MINOR_VERSION 7
 
  struct kfd_ioctl_get_version_args {

         __u32 major_version;    /* from KFD */

include/uapi/linux/kfd_ioctl.h +10 -1
@@ -98,6 +99,12 @@ struct kfd_ioctl_get_queue_wave_state_args {
          __u32 pad;
  };
 
+struct kfd_ioctl_get_available_memory_args {
+       __u64 available;        /* from KFD */
+       __u32 gpu_id;           /* to KFD */
+       __u32 pad;
+};
+
  /* For kfd_ioctl_set_memory_policy_args.default_policy and alternate_policy */
  #define KFD_IOC_CACHE_POLICY_COHERENT 0
  #define KFD_IOC_CACHE_POLICY_NONCOHERENT 1
@@ -742,7 +749,10 @@ struct kfd_ioctl_set_xnack_mode_args {
  #define AMDKFD_IOC_SET_XNACK_MODE               \
                  AMDKFD_IOWR(0x21, struct kfd_ioctl_set_xnack_mode_args)
 
+#define AMDKFD_IOC_AVAILABLE_MEMORY            \
+               AMDKFD_IOR(0x22, struct 
 kfd_ioctl_get_available_memory_args)
+
  #define AMDKFD_COMMAND_START            0x01
-#define AMDKFD_COMMAND_END             0x22
+#define AMDKFD_COMMAND_END             0x23
 

 #endif

Signature · 1 line
2.34.1
maintainer Reviewed-by on v1 1/1
#5 lore raw
In reply to Daniel Phillips
Quoted 1 line · Daniel Phillips

From: Daniel Phillips <[email protected]>

This is weird. Looks like you've set up the your user email in your .gitconfig incorrectly. Or you changed it after you commited this patch locally.

Quoted 3 lines

Add an ioctl to inquire memory available for allocation by libhsakmt per node, allowing for space consumed by page translation tables.

Other than the missing signed-off-by, this patch is

Quoted 120 lines
---
  drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h      |  1 +
  .../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c    | 14 ++++++++++++++
  drivers/gpu/drm/amd/amdkfd/kfd_chardev.c        | 17 +++++++++++++++++
  include/uapi/linux/kfd_ioctl.h                  | 14 ++++++++++++--
  4 files changed, 44 insertions(+), 2 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h +1 -0
index fcbc8a9c9e06..64c6c36685d3 100644
@@ -266,6 +266,7 @@ int amdgpu_amdkfd_gpuvm_acquire_process_vm(struct amdgpu_device *adev,
  void amdgpu_amdkfd_gpuvm_release_process_vm(struct amdgpu_device *adev,
  					void *drm_priv);
  uint64_t amdgpu_amdkfd_gpuvm_get_process_page_dir(void *drm_priv);
+size_t amdgpu_amdkfd_get_available_memory(struct amdgpu_device *adev);
  int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
  		struct amdgpu_device *adev, uint64_t va, uint64_t size,
  		void *drm_priv, struct kgd_mem **mem,
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c +14 -0
index 86a1a6c109d9..b7490a659173 100644
@@ -190,6 +190,20 @@ static int amdgpu_amdkfd_reserve_mem_limit(struct amdgpu_device *adev,
  	return ret;
  }
 
+size_t amdgpu_amdkfd_get_available_memory(struct amdgpu_device *adev)
+{
+	uint64_t reserved_for_pt =
+		ESTIMATE_PT_SIZE(amdgpu_amdkfd_total_mem_size);
+	size_t available_memory;
+
+	spin_lock(&kfd_mem_limit.mem_limit_lock);
+	available_memory =
+		adev->gmc.real_vram_size -
+		adev->kfd.vram_used - reserved_for_pt;
+	spin_unlock(&kfd_mem_limit.mem_limit_lock);
+	return available_memory;
+}
+
  static void unreserve_mem_limit(struct amdgpu_device *adev,
  		uint64_t size, u32 alloc_flag)
  {
drivers/gpu/drm/amd/amdkfd/kfd_chardev.c +17 -0
index 4bfc0c8ab764..5c2f6d97ff1c 100644
@@ -486,6 +486,20 @@ static int kfd_ioctl_get_queue_wave_state(struct file *filep,
  	return r;
  }
 
+static int kfd_ioctl_get_available_memory(struct file *filep,
+			         struct kfd_process *p, void *data)
+{
+	struct kfd_ioctl_get_available_memory_args *args = data;
+	struct kfd_dev *dev;
+
+	dev = kfd_device_by_id(args->gpu_id);
+	if (!dev)
+		return -EINVAL;
+
+	args->available = amdgpu_amdkfd_get_available_memory(dev->adev);
+	return 0;
+}
+
  static int kfd_ioctl_set_memory_policy(struct file *filep,
  					struct kfd_process *p, void *data)
  {
@@ -1959,6 +1973,9 @@ static const struct amdkfd_ioctl_desc amdkfd_ioctls[] = {
 
  	AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_XNACK_MODE,
  			kfd_ioctl_set_xnack_mode, 0),
+
+	AMDKFD_IOCTL_DEF(AMDKFD_IOC_AVAILABLE_MEMORY,
+			kfd_ioctl_get_available_memory, 0),
  };
 
  #define AMDKFD_CORE_IOCTL_COUNT	ARRAY_SIZE(amdkfd_ioctls)
include/uapi/linux/kfd_ioctl.h +12 -2
index af96af174dc4..94a99add2432 100644
@@ -32,9 +32,10 @@
   * - 1.4 - Indicate new SRAM EDC bit in device properties
   * - 1.5 - Add SVM API
   * - 1.6 - Query clear flags in SVM get_attr API
+ * - 1.7 - Add available_memory ioctl
   */
  #define KFD_IOCTL_MAJOR_VERSION 1
-#define KFD_IOCTL_MINOR_VERSION 6
+#define KFD_IOCTL_MINOR_VERSION 7
 
  struct kfd_ioctl_get_version_args {
  	__u32 major_version;	/* from KFD */
@@ -98,6 +99,12 @@ struct kfd_ioctl_get_queue_wave_state_args {
  	__u32 pad;
  };
 
+struct kfd_ioctl_get_available_memory_args {
+	__u64 available;	/* from KFD */
+	__u32 gpu_id;		/* to KFD */
+	__u32 pad;
+};
+
  /* For kfd_ioctl_set_memory_policy_args.default_policy and alternate_policy */
  #define KFD_IOC_CACHE_POLICY_COHERENT 0
  #define KFD_IOC_CACHE_POLICY_NONCOHERENT 1
@@ -742,7 +749,10 @@ struct kfd_ioctl_set_xnack_mode_args {
  #define AMDKFD_IOC_SET_XNACK_MODE		\
  		AMDKFD_IOWR(0x21, struct kfd_ioctl_set_xnack_mode_args)
 
+#define AMDKFD_IOC_AVAILABLE_MEMORY		\
+		AMDKFD_IOR(0x22, struct kfd_ioctl_get_available_memory_args)
+
  #define AMDKFD_COMMAND_START		0x01
-#define AMDKFD_COMMAND_END		0x22
+#define AMDKFD_COMMAND_END		0x23
 
  #endif
#3 lore raw
In reply to Deucher, Alexander

[Public]

From: amd-gfx <[email protected]> On Behalf Of Deucher, Alexander
Sent: Monday, January 10, 2022 4:11 PM
To: Phillips, Daniel <[email protected]>; [email protected]; [email protected]
Subject: Re: [PATCH 1/1] Add available memory ioctl for libhsakmt

[Public]

[Public]

This is missing your signed-off-by. Additionally, for UAPI changes, we need a link the patches for the userspace component that will make use of it.

Alex

________________________________
From: amd-gfx <[email protected]<mailto:[email protected]>> on behalf of Daniel Phillips <[email protected]<mailto:[email protected]>>
Sent: Monday, January 10, 2022 3:54 PM
To: [email protected]<mailto:[email protected]> <[email protected]<mailto:[email protected]>>; [email protected]<mailto:[email protected]> <[email protected]<mailto:[email protected]>>
Cc: Phillips, Daniel <[email protected]<mailto:[email protected]>>
Subject: [PATCH 1/1] Add available memory ioctl for libhsakmt

From: Daniel Phillips <[email protected]<mailto:[email protected]>>

Add an ioctl to inquire memory available for allocation by libhsakmt per node, allowing for space consumed by page translation tables.

---
 drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h      |  1 +
 .../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c    | 14 ++++++++++++++
 drivers/gpu/drm/amd/amdkfd/kfd_chardev.c        | 17 +++++++++++++++++
 include/uapi/linux/kfd_ioctl.h                  | 14 ++++++++++++--
 4 files changed, 44 insertions(+), 2 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h +1 -0
index fcbc8a9c9e06..64c6c36685d3 100644
@@ -266,6 +266,7 @@ int amdgpu_amdkfd_gpuvm_acquire_process_vm(struct amdgpu_device *adev,
 void amdgpu_amdkfd_gpuvm_release_process_vm(struct amdgpu_device *adev,
                                         void *drm_priv);
 uint64_t amdgpu_amdkfd_gpuvm_get_process_page_dir(void *drm_priv);
+size_t amdgpu_amdkfd_get_available_memory(struct amdgpu_device *adev);
 int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
                 struct amdgpu_device *adev, uint64_t va, uint64_t size,
                 void *drm_priv, struct kgd_mem **mem,
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c +14 -0
index 86a1a6c109d9..b7490a659173 100644
@@ -190,6 +190,20 @@ static int amdgpu_amdkfd_reserve_mem_limit(struct amdgpu_device *adev,
         return ret;
 }
 
+size_t amdgpu_amdkfd_get_available_memory(struct amdgpu_device *adev)
+{
+       uint64_t reserved_for_pt =
+               ESTIMATE_PT_SIZE(amdgpu_amdkfd_total_mem_size);
+       size_t available_memory;
+
+       spin_lock(&kfd_mem_limit.mem_limit_lock);
+       available_memory =
+               adev->gmc.real_vram_size -
+               adev->kfd.vram_used - reserved_for_pt;
+       spin_unlock(&kfd_mem_limit.mem_limit_lock);
+       return available_memory;
+}
+
 static void unreserve_mem_limit(struct amdgpu_device *adev,
                 uint64_t size, u32 alloc_flag)
 {
drivers/gpu/drm/amd/amdkfd/kfd_chardev.c +7 -0
index 4bfc0c8ab764..5c2f6d97ff1c 100644
@@ -486,6 +486,20 @@ static int kfd_ioctl_get_queue_wave_state(struct file *filep,
         return r;
 }
 
+static int kfd_ioctl_get_available_memory(struct file *filep,
+                                struct kfd_process *p, void *data)
+{
+       struct kfd_ioctl_get_available_memory_args *args = data;
+       struct kfd_dev *dev;
+
+       dev = kfd_device_by_id(args->gpu_id);

Once CRIU changes land, this need to change to kfd_process_device_data_by_id() and then use pdd->dev

+ if (!dev)
+ return -EINVAL;
+
+ args->available = amdgpu_amdkfd_get_available_memory(dev->adev);
+ return 0;
+}
+
static int kfd_ioctl_set_memory_policy(struct file *filep,
struct kfd_process *p, void *data)
{
@@ -1959,6 +1973,9 @@ static const struct amdkfd_ioctl_desc amdkfd_ioctls[] = {

         AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_XNACK_MODE,
                         kfd_ioctl_set_xnack_mode, 0),

+
+ AMDKFD_IOCTL_DEF(AMDKFD_IOC_AVAILABLE_MEMORY,
+ kfd_ioctl_get_available_memory, 0),
};

#define AMDKFD_CORE_IOCTL_COUNT ARRAY_SIZE(amdkfd_ioctls)

include/uapi/linux/kfd_ioctl.h +12 -2
index af96af174dc4..94a99add2432 100644
@@ -32,9 +32,10 @@
  * - 1.4 - Indicate new SRAM EDC bit in device properties
  * - 1.5 - Add SVM API
  * - 1.6 - Query clear flags in SVM get_attr API
+ * - 1.7 - Add available_memory ioctl
  */
 #define KFD_IOCTL_MAJOR_VERSION 1
-#define KFD_IOCTL_MINOR_VERSION 6
+#define KFD_IOCTL_MINOR_VERSION 7
 
 struct kfd_ioctl_get_version_args {
         __u32 major_version;    /* from KFD */
@@ -98,6 +99,12 @@ struct kfd_ioctl_get_queue_wave_state_args {
         __u32 pad;
 };
 
+struct kfd_ioctl_get_available_memory_args {
+       __u64 available;        /* from KFD */
+       __u32 gpu_id;           /* to KFD */
+       __u32 pad;
+};
+
 /* For kfd_ioctl_set_memory_policy_args.default_policy and alternate_policy */
 #define KFD_IOC_CACHE_POLICY_COHERENT 0
 #define KFD_IOC_CACHE_POLICY_NONCOHERENT 1
@@ -742,7 +749,10 @@ struct kfd_ioctl_set_xnack_mode_args {
 #define AMDKFD_IOC_SET_XNACK_MODE               \
                 AMDKFD_IOWR(0x21, struct kfd_ioctl_set_xnack_mode_args)
 
+#define AMDKFD_IOC_AVAILABLE_MEMORY            \
+               AMDKFD_IOR(0x22, struct kfd_ioctl_get_available_memory_args)
+
 #define AMDKFD_COMMAND_START            0x01
-#define AMDKFD_COMMAND_END             0x22
+#define AMDKFD_COMMAND_END             0x23
 
 #endif

--
2.34.1

maintainer
#4 lore raw

Am 2022-01-17 um 9:50 a.m. schrieb Bhardwaj, Rajneesh:

Quoted 124 lines

[Public]

*From:* amd-gfx <[email protected]> *On Behalf Of *Deucher, Alexander
*Sent:* Monday, January 10, 2022 4:11 PM
*To:* Phillips, Daniel <[email protected]>; [email protected]; [email protected] *Subject:* Re: [PATCH 1/1] Add available memory ioctl for libhsakmt

[Public]

[Public]

This is missing your signed-off-by.  Additionally, for UAPI changes, we need a link the patches for the userspace component that will make use of it.

Alex

------------------------------------------------------------------------

*From:*amd-gfx <[email protected] <mailto:[email protected]>> on behalf of Daniel Phillips <[email protected] <mailto:[email protected]>> *Sent:* Monday, January 10, 2022 3:54 PM
*To:* [email protected] <mailto:[email protected]> <[email protected] <mailto:[email protected]>>; [email protected] <mailto:[email protected]> <[email protected] <mailto:[email protected]>> *Cc:* Phillips, Daniel <[email protected] <mailto:[email protected]>>
*Subject:* [PATCH 1/1] Add available memory ioctl for libhsakmt

From: Daniel Phillips <[email protected] <mailto:[email protected]>>

Add an ioctl to inquire memory available for allocation by libhsakmt per node, allowing for space consumed by page translation tables.
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h      |  1 + .../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c    | 14 ++++++++++++++ drivers/gpu/drm/amd/amdkfd/kfd_chardev.c        | 17 +++++++++++++++++ include/uapi/linux/kfd_ioctl.h                  | 14 ++++++++++++--

 4 files changed, 44 insertions(+), 2 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h +1 -0
index fcbc8a9c9e06..64c6c36685d3 100644
@@ -266,6 +266,7 @@ int amdgpu_amdkfd_gpuvm_acquire_process_vm(struct
 amdgpu_device *adev,
  void amdgpu_amdkfd_gpuvm_release_process_vm(struct amdgpu_device *adev,
                                          void *drm_priv);
  uint64_t amdgpu_amdkfd_gpuvm_get_process_page_dir(void *drm_priv);
+size_t amdgpu_amdkfd_get_available_memory(struct amdgpu_device *adev);

 int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
                 struct amdgpu_device *adev, uint64_t va, uint64_t size, void *drm_priv, struct kgd_mem **mem,

drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c +0 -0
index 86a1a6c109d9..b7490a659173 100644
@@ -190,6 +190,20 @@ static int amdgpu_amdkfd_reserve_mem_limit(struct
 amdgpu_device *adev,
          return ret;
  }

drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c +14 -0
+size_t amdgpu_amdkfd_get_available_memory(struct amdgpu_device *adev)
+{
+       uint64_t reserved_for_pt =
+               ESTIMATE_PT_SIZE(amdgpu_amdkfd_total_mem_size);
+       size_t available_memory;
+
+       spin_lock(&kfd_mem_limit.mem_limit_lock);
+       available_memory =
+               adev->gmc.real_vram_size -
+               adev->kfd.vram_used - reserved_for_pt;
+       spin_unlock(&kfd_mem_limit.mem_limit_lock);
+       return available_memory;
+}
+

 static void unreserve_mem_limit(struct amdgpu_device *adev, uint64_t size, u32 alloc_flag)
 {

drivers/gpu/drm/amd/amdkfd/kfd_chardev.c +0 -0
index 4bfc0c8ab764..5c2f6d97ff1c 100644
@@ -486,6 +486,20 @@ static int kfd_ioctl_get_queue_wave_state(struct
 file *filep,
          return r;
  }

drivers/gpu/drm/amd/amdkfd/kfd_chardev.c +7 -0
+static int kfd_ioctl_get_available_memory(struct file *filep,
+                                struct kfd_process *p, void *data)
+{
+       struct kfd_ioctl_get_available_memory_args *args = data;
+       struct kfd_dev *dev;
+
+       dev = kfd_device_by_id(args->gpu_id);
Quoted 3 lines

Once CRIU changes land, this need to change to kfd_process_device_data_by_id() and then use pdd->dev

The CRIU patches will apply on top of this patch. So the CRIU patches have to adapt this code, just like other existing ioctl functions.

Regards,
  Felix

Quoted 72 lines

+       if (!dev)
+               return -EINVAL;

drivers/gpu/drm/amd/amdkfd/kfd_chardev.c +5 -0
+
+       args->available = amdgpu_amdkfd_get_available_memory(dev->adev);
+       return 0;
+}
+

 static int kfd_ioctl_set_memory_policy(struct file *filep, struct kfd_process *p, void *data)
 {

drivers/gpu/drm/amd/amdkfd/kfd_chardev.c +0 -0
@@ -1959,6 +1973,9 @@ static const struct amdkfd_ioctl_desc
 amdkfd_ioctls[] = {


         AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_XNACK_MODE, kfd_ioctl_set_xnack_mode, 0), +
+       AMDKFD_IOCTL_DEF(AMDKFD_IOC_AVAILABLE_MEMORY, +                       kfd_ioctl_get_available_memory, 0), };

 #define AMDKFD_CORE_IOCTL_COUNT ARRAY_SIZE(amdkfd_ioctls)

include/uapi/linux/kfd_ioctl.h +2 -1
index af96af174dc4..94a99add2432 100644
@@ -32,9 +32,10 @@
   * - 1.4 - Indicate new SRAM EDC bit in device properties
   * - 1.5 - Add SVM API
   * - 1.6 - Query clear flags in SVM get_attr API
+ * - 1.7 - Add available_memory ioctl
   */
  #define KFD_IOCTL_MAJOR_VERSION 1
-#define KFD_IOCTL_MINOR_VERSION 6
+#define KFD_IOCTL_MINOR_VERSION 7


 struct kfd_ioctl_get_version_args {
         __u32 major_version;    /* from KFD */

include/uapi/linux/kfd_ioctl.h +0 -0
@@ -98,6 +99,12 @@ struct kfd_ioctl_get_queue_wave_state_args {
          __u32 pad;
  };

include/uapi/linux/kfd_ioctl.h +6 -0
+struct kfd_ioctl_get_available_memory_args {
+       __u64 available;        /* from KFD */
+       __u32 gpu_id;           /* to KFD */
+       __u32 pad;
+};
+

 /* For kfd_ioctl_set_memory_policy_args.default_policy and alternate_policy */
 #define KFD_IOC_CACHE_POLICY_COHERENT 0
 #define KFD_IOC_CACHE_POLICY_NONCOHERENT 1

include/uapi/linux/kfd_ioctl.h +0 -0
@@ -742,7 +749,10 @@ struct kfd_ioctl_set_xnack_mode_args {
  #define AMDKFD_IOC_SET_XNACK_MODE               \
                  AMDKFD_IOWR(0x21, struct kfd_ioctl_set_xnack_mode_args)


+#define AMDKFD_IOC_AVAILABLE_MEMORY            \
+               AMDKFD_IOR(0x22, struct
kfd_ioctl_get_available_memory_args)
+
 #define AMDKFD_COMMAND_START            0x01

include/uapi/linux/kfd_ioctl.h +1 -1
-#define AMDKFD_COMMAND_END             0x22
+#define AMDKFD_COMMAND_END             0x23


 #endif

Signature · 1 line
2.34.1