Skip to contentSkip to conversation

drm/amdgpu: add psp command to get num xgmi links between direct peers

PATCH In review Jonathan Kim 3 participants 9 messages Jul 16, 2021 – Jul 19, 2021

Subsystems (3)
lore
Share & tools
lore
#0 lore raw

PATCH 1/3 drm/amdgpu: add psp command to get num xgmi links between direct peers

The TA can now be invoked to provide the number of xgmi links connecting a direct source and destination peer.
Non-direct peers will report zero links.

---
 drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c | 23 +++++++++++++++++++++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h |  1 +
 drivers/gpu/drm/amd/amdgpu/ta_xgmi_if.h | 14 +++++++++++++-
 3 files changed, 37 insertions(+), 1 deletion(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c +23 -0
index dfb481a0780f..4d6752ec8adc 100644
@@ -1074,6 +1074,12 @@ int psp_xgmi_get_node_id(struct psp_context *psp, uint64_t *node_id)
 	return 0;
 }
 
+static bool psp_xgmi_peer_link_info_supported(struct psp_context *psp)
+{
+	return psp->adev->asic_type == CHIP_ALDEBARAN &&
+				psp->ta_xgmi_ucode_version >= 0x2000000b;
+}
+
 int psp_xgmi_get_topology_info(struct psp_context *psp,
 			       int number_devices,
 			       struct psp_xgmi_topology_info *topology)
@@ -1117,6 +1123,23 @@ int psp_xgmi_get_topology_info(struct psp_context *psp,
 		topology->nodes[i].sdma_engine = topology_info_output->nodes[i].sdma_engine;
 	}
 
+	/* Invoke xgmi ta again to get the link information */
+	if (psp_xgmi_peer_link_info_supported(psp)) {
+		struct ta_xgmi_cmd_get_peer_link_info_output *link_info_output;
+
+		xgmi_cmd->cmd_id = TA_COMMAND_XGMI__GET_PEER_LINKS;
+
+		ret = psp_xgmi_invoke(psp, TA_COMMAND_XGMI__GET_PEER_LINKS);
+
+		if (ret)
+			return ret;
+
+		link_info_output = &xgmi_cmd->xgmi_out_message.get_link_info;
+		for (i = 0; i < topology->num_nodes; i++)
+			topology->nodes[i].num_links =
+					link_info_output->nodes[i].num_links;
+	}
+
 	return 0;
 }
 
drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h +1 -0
index 200d19139e73..f5b967b18e3f 100644
@@ -116,6 +116,7 @@ struct psp_xgmi_node_info {
 	uint8_t					num_hops;
 	uint8_t					is_sharing_enabled;
 	enum ta_xgmi_assigned_sdma_engine	sdma_engine;
+	uint8_t					num_links;
 };
 
 struct psp_xgmi_topology_info {
drivers/gpu/drm/amd/amdgpu/ta_xgmi_if.h +13 -1
index ac2c27b7630c..cce7127afeaa 100644
@@ -33,7 +33,8 @@ enum ta_command_xgmi {
 	TA_COMMAND_XGMI__GET_NODE_ID			= 0x01,
 	TA_COMMAND_XGMI__GET_HIVE_ID			= 0x02,
 	TA_COMMAND_XGMI__GET_GET_TOPOLOGY_INFO		= 0x03,
-	TA_COMMAND_XGMI__SET_TOPOLOGY_INFO		= 0x04
+	TA_COMMAND_XGMI__SET_TOPOLOGY_INFO		= 0x04,
+	TA_COMMAND_XGMI__GET_PEER_LINKS			= 0x0B
 };
 
 /* XGMI related enumerations */
@@ -75,6 +76,11 @@ struct ta_xgmi_node_info {
 	enum ta_xgmi_assigned_sdma_engine	sdma_engine;
 };
 
+struct ta_xgmi_peer_link_info {
+	uint64_t				node_id;
+	uint8_t					num_links;
+};
+
 struct ta_xgmi_cmd_initialize_output {
 	uint32_t	status;
 };
@@ -97,6 +103,11 @@ struct ta_xgmi_cmd_get_topology_info_output {
 	struct ta_xgmi_node_info	nodes[TA_XGMI__MAX_CONNECTED_NODES];
 };
 
+struct ta_xgmi_cmd_get_peer_link_info_output {
+	uint32_t			num_nodes;
+	struct ta_xgmi_peer_link_info	nodes[TA_XGMI__MAX_CONNECTED_NODES];
+};
+
 struct ta_xgmi_cmd_set_topology_info_input {
 	uint32_t			num_nodes;
 	struct ta_xgmi_node_info	nodes[TA_XGMI__MAX_CONNECTED_NODES];
@@ -115,6 +126,7 @@ union ta_xgmi_cmd_output {
 	struct ta_xgmi_cmd_get_node_id_output		get_node_id;
 	struct ta_xgmi_cmd_get_hive_id_output		get_hive_id;
 	struct ta_xgmi_cmd_get_topology_info_output	get_topology_info;
+	struct ta_xgmi_cmd_get_peer_link_info_output	get_link_info;
 };
 /**********************************************************/
 
Signature · 6 lines
2.25.1

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
#1 lore raw

PATCH 2/3 drm/amdkfd: report xgmi bandwidth between direct peers to the kfd

Report the min/max bandwidth in megabytes to the kfd for direct xgmi connections only.

v2: change reporting from num links to bandwidth

---
 drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c | 23 ++++++++++++++++++++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h |  1 +
 drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c   | 12 +++++++++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h   |  2 ++
 drivers/gpu/drm/amd/amdkfd/kfd_crat.c      | 12 +++++++++++
 5 files changed, 50 insertions(+)
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +23 -0
index bfab2f9fdd17..3978578a1c49 100644
@@ -553,6 +553,29 @@ uint8_t amdgpu_amdkfd_get_xgmi_hops_count(struct kgd_dev *dst, struct kgd_dev *s
 	return  (uint8_t)ret;
 }
 
+int amdgpu_amdkfd_get_xgmi_bandwidth_mbytes(struct kgd_dev *dst, struct kgd_dev *src, bool is_min)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)dst, *peer_adev;
+	int num_links;
+
+	if (adev->asic_type != CHIP_ALDEBARAN)
+		return 0;
+
+	if (src)
+		peer_adev = (struct amdgpu_device *)src;
+
+	num_links = is_min ? 1 : amdgpu_xgmi_get_num_links(adev, peer_adev);
+	if (num_links < 0) {
+		DRM_ERROR("amdgpu: failed to get xgmi num links between node %d and %d. ret = %d\n",
+			adev->gmc.xgmi.physical_node_id,
+			peer_adev->gmc.xgmi.physical_node_id, num_links);
+		num_links = 0;
+	}
+
+	/* Aldebaran xGMI DPM is defeatured so assume x16 x 25Gbps for bandwidth. */
+	return (num_links * 16 * 25000)/BITS_PER_BYTE;
+}
+
 uint64_t amdgpu_amdkfd_get_mmio_remap_phys_addr(struct kgd_dev *kgd)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h +1 -0
index 81264517d532..e12fccb2d2c4 100644
@@ -226,6 +226,7 @@ uint32_t amdgpu_amdkfd_get_num_gws(struct kgd_dev *kgd);
 uint32_t amdgpu_amdkfd_get_asic_rev_id(struct kgd_dev *kgd);
 int amdgpu_amdkfd_get_noretry(struct kgd_dev *kgd);
 uint8_t amdgpu_amdkfd_get_xgmi_hops_count(struct kgd_dev *dst, struct kgd_dev *src);
+int amdgpu_amdkfd_get_xgmi_bandwidth_mbytes(struct kgd_dev *dst, struct kgd_dev *src, bool is_min);
 
 /* Read user wptr from a specified user address space with page fault
  * disabled. The memory must be pinned and mapped to the hardware when
drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c +12 -0
index 8567d5d77346..258cf86b32f6 100644
@@ -486,6 +486,18 @@ int amdgpu_xgmi_get_hops_count(struct amdgpu_device *adev,
 	return	-EINVAL;
 }
 
+int amdgpu_xgmi_get_num_links(struct amdgpu_device *adev,
+		struct amdgpu_device *peer_adev)
+{
+	struct psp_xgmi_topology_info *top = &adev->psp.xgmi_context.top_info;
+	int i;
+
+	for (i = 0 ; i < top->num_nodes; ++i)
+		if (top->nodes[i].node_id == peer_adev->gmc.xgmi.node_id)
+			return top->nodes[i].num_links;
+	return	-EINVAL;
+}
+
 int amdgpu_xgmi_add_device(struct amdgpu_device *adev)
 {
 	struct psp_xgmi_topology_info *top_info;
drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h +2 -0
index 12969c0830d5..d2189bf7d428 100644
@@ -59,6 +59,8 @@ int amdgpu_xgmi_remove_device(struct amdgpu_device *adev);
 int amdgpu_xgmi_set_pstate(struct amdgpu_device *adev, int pstate);
 int amdgpu_xgmi_get_hops_count(struct amdgpu_device *adev,
 		struct amdgpu_device *peer_adev);
+int amdgpu_xgmi_get_num_links(struct amdgpu_device *adev,
+		struct amdgpu_device *peer_adev);
 uint64_t amdgpu_xgmi_get_relative_phy_addr(struct amdgpu_device *adev,
 					   uint64_t addr);
 static inline bool amdgpu_xgmi_same_hive(struct amdgpu_device *adev,
drivers/gpu/drm/amd/amdkfd/kfd_crat.c +12 -0
index c6b02aee4993..40ce6239c813 100644
@@ -1989,6 +1989,13 @@ static int kfd_fill_gpu_direct_io_link_to_cpu(int *avail_size,
 		sub_type_hdr->flags |= CRAT_IOLINK_FLAGS_BI_DIRECTIONAL;
 		sub_type_hdr->io_interface_type = CRAT_IOLINK_TYPE_XGMI;
 		sub_type_hdr->num_hops_xgmi = 1;
+		if (adev->asic_type == CHIP_ALDEBARAN) {
+			sub_type_hdr->minimum_bandwidth_mbs =
+					amdgpu_amdkfd_get_xgmi_bandwidth_mbytes(
+							kdev->kgd, NULL, true);
+			sub_type_hdr->maximum_bandwidth_mbs =
+					sub_type_hdr->minimum_bandwidth_mbs;
+		}
 	} else {
 		sub_type_hdr->io_interface_type = CRAT_IOLINK_TYPE_PCIEXPRESS;
 	}
@@ -2033,6 +2040,11 @@ static int kfd_fill_gpu_xgmi_link_to_gpu(int *avail_size,
 	sub_type_hdr->proximity_domain_to = proximity_domain_to;
 	sub_type_hdr->num_hops_xgmi =
 		amdgpu_amdkfd_get_xgmi_hops_count(kdev->kgd, peer_kdev->kgd);
+	sub_type_hdr->maximum_bandwidth_mbs =
+		amdgpu_amdkfd_get_xgmi_bandwidth_mbytes(kdev->kgd, peer_kdev->kgd, false);
+	sub_type_hdr->minimum_bandwidth_mbs = sub_type_hdr->maximum_bandwidth_mbs ?
+		amdgpu_amdkfd_get_xgmi_bandwidth_mbytes(kdev->kgd, NULL, true) : 0;
+
 	return 0;
 }
 
Signature · 6 lines
2.25.1

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
maintainer Reviewed-by on v1 2/3
#2 lore raw

Re: [PATCH 2/3] drm/amdkfd: report xgmi bandwidth between direct peers to the kfd

Am 2021-07-16 um 12:43 p.m. schrieb Jonathan Kim:

Quoted 2 lines

Report the min/max bandwidth in megabytes to the kfd for direct xgmi connections only.

By "direct XGMI connections", you mean this doesn't work for links with more than one hop? Will that spew out DRM_ERROR messages for such links? Then it's probably better to downgrade that to an INFO.

Quoted 4 lines

v2: change reporting from num links to bandwidth

This patch is OK to provide bandwidth information on Aldebaran. What can we do on older GPUs? Can we assume num_links = 1? Or maybe have some hard-coded numbers depending on the number of nodes in the hive?

Either way, patch 1 and 2 are

Quoted 110 lines
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c | 23 ++++++++++++++++++++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h |  1 +
 drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c   | 12 +++++++++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h   |  2 ++
 drivers/gpu/drm/amd/amdkfd/kfd_crat.c      | 12 +++++++++++
 5 files changed, 50 insertions(+)
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +23 -0
index bfab2f9fdd17..3978578a1c49 100644
@@ -553,6 +553,29 @@ uint8_t amdgpu_amdkfd_get_xgmi_hops_count(struct kgd_dev *dst, struct kgd_dev *s
 	return  (uint8_t)ret;
 }
 
+int amdgpu_amdkfd_get_xgmi_bandwidth_mbytes(struct kgd_dev *dst, struct kgd_dev *src, bool is_min)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)dst, *peer_adev;
+	int num_links;
+
+	if (adev->asic_type != CHIP_ALDEBARAN)
+		return 0;
+
+	if (src)
+		peer_adev = (struct amdgpu_device *)src;
+
+	num_links = is_min ? 1 : amdgpu_xgmi_get_num_links(adev, peer_adev);
+	if (num_links < 0) {
+		DRM_ERROR("amdgpu: failed to get xgmi num links between node %d and %d. ret = %d\n",
+			adev->gmc.xgmi.physical_node_id,
+			peer_adev->gmc.xgmi.physical_node_id, num_links);
+		num_links = 0;
+	}
+
+	/* Aldebaran xGMI DPM is defeatured so assume x16 x 25Gbps for bandwidth. */
+	return (num_links * 16 * 25000)/BITS_PER_BYTE;
+}
+
 uint64_t amdgpu_amdkfd_get_mmio_remap_phys_addr(struct kgd_dev *kgd)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h +1 -0
index 81264517d532..e12fccb2d2c4 100644
@@ -226,6 +226,7 @@ uint32_t amdgpu_amdkfd_get_num_gws(struct kgd_dev *kgd);
 uint32_t amdgpu_amdkfd_get_asic_rev_id(struct kgd_dev *kgd);
 int amdgpu_amdkfd_get_noretry(struct kgd_dev *kgd);
 uint8_t amdgpu_amdkfd_get_xgmi_hops_count(struct kgd_dev *dst, struct kgd_dev *src);
+int amdgpu_amdkfd_get_xgmi_bandwidth_mbytes(struct kgd_dev *dst, struct kgd_dev *src, bool is_min);
 
 /* Read user wptr from a specified user address space with page fault
  * disabled. The memory must be pinned and mapped to the hardware when
drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c +12 -0
index 8567d5d77346..258cf86b32f6 100644
@@ -486,6 +486,18 @@ int amdgpu_xgmi_get_hops_count(struct amdgpu_device *adev,
 	return	-EINVAL;
 }
 
+int amdgpu_xgmi_get_num_links(struct amdgpu_device *adev,
+		struct amdgpu_device *peer_adev)
+{
+	struct psp_xgmi_topology_info *top = &adev->psp.xgmi_context.top_info;
+	int i;
+
+	for (i = 0 ; i < top->num_nodes; ++i)
+		if (top->nodes[i].node_id == peer_adev->gmc.xgmi.node_id)
+			return top->nodes[i].num_links;
+	return	-EINVAL;
+}
+
 int amdgpu_xgmi_add_device(struct amdgpu_device *adev)
 {
 	struct psp_xgmi_topology_info *top_info;
drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h +2 -0
index 12969c0830d5..d2189bf7d428 100644
@@ -59,6 +59,8 @@ int amdgpu_xgmi_remove_device(struct amdgpu_device *adev);
 int amdgpu_xgmi_set_pstate(struct amdgpu_device *adev, int pstate);
 int amdgpu_xgmi_get_hops_count(struct amdgpu_device *adev,
 		struct amdgpu_device *peer_adev);
+int amdgpu_xgmi_get_num_links(struct amdgpu_device *adev,
+		struct amdgpu_device *peer_adev);
 uint64_t amdgpu_xgmi_get_relative_phy_addr(struct amdgpu_device *adev,
 					   uint64_t addr);
 static inline bool amdgpu_xgmi_same_hive(struct amdgpu_device *adev,
drivers/gpu/drm/amd/amdkfd/kfd_crat.c +7 -0
index c6b02aee4993..40ce6239c813 100644
@@ -1989,6 +1989,13 @@ static int kfd_fill_gpu_direct_io_link_to_cpu(int *avail_size,
 		sub_type_hdr->flags |= CRAT_IOLINK_FLAGS_BI_DIRECTIONAL;
 		sub_type_hdr->io_interface_type = CRAT_IOLINK_TYPE_XGMI;
 		sub_type_hdr->num_hops_xgmi = 1;
+		if (adev->asic_type == CHIP_ALDEBARAN) {
+			sub_type_hdr->minimum_bandwidth_mbs =
+					amdgpu_amdkfd_get_xgmi_bandwidth_mbytes(
+							kdev->kgd, NULL, true);
+			sub_type_hdr->maximum_bandwidth_mbs =
+					sub_type_hdr->minimum_bandwidth_mbs;
+		}
 	} else {
 		sub_type_hdr->io_interface_type = CRAT_IOLINK_TYPE_PCIEXPRESS;
 	}
@@ -2033,6 +2040,11 @@ static int kfd_fill_gpu_xgmi_link_to_gpu(int *avail_size,
 	sub_type_hdr->proximity_domain_to = proximity_domain_to;

drivers/gpu/drm/amd/amdkfd/kfd_crat.c static int kfd_fill_gpu_xgmi_link_to_gpu(int *avail_size, · line 2050

Quoted 10 lines
drivers/gpu/drm/amd/amdkfd/kfd_crat.c +5 -0
 	sub_type_hdr->num_hops_xgmi =
 		amdgpu_amdkfd_get_xgmi_hops_count(kdev->kgd, peer_kdev->kgd);
+	sub_type_hdr->maximum_bandwidth_mbs =
+		amdgpu_amdkfd_get_xgmi_bandwidth_mbytes(kdev->kgd, peer_kdev->kgd, false);
+	sub_type_hdr->minimum_bandwidth_mbs = sub_type_hdr->maximum_bandwidth_mbs ?
+		amdgpu_amdkfd_get_xgmi_bandwidth_mbytes(kdev->kgd, NULL, true) : 0;
+
 	return 0;
 }
 

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

#3 lore raw

RE: [PATCH 2/3] drm/amdkfd: report xgmi bandwidth between direct peers to the kfd

[AMD Official Use Only]

Quoted 8 lines

-----Original Message-----
From: Kuehling, Felix <[email protected]>
Sent: Saturday, July 17, 2021 1:47 AM
To: Kim, Jonathan <[email protected]>; amd- [email protected]
Subject: Re: [PATCH 2/3] drm/amdkfd: report xgmi bandwidth between direct peers to the kfd

Quoted 7 lines

Am 2021-07-16 um 12:43 p.m. schrieb Jonathan Kim:

Quoted 2 lines

Report the min/max bandwidth in megabytes to the kfd for direct xgmi connections only.

By "direct XGMI connections", you mean this doesn't work for links with more than one hop? Will that spew out DRM_ERROR messages for such links? Then it's probably better to downgrade that to an INFO.

No DRM_ERROR only happens if psp fails on invoke.
I've added footnotes to the description and code to clear this up.
Non-adjacent peers return num_links as 0 since indirect route is unknown and linkage is asymmetrical.

Quoted 10 lines
Quoted 6 lines

v2: change reporting from num links to bandwidth

This patch is OK to provide bandwidth information on Aldebaran. What can we do on older GPUs? Can we assume num_links = 1? Or maybe have some hard-coded numbers depending on the number of nodes in the hive?

We could assume num_links = 1 but that wouldn't represent certain non-Aldebaran setups well.
For non-Aldebaran min/max bandwidth, we may be able to get away with setting non-zero values on non-adjacent peers since setup is symmetrical to date but that may raise questions on why Aldebaran indirect min/max-bandwidth is 0. For consistency, we'd have to use num_hops then to check directness. Maybe it's worth making a bid to the FW team to support all other chips moving forward ...

Thanks,

Jon

Quoted 145 lines

Either way, patch 1 and 2 are

Quoted 3 lines
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c | 23

++++++++++++++++++++++

drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h |  1 +
>  drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c   | 12 +++++++++++
>  drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h   |  2 ++
>  drivers/gpu/drm/amd/amdkfd/kfd_crat.c      | 12 +++++++++++

drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c uint8_t

Quoted 8 lines
 5 files changed, 50 insertions(+)
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +0 -0
index bfab2f9fdd17..3978578a1c49 100644
@@ -553,6 +553,29 @@ uint8_t

amdgpu_amdkfd_get_xgmi_hops_count(struct kgd_dev *dst, struct kgd_dev *s

drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c uint8_t amdgpu_amdkfd_get_xgmi_hops_count(struct kgd_dev *dst, struct kgd_dev *s

Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +3 -0
    return  (uint8_t)ret;
 }
 
+int amdgpu_amdkfd_get_xgmi_bandwidth_mbytes(struct kgd_dev *dst,
+struct kgd_dev *src, bool is_min) {
+   struct amdgpu_device *adev = (struct amdgpu_device *)dst,

*peer_adev;

drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c uint8_t amdgpu_amdkfd_get_xgmi_hops_count(struct kgd_dev *dst, struct kgd_dev *s

Quoted 9 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +9 -0
+   int num_links;
+
+   if (adev->asic_type != CHIP_ALDEBARAN)
+           return 0;
+
+   if (src)
+           peer_adev = (struct amdgpu_device *)src;
+
+   num_links = is_min ? 1 : amdgpu_xgmi_get_num_links(adev,

peer_adev);

drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c uint8_t · line 569

Quoted 2 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +2 -0
+   if (num_links < 0) {
+           DRM_ERROR("amdgpu: failed to get xgmi num links between

node %d and %d. ret = %d\n",

drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c uint8_t amdgpu_amdkfd_get_xgmi_hops_count(struct kgd_dev *dst, struct kgd_dev *s

Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +6 -0
+                   adev->gmc.xgmi.physical_node_id,
+                   peer_adev->gmc.xgmi.physical_node_id, num_links);
+           num_links = 0;
+   }
+
+   /* Aldebaran xGMI DPM is defeatured so assume x16 x 25Gbps for

bandwidth. */

drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c uint8_t · line 578

Quoted 3 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +2 -0
+   return (num_links * 16 * 25000)/BITS_PER_BYTE; }
+
 uint64_t amdgpu_amdkfd_get_mmio_remap_phys_addr(struct kgd_dev

*kgd)

drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h uint32_t amdgpu_amdkfd_get_num_gws(struct

Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +0 -0
 {
    struct amdgpu_device *adev = (struct amdgpu_device *)kgd; diff --git

a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
index 81264517d532..e12fccb2d2c4 100644

drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h +0 -0
@@ -226,6 +226,7 @@ uint32_t amdgpu_amdkfd_get_num_gws(struct

kgd_dev

Quoted 3 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h +0 -0
 *kgd);  uint32_t amdgpu_amdkfd_get_asic_rev_id(struct kgd_dev *kgd);

int amdgpu_amdkfd_get_noretry(struct kgd_dev *kgd); uint8_t amdgpu_amdkfd_get_xgmi_hops_count(struct kgd_dev *dst, struct

kgd_dev

drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c int amdgpu_xgmi_get_hops_count(struct

Quoted 12 lines

*src);

drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h +2 -0
+int amdgpu_amdkfd_get_xgmi_bandwidth_mbytes(struct kgd_dev *dst,
+struct kgd_dev *src, bool is_min);
 
 /* Read user wptr from a specified user address space with page fault
  * disabled. The memory must be pinned and mapped to the hardware

when diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c
index 8567d5d77346..258cf86b32f6 100644

drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c +0 -0
@@ -486,6 +486,18 @@ int amdgpu_xgmi_get_hops_count(struct

amdgpu_device *adev,

drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h int amdgpu_xgmi_remove_device(struct

Quoted 24 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c +12 -0
    return  -EINVAL;
 }
 
+int amdgpu_xgmi_get_num_links(struct amdgpu_device *adev,
+           struct amdgpu_device *peer_adev)
+{
+   struct psp_xgmi_topology_info *top = &adev-
 psp.xgmi_context.top_info;
+   int i;
+
+   for (i = 0 ; i < top->num_nodes; ++i)
+           if (top->nodes[i].node_id == peer_adev->gmc.xgmi.node_id)
+                   return top->nodes[i].num_links;
+   return  -EINVAL;
+}
+
 int amdgpu_xgmi_add_device(struct amdgpu_device *adev)  {
    struct psp_xgmi_topology_info *top_info; diff --git

a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h
b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h
index 12969c0830d5..d2189bf7d428 100644

drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h +0 -0
@@ -59,6 +59,8 @@ int amdgpu_xgmi_remove_device(struct

amdgpu_device

drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h int amdgpu_xgmi_remove_device(struct amdgpu_device *adev);

Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h +2 -0
 *adev);  int amdgpu_xgmi_set_pstate(struct amdgpu_device *adev, int
 pstate);  int amdgpu_xgmi_get_hops_count(struct amdgpu_device *adev,
            struct amdgpu_device *peer_adev);
+int amdgpu_xgmi_get_num_links(struct amdgpu_device *adev,
+           struct amdgpu_device *peer_adev);
 uint64_t amdgpu_xgmi_get_relative_phy_addr(struct amdgpu_device

*adev,

drivers/gpu/drm/amd/amdkfd/kfd_crat.c static int kfd_fill_gpu_direct_io_link_to_cpu(int

Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h +0 -0
                                       uint64_t addr);
 static inline bool amdgpu_xgmi_same_hive(struct amdgpu_device *adev,
drivers/gpu/drm/amd/amdkfd/kfd_crat.c +0 -0
index c6b02aee4993..40ce6239c813 100644
@@ -1989,6 +1989,13 @@ static int kfd_fill_gpu_direct_io_link_to_cpu(int

*avail_size,

drivers/gpu/drm/amd/amdkfd/kfd_crat.c static int kfd_fill_gpu_direct_io_link_to_cpu(int · line 1989

Quoted 1 line
drivers/gpu/drm/amd/amdkfd/kfd_crat.c +0 -0
            sub_type_hdr->flags |=

CRAT_IOLINK_FLAGS_BI_DIRECTIONAL;

drivers/gpu/drm/amd/amdkfd/kfd_crat.c static int kfd_fill_gpu_direct_io_link_to_cpu(int · line 1990

Quoted 1 line
drivers/gpu/drm/amd/amdkfd/kfd_crat.c +0 -0
            sub_type_hdr->io_interface_type =

CRAT_IOLINK_TYPE_XGMI;

drivers/gpu/drm/amd/amdkfd/kfd_crat.c static int kfd_fill_gpu_direct_io_link_to_cpu(int *avail_size,

Quoted 4 lines
drivers/gpu/drm/amd/amdkfd/kfd_crat.c +3 -0
            sub_type_hdr->num_hops_xgmi = 1;
+           if (adev->asic_type == CHIP_ALDEBARAN) {
+                   sub_type_hdr->minimum_bandwidth_mbs =
+
      amdgpu_amdkfd_get_xgmi_bandwidth_mbytes(

drivers/gpu/drm/amd/amdkfd/kfd_crat.c static int kfd_fill_gpu_direct_io_link_to_cpu(int · line 1995

Quoted 1 line
drivers/gpu/drm/amd/amdkfd/kfd_crat.c +1 -0
+                                                   kdev->kgd, NULL,

true);

drivers/gpu/drm/amd/amdkfd/kfd_crat.c static int kfd_fill_gpu_direct_io_link_to_cpu(int *avail_size,

Quoted 6 lines
drivers/gpu/drm/amd/amdkfd/kfd_crat.c +3 -0
+                   sub_type_hdr->maximum_bandwidth_mbs =
+                                   sub_type_hdr-
 minimum_bandwidth_mbs;
+           }
    } else {
            sub_type_hdr->io_interface_type =

CRAT_IOLINK_TYPE_PCIEXPRESS;

drivers/gpu/drm/amd/amdkfd/kfd_crat.c static int kfd_fill_gpu_xgmi_link_to_gpu(int

Quoted 2 lines
    }
drivers/gpu/drm/amd/amdkfd/kfd_crat.c +0 -0
@@ -2033,6 +2040,11 @@ static int kfd_fill_gpu_xgmi_link_to_gpu(int

*avail_size,

drivers/gpu/drm/amd/amdkfd/kfd_crat.c static int kfd_fill_gpu_xgmi_link_to_gpu(int *avail_size,

Quoted 3 lines
drivers/gpu/drm/amd/amdkfd/kfd_crat.c +0 -0
    sub_type_hdr->proximity_domain_to = proximity_domain_to;
    sub_type_hdr->num_hops_xgmi =
            amdgpu_amdkfd_get_xgmi_hops_count(kdev->kgd,
Quoted 12 lines

peer_kdev->kgd);

drivers/gpu/drm/amd/amdkfd/kfd_crat.c static int kfd_fill_gpu_xgmi_link_to_gpu(int · line 2044

Quoted 2 lines
drivers/gpu/drm/amd/amdkfd/kfd_crat.c +2 -0
+   sub_type_hdr->maximum_bandwidth_mbs =
+           amdgpu_amdkfd_get_xgmi_bandwidth_mbytes(kdev->kgd,

peer_kdev->kgd, false);

Quoted 3 lines
drivers/gpu/drm/amd/amdkfd/kfd_crat.c +1 -0
+   sub_type_hdr->minimum_bandwidth_mbs = sub_type_hdr-

maximum_bandwidth_mbs ?
+ amdgpu_amdkfd_get_xgmi_bandwidth_mbytes(kdev->kgd,

NULL, true) : 0;

Quoted 4 lines

+

    return 0;

}

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

#4 lore raw

Re: [PATCH 2/3] drm/amdkfd: report xgmi bandwidth between direct peers to the kfd

In reply to Jonathan Kim
Quoted 33 lines · Jonathan Kim

Report the min/max bandwidth in megabytes to the kfd for direct xgmi connections only.

v2: change reporting from num links to bandwidth

---
  drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c | 23 ++++++++++++++++++++++
  drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h |  1 +
  drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c   | 12 +++++++++++
  drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h   |  2 ++
  drivers/gpu/drm/amd/amdkfd/kfd_crat.c      | 12 +++++++++++
  5 files changed, 50 insertions(+)
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +11 -0
index bfab2f9fdd17..3978578a1c49 100644
@@ -553,6 +553,29 @@ uint8_t amdgpu_amdkfd_get_xgmi_hops_count(struct kgd_dev *dst, struct kgd_dev *s
  	return  (uint8_t)ret;
  }
 
+int amdgpu_amdkfd_get_xgmi_bandwidth_mbytes(struct kgd_dev *dst, struct kgd_dev *src, bool is_min)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)dst, *peer_adev;
+	int num_links;
+
+	if (adev->asic_type != CHIP_ALDEBARAN)
+		return 0;
+
+	if (src)
+		peer_adev = (struct amdgpu_device *)src;
+

drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c uint8_t amdgpu_amdkfd_get_xgmi_hops_count(struct kgd_dev *dst, struct kgd_dev *s · line 576

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +10 -0
+	num_links = is_min ? 1 : amdgpu_xgmi_get_num_links(adev, peer_adev);
+	if (num_links < 0) {
+		DRM_ERROR("amdgpu: failed to get xgmi num links between node %d and %d. ret = %d\n",
+			adev->gmc.xgmi.physical_node_id,
+			peer_adev->gmc.xgmi.physical_node_id, num_links);
+		num_links = 0;
+	}
+
+	/* Aldebaran xGMI DPM is defeatured so assume x16 x 25Gbps for bandwidth. */
+	return (num_links * 16 * 25000)/BITS_PER_BYTE;

Instead of having ASIC family checks and bandwidth info in interface functions, better to have this info come from base layer (amdgpu_xgmi or xgmi ip). That will help to handle other ASICs.

Thanks,
Lijo

Quoted 72 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +0 -0
  uint64_t amdgpu_amdkfd_get_mmio_remap_phys_addr(struct kgd_dev *kgd)
  {
  	struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h +1 -0
index 81264517d532..e12fccb2d2c4 100644
@@ -226,6 +226,7 @@ uint32_t amdgpu_amdkfd_get_num_gws(struct kgd_dev *kgd);
  uint32_t amdgpu_amdkfd_get_asic_rev_id(struct kgd_dev *kgd);
  int amdgpu_amdkfd_get_noretry(struct kgd_dev *kgd);
  uint8_t amdgpu_amdkfd_get_xgmi_hops_count(struct kgd_dev *dst, struct kgd_dev *src);
+int amdgpu_amdkfd_get_xgmi_bandwidth_mbytes(struct kgd_dev *dst, struct kgd_dev *src, bool is_min);
 
  /* Read user wptr from a specified user address space with page fault
   * disabled. The memory must be pinned and mapped to the hardware when
drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c +12 -0
index 8567d5d77346..258cf86b32f6 100644
@@ -486,6 +486,18 @@ int amdgpu_xgmi_get_hops_count(struct amdgpu_device *adev,
  	return	-EINVAL;
  }
 
+int amdgpu_xgmi_get_num_links(struct amdgpu_device *adev,
+		struct amdgpu_device *peer_adev)
+{
+	struct psp_xgmi_topology_info *top = &adev->psp.xgmi_context.top_info;
+	int i;
+
+	for (i = 0 ; i < top->num_nodes; ++i)
+		if (top->nodes[i].node_id == peer_adev->gmc.xgmi.node_id)
+			return top->nodes[i].num_links;
+	return	-EINVAL;
+}
+
  int amdgpu_xgmi_add_device(struct amdgpu_device *adev)
  {
  	struct psp_xgmi_topology_info *top_info;
drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h +2 -0
index 12969c0830d5..d2189bf7d428 100644
@@ -59,6 +59,8 @@ int amdgpu_xgmi_remove_device(struct amdgpu_device *adev);
  int amdgpu_xgmi_set_pstate(struct amdgpu_device *adev, int pstate);
  int amdgpu_xgmi_get_hops_count(struct amdgpu_device *adev,
  		struct amdgpu_device *peer_adev);
+int amdgpu_xgmi_get_num_links(struct amdgpu_device *adev,
+		struct amdgpu_device *peer_adev);
  uint64_t amdgpu_xgmi_get_relative_phy_addr(struct amdgpu_device *adev,
  					   uint64_t addr);
  static inline bool amdgpu_xgmi_same_hive(struct amdgpu_device *adev,
drivers/gpu/drm/amd/amdkfd/kfd_crat.c +7 -0
index c6b02aee4993..40ce6239c813 100644
@@ -1989,6 +1989,13 @@ static int kfd_fill_gpu_direct_io_link_to_cpu(int *avail_size,
  		sub_type_hdr->flags |= CRAT_IOLINK_FLAGS_BI_DIRECTIONAL;
  		sub_type_hdr->io_interface_type = CRAT_IOLINK_TYPE_XGMI;
  		sub_type_hdr->num_hops_xgmi = 1;
+		if (adev->asic_type == CHIP_ALDEBARAN) {
+			sub_type_hdr->minimum_bandwidth_mbs =
+					amdgpu_amdkfd_get_xgmi_bandwidth_mbytes(
+							kdev->kgd, NULL, true);
+			sub_type_hdr->maximum_bandwidth_mbs =
+					sub_type_hdr->minimum_bandwidth_mbs;
+		}
  	} else {
  		sub_type_hdr->io_interface_type = CRAT_IOLINK_TYPE_PCIEXPRESS;
  	}
@@ -2033,6 +2040,11 @@ static int kfd_fill_gpu_xgmi_link_to_gpu(int *avail_size,
  	sub_type_hdr->proximity_domain_to = proximity_domain_to;

drivers/gpu/drm/amd/amdkfd/kfd_crat.c static int kfd_fill_gpu_xgmi_link_to_gpu(int *avail_size, · line 2050

Quoted 10 lines
drivers/gpu/drm/amd/amdkfd/kfd_crat.c +5 -0
  	sub_type_hdr->num_hops_xgmi =
  		amdgpu_amdkfd_get_xgmi_hops_count(kdev->kgd, peer_kdev->kgd);
+	sub_type_hdr->maximum_bandwidth_mbs =
+		amdgpu_amdkfd_get_xgmi_bandwidth_mbytes(kdev->kgd, peer_kdev->kgd, false);
+	sub_type_hdr->minimum_bandwidth_mbs = sub_type_hdr->maximum_bandwidth_mbs ?
+		amdgpu_amdkfd_get_xgmi_bandwidth_mbytes(kdev->kgd, NULL, true) : 0;
+
  	return 0;
  }
 

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

#5 lore raw

RE: [PATCH 2/3] drm/amdkfd: report xgmi bandwidth between direct peers to the kfd

[AMD Official Use Only]

Quoted 61 lines

-----Original Message-----
From: Lazar, Lijo <[email protected]>
Sent: Monday, July 19, 2021 3:22 AM
To: Kim, Jonathan <[email protected]>; amd- [email protected]
Cc: Kuehling, Felix <[email protected]>
Subject: Re: [PATCH 2/3] drm/amdkfd: report xgmi bandwidth between direct peers to the kfd

Quoted 8 lines · Jonathan Kim

Report the min/max bandwidth in megabytes to the kfd for direct xgmi connections only.

v2: change reporting from num links to bandwidth

---
  drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c | 23

++++++++++++++++++++++

drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c uint8_t

Quoted 12 lines
  drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h |  1 +
  drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c   | 12 +++++++++++
  drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h   |  2 ++
  drivers/gpu/drm/amd/amdkfd/kfd_crat.c      | 12 +++++++++++
  5 files changed, 50 insertions(+)
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +0 -0
index bfab2f9fdd17..3978578a1c49 100644
@@ -553,6 +553,29 @@ uint8_t

amdgpu_amdkfd_get_xgmi_hops_count(struct kgd_dev *dst, struct kgd_dev *s

drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c uint8_t amdgpu_amdkfd_get_xgmi_hops_count(struct kgd_dev *dst, struct kgd_dev *s

Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +3 -0
    return  (uint8_t)ret;
  }
 
+int amdgpu_amdkfd_get_xgmi_bandwidth_mbytes(struct kgd_dev *dst,
+struct kgd_dev *src, bool is_min) {
+   struct amdgpu_device *adev = (struct amdgpu_device *)dst,

*peer_adev;

drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c uint8_t amdgpu_amdkfd_get_xgmi_hops_count(struct kgd_dev *dst, struct kgd_dev *s

Quoted 9 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +9 -0
+   int num_links;
+
+   if (adev->asic_type != CHIP_ALDEBARAN)
+           return 0;
+
+   if (src)
+           peer_adev = (struct amdgpu_device *)src;
+
+   num_links = is_min ? 1 : amdgpu_xgmi_get_num_links(adev,

peer_adev);

drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c uint8_t · line 569

Quoted 2 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +2 -0
+   if (num_links < 0) {
+           DRM_ERROR("amdgpu: failed to get xgmi num links between

node %d and %d. ret = %d\n",

drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c uint8_t amdgpu_amdkfd_get_xgmi_hops_count(struct kgd_dev *dst, struct kgd_dev *s

Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +6 -0
+                   adev->gmc.xgmi.physical_node_id,
+                   peer_adev->gmc.xgmi.physical_node_id, num_links);
+           num_links = 0;
+   }
+
+   /* Aldebaran xGMI DPM is defeatured so assume x16 x 25Gbps for
Quoted 6 lines

bandwidth. */

drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c uint8_t amdgpu_amdkfd_get_xgmi_hops_count(struct kgd_dev *dst, struct kgd_dev *s · line 576

Quoted 1 line
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +1 -0
+   return (num_links * 16 * 25000)/BITS_PER_BYTE;

Instead of having ASIC family checks and bandwidth info in interface functions, better to have this info come from base layer (amdgpu_xgmi or xgmi ip). That will help to handle other ASICs.

Ok. We can revisit this as a follow up. Maybe the full solution is a link width/speed support mask analogous to pcie.

Thanks,

Jon

Quoted 97 lines

Thanks,
Lijo

drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c uint8_t · line 577

Quoted 1 line
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +0 -0
  uint64_t amdgpu_amdkfd_get_mmio_remap_phys_addr(struct kgd_dev

*kgd)

drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h uint32_t amdgpu_amdkfd_get_num_gws(struct

Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +0 -1
  {
    struct amdgpu_device *adev = (struct amdgpu_device *)kgd; diff
--git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h

b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
index 81264517d532..e12fccb2d2c4 100644

drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h +0 -0
@@ -226,6 +226,7 @@ uint32_t amdgpu_amdkfd_get_num_gws(struct

kgd_dev *kgd);

drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c int amdgpu_xgmi_get_hops_count(struct

Quoted 15 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h +2 -0
  uint32_t amdgpu_amdkfd_get_asic_rev_id(struct kgd_dev *kgd);
  int amdgpu_amdkfd_get_noretry(struct kgd_dev *kgd);
  uint8_t amdgpu_amdkfd_get_xgmi_hops_count(struct kgd_dev *dst,
 struct kgd_dev *src);
+int amdgpu_amdkfd_get_xgmi_bandwidth_mbytes(struct kgd_dev *dst,
+struct kgd_dev *src, bool is_min);
 
  /* Read user wptr from a specified user address space with page fault
   * disabled. The memory must be pinned and mapped to the hardware

when diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c
index 8567d5d77346..258cf86b32f6 100644

drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c +0 -0
@@ -486,6 +486,18 @@ int amdgpu_xgmi_get_hops_count(struct

amdgpu_device *adev,

drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h int amdgpu_xgmi_remove_device(struct

Quoted 25 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c +12 -0
    return  -EINVAL;
  }
 
+int amdgpu_xgmi_get_num_links(struct amdgpu_device *adev,
+           struct amdgpu_device *peer_adev)
+{
+   struct psp_xgmi_topology_info *top = &adev-
 psp.xgmi_context.top_info;
+   int i;
+
+   for (i = 0 ; i < top->num_nodes; ++i)
+           if (top->nodes[i].node_id == peer_adev->gmc.xgmi.node_id)
+                   return top->nodes[i].num_links;
+   return  -EINVAL;
+}
+
  int amdgpu_xgmi_add_device(struct amdgpu_device *adev)
  {
    struct psp_xgmi_topology_info *top_info; diff --git

a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h
b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h
index 12969c0830d5..d2189bf7d428 100644

drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h +0 -0
@@ -59,6 +59,8 @@ int amdgpu_xgmi_remove_device(struct

amdgpu_device *adev);

drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h int amdgpu_xgmi_remove_device(struct amdgpu_device *adev);

Quoted 6 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h +2 -0
  int amdgpu_xgmi_set_pstate(struct amdgpu_device *adev, int pstate);
  int amdgpu_xgmi_get_hops_count(struct amdgpu_device *adev,
            struct amdgpu_device *peer_adev);
+int amdgpu_xgmi_get_num_links(struct amdgpu_device *adev,
+           struct amdgpu_device *peer_adev);
  uint64_t amdgpu_xgmi_get_relative_phy_addr(struct amdgpu_device

*adev,

drivers/gpu/drm/amd/amdkfd/kfd_crat.c static int kfd_fill_gpu_direct_io_link_to_cpu(int

Quoted 8 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h +0 -0
                                       uint64_t addr);
  static inline bool amdgpu_xgmi_same_hive(struct amdgpu_device *adev,
drivers/gpu/drm/amd/amdkfd/kfd_crat.c +0 -0
index c6b02aee4993..40ce6239c813 100644
@@ -1989,6 +1989,13 @@ static int kfd_fill_gpu_direct_io_link_to_cpu(int

*avail_size,

drivers/gpu/drm/amd/amdkfd/kfd_crat.c static int kfd_fill_gpu_direct_io_link_to_cpu(int · line 1989

Quoted 1 line
drivers/gpu/drm/amd/amdkfd/kfd_crat.c +0 -0
            sub_type_hdr->flags |=

CRAT_IOLINK_FLAGS_BI_DIRECTIONAL;

drivers/gpu/drm/amd/amdkfd/kfd_crat.c static int kfd_fill_gpu_direct_io_link_to_cpu(int · line 1990

Quoted 1 line
drivers/gpu/drm/amd/amdkfd/kfd_crat.c +0 -0
            sub_type_hdr->io_interface_type =

CRAT_IOLINK_TYPE_XGMI;

drivers/gpu/drm/amd/amdkfd/kfd_crat.c static int kfd_fill_gpu_direct_io_link_to_cpu(int *avail_size,

Quoted 4 lines
drivers/gpu/drm/amd/amdkfd/kfd_crat.c +3 -0
            sub_type_hdr->num_hops_xgmi = 1;
+           if (adev->asic_type == CHIP_ALDEBARAN) {
+                   sub_type_hdr->minimum_bandwidth_mbs =
+
      amdgpu_amdkfd_get_xgmi_bandwidth_mbytes(

drivers/gpu/drm/amd/amdkfd/kfd_crat.c static int kfd_fill_gpu_direct_io_link_to_cpu(int · line 1995

Quoted 1 line
drivers/gpu/drm/amd/amdkfd/kfd_crat.c +1 -0
+                                                   kdev->kgd, NULL,

true);

drivers/gpu/drm/amd/amdkfd/kfd_crat.c static int kfd_fill_gpu_direct_io_link_to_cpu(int *avail_size,

Quoted 6 lines
drivers/gpu/drm/amd/amdkfd/kfd_crat.c +3 -0
+                   sub_type_hdr->maximum_bandwidth_mbs =
+                                   sub_type_hdr-
 minimum_bandwidth_mbs;
+           }
    } else {
            sub_type_hdr->io_interface_type =

CRAT_IOLINK_TYPE_PCIEXPRESS;

drivers/gpu/drm/amd/amdkfd/kfd_crat.c static int kfd_fill_gpu_xgmi_link_to_gpu(int

Quoted 2 lines
    }
drivers/gpu/drm/amd/amdkfd/kfd_crat.c +0 -0
@@ -2033,6 +2040,11 @@ static int kfd_fill_gpu_xgmi_link_to_gpu(int

*avail_size,

drivers/gpu/drm/amd/amdkfd/kfd_crat.c static int kfd_fill_gpu_xgmi_link_to_gpu(int *avail_size,

Quoted 3 lines
drivers/gpu/drm/amd/amdkfd/kfd_crat.c +0 -0
    sub_type_hdr->proximity_domain_to = proximity_domain_to;
    sub_type_hdr->num_hops_xgmi =
            amdgpu_amdkfd_get_xgmi_hops_count(kdev->kgd,
Quoted 13 lines

peer_kdev->kgd);

drivers/gpu/drm/amd/amdkfd/kfd_crat.c static int kfd_fill_gpu_xgmi_link_to_gpu(int · line 2044

Quoted 2 lines
drivers/gpu/drm/amd/amdkfd/kfd_crat.c +2 -0
+   sub_type_hdr->maximum_bandwidth_mbs =
+           amdgpu_amdkfd_get_xgmi_bandwidth_mbytes(kdev->kgd,

peer_kdev->kgd, false);

Quoted 3 lines
drivers/gpu/drm/amd/amdkfd/kfd_crat.c +1 -0
+   sub_type_hdr->minimum_bandwidth_mbs = sub_type_hdr-

maximum_bandwidth_mbs ?
+ amdgpu_amdkfd_get_xgmi_bandwidth_mbytes(kdev->kgd,

NULL, true) : 0;

Quoted 5 lines

+

    return 0;

}

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

#6 lore raw

PATCH 3/3 drm/amdkfd: report pcie bandwidth to the kfd

In reply to Jonathan Kim

Similar to xGMI reporting the min/max bandwidth between direct peers, PCIe will report the min/max bandwidth to the KFD.

v2: change to bandwidth

---
 drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c | 61 ++++++++++++++++++++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h |  1 +
 drivers/gpu/drm/amd/amdkfd/kfd_crat.c      |  4 ++
 3 files changed, 66 insertions(+)
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +61 -0
index 3978578a1c49..b7db52f1a9d1 100644
@@ -21,6 +21,7 @@
  */
 
 #include "amdgpu_amdkfd.h"
+#include "amd_pcie.h"
 #include "amd_shared.h"
 
 #include "amdgpu.h"
@@ -576,6 +577,66 @@ int amdgpu_amdkfd_get_xgmi_bandwidth_mbytes(struct kgd_dev *dst, struct kgd_dev
 	return (num_links * 16 * 25000)/BITS_PER_BYTE;
 }
 
+int amdgpu_amdkfd_get_pcie_bandwidth_mbytes(struct kgd_dev *dev, bool is_min)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)dev;
+	int num_lanes_shift = is_min ? ffs(adev->pm.pcie_mlw_mask >>
+					CAIL_PCIE_LINK_WIDTH_SUPPORT_SHIFT) - 1 :
+				fls(adev->pm.pcie_mlw_mask >>
+					CAIL_PCIE_LINK_WIDTH_SUPPORT_SHIFT) - 1;
+	int gen_speed_shift = is_min ? ffs(adev->pm.pcie_gen_mask >>
+					CAIL_PCIE_LINK_SPEED_SUPPORT_SHIFT) - 1 :
+				fls(adev->pm.pcie_gen_mask >>
+					CAIL_PCIE_LINK_SPEED_SUPPORT_SHIFT) - 1;
+	uint32_t num_lanes_mask = (1UL << num_lanes_shift) << CAIL_PCIE_LINK_WIDTH_SUPPORT_SHIFT;
+	uint32_t gen_speed_mask = (1UL << gen_speed_shift) << CAIL_PCIE_LINK_SPEED_SUPPORT_SHIFT;
+	int num_lanes_factor = 0, gen_speed_mbits_factor = 0;
+
+	switch (num_lanes_mask) {
+	case CAIL_PCIE_LINK_WIDTH_SUPPORT_X1:
+		num_lanes_factor = 1;
+		break;
+	case CAIL_PCIE_LINK_WIDTH_SUPPORT_X2:
+		num_lanes_factor = 2;
+		break;
+	case CAIL_PCIE_LINK_WIDTH_SUPPORT_X4:
+		num_lanes_factor = 4;
+		break;
+	case CAIL_PCIE_LINK_WIDTH_SUPPORT_X8:
+		num_lanes_factor = 8;
+		break;
+	case CAIL_PCIE_LINK_WIDTH_SUPPORT_X12:
+		num_lanes_factor = 12;
+		break;
+	case CAIL_PCIE_LINK_WIDTH_SUPPORT_X16:
+		num_lanes_factor = 16;
+		break;
+	case CAIL_PCIE_LINK_WIDTH_SUPPORT_X32:
+		num_lanes_factor = 32;
+		break;
+	}
+
+	switch (gen_speed_mask) {
+	case CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1:
+		gen_speed_mbits_factor = 2500;
+		break;
+	case CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2:
+		gen_speed_mbits_factor = 5000;
+		break;
+	case CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3:
+		gen_speed_mbits_factor = 8000;
+		break;
+	case CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4:
+		gen_speed_mbits_factor = 16000;
+		break;
+	case CAIL_PCIE_LINK_SPEED_SUPPORT_GEN5:
+		gen_speed_mbits_factor = 32000;
+		break;
+	}
+
+	return (num_lanes_factor * gen_speed_mbits_factor)/BITS_PER_BYTE;
+}
+
 uint64_t amdgpu_amdkfd_get_mmio_remap_phys_addr(struct kgd_dev *kgd)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h +1 -0
index e12fccb2d2c4..5d73f3108d13 100644
@@ -227,6 +227,7 @@ uint32_t amdgpu_amdkfd_get_asic_rev_id(struct kgd_dev *kgd);
 int amdgpu_amdkfd_get_noretry(struct kgd_dev *kgd);
 uint8_t amdgpu_amdkfd_get_xgmi_hops_count(struct kgd_dev *dst, struct kgd_dev *src);
 int amdgpu_amdkfd_get_xgmi_bandwidth_mbytes(struct kgd_dev *dst, struct kgd_dev *src, bool is_min);
+int amdgpu_amdkfd_get_pcie_bandwidth_mbytes(struct kgd_dev *dev, bool is_min);
 
 /* Read user wptr from a specified user address space with page fault
  * disabled. The memory must be pinned and mapped to the hardware when
drivers/gpu/drm/amd/amdkfd/kfd_crat.c +4 -0
index 40ce6239c813..eada22b9ea69 100644
@@ -1998,6 +1998,10 @@ static int kfd_fill_gpu_direct_io_link_to_cpu(int *avail_size,
 		}
 	} else {
 		sub_type_hdr->io_interface_type = CRAT_IOLINK_TYPE_PCIEXPRESS;
+		sub_type_hdr->minimum_bandwidth_mbs =
+				amdgpu_amdkfd_get_pcie_bandwidth_mbytes(kdev->kgd, true);
+		sub_type_hdr->maximum_bandwidth_mbs =
+				amdgpu_amdkfd_get_pcie_bandwidth_mbytes(kdev->kgd, false);
 	}
 
 	sub_type_hdr->proximity_domain_from = proximity_domain;
Signature · 6 lines
2.25.1

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
maintainer Reviewed-by on v1 3/3
#7 lore raw

Re: [PATCH 3/3] drm/amdkfd: report pcie bandwidth to the kfd

Am 2021-07-16 um 12:43 p.m. schrieb Jonathan Kim:

Quoted 29 lines

Similar to xGMI reporting the min/max bandwidth between direct peers, PCIe will report the min/max bandwidth to the KFD.

v2: change to bandwidth

---
 drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c | 61 ++++++++++++++++++++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h |  1 +
 drivers/gpu/drm/amd/amdkfd/kfd_crat.c      |  4 ++
 3 files changed, 66 insertions(+)
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +2 -0
index 3978578a1c49..b7db52f1a9d1 100644
@@ -21,6 +21,7 @@
  */
 
 #include "amdgpu_amdkfd.h"
+#include "amd_pcie.h"
 #include "amd_shared.h"
 
 #include "amdgpu.h"
@@ -576,6 +577,66 @@ int amdgpu_amdkfd_get_xgmi_bandwidth_mbytes(struct kgd_dev *dst, struct kgd_dev
 	return (num_links * 16 * 25000)/BITS_PER_BYTE;
 }
 
+int amdgpu_amdkfd_get_pcie_bandwidth_mbytes(struct kgd_dev *dev, bool is_min)

drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c int amdgpu_amdkfd_get_xgmi_bandwidth_mbytes(struct kgd_dev *dst, struct kgd_dev · line 590

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +10 -0
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)dev;
+	int num_lanes_shift = is_min ? ffs(adev->pm.pcie_mlw_mask >>
+					CAIL_PCIE_LINK_WIDTH_SUPPORT_SHIFT) - 1 :
+				fls(adev->pm.pcie_mlw_mask >>
+					CAIL_PCIE_LINK_WIDTH_SUPPORT_SHIFT) - 1;
+	int gen_speed_shift = is_min ? ffs(adev->pm.pcie_gen_mask >>
+					CAIL_PCIE_LINK_SPEED_SUPPORT_SHIFT) - 1 :
+				fls(adev->pm.pcie_gen_mask >>
+					CAIL_PCIE_LINK_SPEED_SUPPORT_SHIFT) - 1;

The shifting is not necessary because you undo it below. I think this would do the trick and be more readable:

	int num_lanes_shift = (is_min ? ffs(adev->pm.pcie_mlw_mask) :
					fls(adev->pm.pcie_mlw_mask)) - 1;
	int gen_speed_shift = (is_min ? ffs(adev->pm.pcie_gen_mask) :
					fls(adev->pm.pcie_gen_mask)) - 1;
	uint32_t num_lanes_mask = 1 << num_lanes_shift;
	uint32_t gen_speed_mask = 1 << gen_speed_shift;

With that fixed, this patch is

Quoted 69 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +49 -0
+	uint32_t num_lanes_mask = (1UL << num_lanes_shift) << CAIL_PCIE_LINK_WIDTH_SUPPORT_SHIFT;
+	uint32_t gen_speed_mask = (1UL << gen_speed_shift) << CAIL_PCIE_LINK_SPEED_SUPPORT_SHIFT;
+	int num_lanes_factor = 0, gen_speed_mbits_factor = 0;
+
+	switch (num_lanes_mask) {
+	case CAIL_PCIE_LINK_WIDTH_SUPPORT_X1:
+		num_lanes_factor = 1;
+		break;
+	case CAIL_PCIE_LINK_WIDTH_SUPPORT_X2:
+		num_lanes_factor = 2;
+		break;
+	case CAIL_PCIE_LINK_WIDTH_SUPPORT_X4:
+		num_lanes_factor = 4;
+		break;
+	case CAIL_PCIE_LINK_WIDTH_SUPPORT_X8:
+		num_lanes_factor = 8;
+		break;
+	case CAIL_PCIE_LINK_WIDTH_SUPPORT_X12:
+		num_lanes_factor = 12;
+		break;
+	case CAIL_PCIE_LINK_WIDTH_SUPPORT_X16:
+		num_lanes_factor = 16;
+		break;
+	case CAIL_PCIE_LINK_WIDTH_SUPPORT_X32:
+		num_lanes_factor = 32;
+		break;
+	}
+
+	switch (gen_speed_mask) {
+	case CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1:
+		gen_speed_mbits_factor = 2500;
+		break;
+	case CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2:
+		gen_speed_mbits_factor = 5000;
+		break;
+	case CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3:
+		gen_speed_mbits_factor = 8000;
+		break;
+	case CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4:
+		gen_speed_mbits_factor = 16000;
+		break;
+	case CAIL_PCIE_LINK_SPEED_SUPPORT_GEN5:
+		gen_speed_mbits_factor = 32000;
+		break;
+	}
+
+	return (num_lanes_factor * gen_speed_mbits_factor)/BITS_PER_BYTE;
+}
+
 uint64_t amdgpu_amdkfd_get_mmio_remap_phys_addr(struct kgd_dev *kgd)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h +1 -0
index e12fccb2d2c4..5d73f3108d13 100644
@@ -227,6 +227,7 @@ uint32_t amdgpu_amdkfd_get_asic_rev_id(struct kgd_dev *kgd);
 int amdgpu_amdkfd_get_noretry(struct kgd_dev *kgd);
 uint8_t amdgpu_amdkfd_get_xgmi_hops_count(struct kgd_dev *dst, struct kgd_dev *src);
 int amdgpu_amdkfd_get_xgmi_bandwidth_mbytes(struct kgd_dev *dst, struct kgd_dev *src, bool is_min);
+int amdgpu_amdkfd_get_pcie_bandwidth_mbytes(struct kgd_dev *dev, bool is_min);
 
 /* Read user wptr from a specified user address space with page fault
  * disabled. The memory must be pinned and mapped to the hardware when
drivers/gpu/drm/amd/amdkfd/kfd_crat.c +0 -0
index 40ce6239c813..eada22b9ea69 100644

drivers/gpu/drm/amd/amdkfd/kfd_crat.c static int kfd_fill_gpu_direct_io_link_to_cpu(int *avail_size, · line 2007

Quoted 10 lines
drivers/gpu/drm/amd/amdkfd/kfd_crat.c +4 -0
@@ -1998,6 +1998,10 @@ static int kfd_fill_gpu_direct_io_link_to_cpu(int *avail_size,
 		}
 	} else {
 		sub_type_hdr->io_interface_type = CRAT_IOLINK_TYPE_PCIEXPRESS;
+		sub_type_hdr->minimum_bandwidth_mbs =
+				amdgpu_amdkfd_get_pcie_bandwidth_mbytes(kdev->kgd, true);
+		sub_type_hdr->maximum_bandwidth_mbs =
+				amdgpu_amdkfd_get_pcie_bandwidth_mbytes(kdev->kgd, false);
 	}
 
 	sub_type_hdr->proximity_domain_from = proximity_domain;

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

#8 lore raw

RE: [PATCH 3/3] drm/amdkfd: report pcie bandwidth to the kfd

[AMD Official Use Only]

Quoted 56 lines

-----Original Message-----
From: Kuehling, Felix <[email protected]>
Sent: Saturday, July 17, 2021 1:37 AM
To: Kim, Jonathan <[email protected]>; amd- [email protected]
Subject: Re: [PATCH 3/3] drm/amdkfd: report pcie bandwidth to the kfd

Am 2021-07-16 um 12:43 p.m. schrieb Jonathan Kim:

Quoted 9 lines

Similar to xGMI reporting the min/max bandwidth between direct peers, PCIe will report the min/max bandwidth to the KFD.

v2: change to bandwidth

---
 drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c | 61

++++++++++++++++++++++

drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h | 1 +

drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c int

Quoted 17 lines
 drivers/gpu/drm/amd/amdkfd/kfd_crat.c      |  4 ++
 3 files changed, 66 insertions(+)
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +1 -0
index 3978578a1c49..b7db52f1a9d1 100644
@@ -21,6 +21,7 @@
  */
 
 #include "amdgpu_amdkfd.h"
+#include "amd_pcie.h"
 #include "amd_shared.h"
 
 #include "amdgpu.h"
@@ -576,6 +577,66 @@ int

amdgpu_amdkfd_get_xgmi_bandwidth_mbytes(struct kgd_dev *dst, struct kgd_dev

drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c int · line 579

Quoted 3 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +1 -0
    return (num_links * 16 * 25000)/BITS_PER_BYTE;  }
 
+int amdgpu_amdkfd_get_pcie_bandwidth_mbytes(struct kgd_dev *dev,

bool

drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c int amdgpu_amdkfd_get_xgmi_bandwidth_mbytes(struct kgd_dev *dst, struct kgd_dev · line 594

Quoted 4 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +4 -0
+is_min) {
+   struct amdgpu_device *adev = (struct amdgpu_device *)dev;
+   int num_lanes_shift = is_min ? ffs(adev->pm.pcie_mlw_mask >>
+

CAIL_PCIE_LINK_WIDTH_SUPPORT_SHIFT) - 1 :

drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c int · line 585

Quoted 2 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +2 -0
+                           fls(adev->pm.pcie_mlw_mask >>
+
      CAIL_PCIE_LINK_WIDTH_SUPPORT_SHIFT) - 1;

drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c int · line 587

Quoted 2 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +2 -0
+   int gen_speed_shift = is_min ? ffs(adev->pm.pcie_gen_mask >>
+

CAIL_PCIE_LINK_SPEED_SUPPORT_SHIFT) - 1 :

drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c int · line 589

Quoted 2 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +2 -0
+                           fls(adev->pm.pcie_gen_mask >>
+
      CAIL_PCIE_LINK_SPEED_SUPPORT_SHIFT) - 1;
Quoted 7 lines

The shifting is not necessary because you undo it below. I think this would do the trick and be more readable:

      int num_lanes_shift = (is_min ? ffs(adev->pm.pcie_mlw_mask) :
                                      fls(adev->pm.pcie_mlw_mask)) - 1;
      int gen_speed_shift = (is_min ? ffs(adev->pm.pcie_gen_mask) :
                                      fls(adev->pm.pcie_gen_mask)) - 1;

Ok thanks for the review and suggestion. I've adjusted your suggestion by masking pcie_gen_mask with CAIL_PCIE_LINK_SPEED_SUPPORT_MASK as the mask sets some non-speed related lower bits.

Thanks,

Jon

Quoted 90 lines
      uint32_t num_lanes_mask = 1 << num_lanes_shift;
      uint32_t gen_speed_mask = 1 << gen_speed_shift;

With that fixed, this patch is

drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c int · line 590

Quoted 1 line
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +1 -0
+   uint32_t num_lanes_mask = (1UL << num_lanes_shift) <<

CAIL_PCIE_LINK_WIDTH_SUPPORT_SHIFT;

drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c int · line 591

Quoted 1 line
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +1 -0
+   uint32_t gen_speed_mask = (1UL << gen_speed_shift) <<

CAIL_PCIE_LINK_SPEED_SUPPORT_SHIFT;

Quoted 35 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +35 -0
+   int num_lanes_factor = 0, gen_speed_mbits_factor = 0;
+
+   switch (num_lanes_mask) {
+   case CAIL_PCIE_LINK_WIDTH_SUPPORT_X1:
+           num_lanes_factor = 1;
+           break;
+   case CAIL_PCIE_LINK_WIDTH_SUPPORT_X2:
+           num_lanes_factor = 2;
+           break;
+   case CAIL_PCIE_LINK_WIDTH_SUPPORT_X4:
+           num_lanes_factor = 4;
+           break;
+   case CAIL_PCIE_LINK_WIDTH_SUPPORT_X8:
+           num_lanes_factor = 8;
+           break;
+   case CAIL_PCIE_LINK_WIDTH_SUPPORT_X12:
+           num_lanes_factor = 12;
+           break;
+   case CAIL_PCIE_LINK_WIDTH_SUPPORT_X16:
+           num_lanes_factor = 16;
+           break;
+   case CAIL_PCIE_LINK_WIDTH_SUPPORT_X32:
+           num_lanes_factor = 32;
+           break;
+   }
+
+   switch (gen_speed_mask) {
+   case CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1:
+           gen_speed_mbits_factor = 2500;
+           break;
+   case CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2:
+           gen_speed_mbits_factor = 5000;
+           break;
+   case CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3:
+           gen_speed_mbits_factor = 8000;

drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c int amdgpu_amdkfd_get_xgmi_bandwidth_mbytes(struct kgd_dev *dst, struct kgd_dev

Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +10 -0
+           break;
+   case CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4:
+           gen_speed_mbits_factor = 16000;
+           break;
+   case CAIL_PCIE_LINK_SPEED_SUPPORT_GEN5:
+           gen_speed_mbits_factor = 32000;
+           break;
+   }
+
+   return (num_lanes_factor *

gen_speed_mbits_factor)/BITS_PER_BYTE;

drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c int amdgpu_amdkfd_get_xgmi_bandwidth_mbytes(struct kgd_dev *dst, struct kgd_dev

Quoted 3 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +2 -0
+}
+
 uint64_t amdgpu_amdkfd_get_mmio_remap_phys_addr(struct kgd_dev

*kgd)

Quoted 9 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +0 -0
 {
    struct amdgpu_device *adev = (struct amdgpu_device *)kgd; diff --git

a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
index e12fccb2d2c4..5d73f3108d13 100644

drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h +0 -0
@@ -227,6 +227,7 @@ uint32_t amdgpu_amdkfd_get_asic_rev_id(struct
 kgd_dev *kgd);  int amdgpu_amdkfd_get_noretry(struct kgd_dev *kgd);
Quoted 4 lines

uint8_t amdgpu_amdkfd_get_xgmi_hops_count(struct kgd_dev *dst, struct kgd_dev *src); int amdgpu_amdkfd_get_xgmi_bandwidth_mbytes(struct kgd_dev *dst, struct kgd_dev *src, bool is_min);
+int amdgpu_amdkfd_get_pcie_bandwidth_mbytes(struct kgd_dev *dev,

bool

drivers/gpu/drm/amd/amdkfd/kfd_crat.c static int kfd_fill_gpu_direct_io_link_to_cpu(int

Quoted 10 lines

+is_min);

/* Read user wptr from a specified user address space with page fault
* disabled. The memory must be pinned and mapped to the hardware
when diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
index 40ce6239c813..eada22b9ea69 100644

drivers/gpu/drm/amd/amdkfd/kfd_crat.c +0 -0
@@ -1998,6 +1998,10 @@ static int kfd_fill_gpu_direct_io_link_to_cpu(int

*avail_size,

drivers/gpu/drm/amd/amdkfd/kfd_crat.c static int kfd_fill_gpu_direct_io_link_to_cpu(int *avail_size,

Quoted 3 lines
drivers/gpu/drm/amd/amdkfd/kfd_crat.c +0 -0
            }
    } else {
            sub_type_hdr->io_interface_type =
Quoted 10 lines

CRAT_IOLINK_TYPE_PCIEXPRESS;

drivers/gpu/drm/amd/amdkfd/kfd_crat.c static int kfd_fill_gpu_direct_io_link_to_cpu(int · line 2002

Quoted 2 lines
drivers/gpu/drm/amd/amdkfd/kfd_crat.c +2 -0
+           sub_type_hdr->minimum_bandwidth_mbs =
+
      amdgpu_amdkfd_get_pcie_bandwidth_mbytes(kdev->kgd, true);

drivers/gpu/drm/amd/amdkfd/kfd_crat.c static int kfd_fill_gpu_direct_io_link_to_cpu(int · line 2004

Quoted 2 lines
drivers/gpu/drm/amd/amdkfd/kfd_crat.c +2 -0
+           sub_type_hdr->maximum_bandwidth_mbs =
+
      amdgpu_amdkfd_get_pcie_bandwidth_mbytes(kdev->kgd, false);
Quoted 3 lines
drivers/gpu/drm/amd/amdkfd/kfd_crat.c +0 -0
    }
 
    sub_type_hdr->proximity_domain_from = proximity_domain;

_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx