drm/amdgpu: support reserve bad page for virt
Subsystems (2)
PATCH 1/1 drm/amdgpu: support reserve bad page for virt
--- drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 7 +- drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c | 164 +++++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h | 30 +++- 3 files changed, 196 insertions(+), 5 deletions(-) mode change 100644 => 100755 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c mode change 100644 => 100755 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +6 -1
@@ -2001,8 +2001,10 @@ static int amdgpu_device_ip_init(struct amdgpu_device *adev) } } -if (amdgpu_sriov_vf (adev)) +if (amdgpu_sriov_vf (adev)) { +amdgpu_virt_init_err_handler_data (adev);amdgpu_virt_init_data_exchange (adev); + } r =amdgpu_ib_pool_init (adev);if (r) {
@@ -2306,6 +2308,9 @@ static int amdgpu_device_ip_fini(struct amdgpu_device *adev) {int i, r; +if (amdgpu_sriov_vf (adev)) +amdgpu_release_virt_err_handler_data (adev); +amdgpu_ras_pre_fini (adev);if (adev->gmc.xgmi.num_physical_nodes >1 )
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +164 -0
188 lines not on this page · raw message
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +26 -4
@@ -143,19 +143,27 @@ struct amdgim_pf2vf_info_v2 {uint32_t vce_enc_max_pixels_count;/* 16x16 pixels/sec, codec independent */ uint32_t vce_enc_max_bandwidth; +/* Bad pages block position in BYTE */ +uint32_t bp_block_offset_L; +uint32_t bp_block_offset_H; +/* Bad pages block size in BYTE */ +uint32_t bp_block_size;/* MEC FW position in kb from the start of VF visible frame buffer */ -uint64_t mecfw_kboffset; +uint32_t mecfw_kboffset_L; +uint32_t mecfw_kboffset_H;/* MEC FW size in KB */ uint32_t mecfw_ksize;/* UVD FW position in kb from the start of VF visible frame buffer */ -uint64_t uvdfw_kboffset; +uint32_t uvdfw_kboffset_L; +uint32_t uvdfw_kboffset_H;/* UVD FW size in KB */ uint32_t uvdfw_ksize;/* VCE FW position in kb from the start of VF visible frame buffer */ -uint64_t vcefw_kboffset; +uint32_t vcefw_kboffset_L; +uint32_t vcefw_kboffset_H;/* VCE FW size in KB */ uint32_t vcefw_ksize; -uint32_t reserved[AMDGIM_GET_STRUCTURE_RESERVED_SIZE (256 ,0 ,0 , (9 +sizeof (struct amd_sriov_msg_pf2vf_info_header )/sizeof (uint32_t )),3 )]; +uint32_t reserved[AMDGIM_GET_STRUCTURE_RESERVED_SIZE (256 ,0 ,0 , (18 +sizeof (struct amd_sriov_msg_pf2vf_info_header )/sizeof (uint32_t )),0 )]; }__aligned(4) ;
@@ -254,6 +262,17 @@ typedef struct amdgim_vf2pf_info_v2 amdgim_vf2pf_info ; } \ }while (0 ) +struct virt_ras_err_handler_data { +/* point to bad page records array */ +struct eeprom_table_record *bps; +/* point to reserved bo array */ +struct amdgpu_bo **bps_bo; +/* the count of entries */ +int count; +/* last reserved entry's index + 1 */ +int last_reserved; +}; +/* GPU virtualization */ struct amdgpu_virt {uint32_t caps;
@@ -272,6 +291,7 @@ struct amdgpu_virt {uint32_t reg_access_mode;int req_init_data_ver;bool tdr_debug; +struct virt_ras_err_handler_data *virt_eh_data; };#define amdgpu_sriov_enabled (adev) \
@@ -323,6 +343,8 @@ void amdgpu_virt_free_mm_table(struct amdgpu_device *adev);int amdgpu_virt_fw_reserve_get_checksum (void *obj,unsigned long obj_size,unsigned int key,unsigned int chksum); +void amdgpu_release_virt_err_handler_data (struct amdgpu_device *adev); +int amdgpu_virt_init_err_handler_data (struct amdgpu_device *adev);void amdgpu_virt_init_data_exchange (struct amdgpu_device *adev);void amdgpu_detect_virtualization (struct amdgpu_device *adev);
Signature · 6 lines
2.25.1 _______________________________________________ amd-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/amd-gfx
[AMD Public Use]
Please check my comments inline.
Regards,
Hawking
-----Original Message-----
From: Stanley.Yang <[email protected]>
Sent: Wednesday, June 3, 2020 22:10
To: [email protected]
Cc: Zhang, Hawking <[email protected]>; Chen, Guchun <[email protected]>; Liu, Monk <[email protected]>; Clements, John <[email protected]>; Zhou1, Tao <[email protected]>; Li, Dennis <[email protected]>; Yang, Stanley <[email protected]>
Subject: [PATCH] drm/amdgpu: support reserve bad page for virt
--- drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 7 +- drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c | 164 +++++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h | 30 +++- 3 files changed, 196 insertions(+), 5 deletions(-) mode change 100644 => 100755 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c mode change 100644 => 100755 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +2 -1
@@ -2001,8 +2001,10 @@ static int amdgpu_device_ip_init(struct amdgpu_device *adev) } } -if (amdgpu_sriov_vf (adev)) +if (amdgpu_sriov_vf (adev)) { +amdgpu_virt_init_err_handler_data (adev);
[Hawking]:
1. It might be better to rename the function to amdgpu_virt_init_ras_err_handler_data
2. we shall apply either asic type check or more general data structure check before call into amdgpu_virt_init_ras_err_handler_data. Please check my comments in amdgpu_virt_init_ras_err_handler_data function
amdgpu_virt_init_data_exchange(adev);
+ }
r = amdgpu_ib_pool_init(adev);
if (r) {
@@ -2306,6 +2308,9 @@ static int amdgpu_device_ip_fini(struct amdgpu_device *adev) {
int i, r;
+ if (amdgpu_sriov_vf(adev))
+ amdgpu_release_virt_err_handler_data(adev);
[Hawking]:
1. It might be better to rename the function to amdgpu_virt_release_ras_err_handler_data
2. we shall apply either asic type check or more general data structure check In case it introduce regression on MI100 and V340L
+
amdgpu_ras_pre_fini(adev);
if (adev->gmc.xgmi.num_physical_nodes > 1) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
old mode 100644
new mode 100755
index f3b38c9e04ca..c1554562a2ce
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +1 -0
@@ -26,6 +26,7 @@#include <drm/drm_drv.h> #include "amdgpu.h" +#include "amdgpu_ras.h" bool amdgpu_virt_mmio_blocked(struct amdgpu_device *adev) { @@ -255,12 +256,164 @@ int amdgpu_virt_fw_reserve_get_checksum(void *obj, return ret;
}
+int amdgpu_virt_init_err_handler_data(struct amdgpu_device *adev) {
[Hawking]:
1. same as above, rename it to amdgpu_virt_init_ras_err_handler_data
+ struct amdgpu_virt *virt = &adev->virt;
+ struct virt_ras_err_handler_data **data = &virt->virt_eh_data;
+ /* GPU will be marked bad on host if bp count more then 10,
+ * so alloc 512 is enough.
+ */
+ unsigned int align_space = 512;
+ void *bps = NULL;
+ struct amdgpu_bo **bps_bo = NULL;
+
[Hawking]:
1. This will result to regression on MI100 SRIOV if we don't apply check. Or leverage general data structure for the purpose, thoughts?
+ *data = kmalloc(sizeof(struct virt_ras_err_handler_data), GFP_KERNEL);
[Hawking]:
1. rename the data structure to amdgpu_virt_ras_err_handler_data
+ if (!*data)
+ return -ENOMEM;
+
+ bps = kmalloc(align_space * sizeof((*data)->bps), GFP_KERNEL);
+ bps_bo = kmalloc(align_space * sizeof((*data)->bps_bo), GFP_KERNEL);
+
+ if (!bps || !bps_bo) {
+ kfree(bps);
+ kfree(bps_bo);
+ return -ENOMEM;
+ }
+
+ (*data)->bps = bps;
+ (*data)->bps_bo = bps_bo;
+ (*data)->count = 0;
+ (*data)->last_reserved = 0;
+ return 0;
+}
+
+static void amdgpu_virt_release_bp(struct amdgpu_device *adev) {
+ struct amdgpu_virt *virt = &adev->virt;
+ struct virt_ras_err_handler_data *data = virt->virt_eh_data;
[Hawking]:
1. same as above, please rename the virt_ras_err_handler_data to amdgpu_virt_ras_err_handler_data
+ struct amdgpu_bo *bo;
+ int i;
+
+ if (!data)
+ return;
+
+ for (i = data->last_reserved - 1; i >= 0; i--) {
+ bo = data->bps_bo[i];
+ amdgpu_bo_free_kernel(&bo, NULL, NULL);
+ data->bps_bo[i] = bo;
+ data->last_reserved = i;
+ }
+}
+
+void amdgpu_release_virt_err_handler_data(struct amdgpu_device *adev) {
[Hawking]:
1. same as above, rename the function to amdgpu_virt_release_ras_err_handler_data
+ struct amdgpu_virt *virt = &adev->virt;
+ struct virt_ras_err_handler_data *data = virt->virt_eh_data;
+
+ if (!data)
+ return;
+
+ amdgpu_virt_release_bp(adev);
+
+ kfree(data->bps);
+ kfree(data->bps_bo);
+ kfree(data);
+ virt->virt_eh_data = NULL;
+}
+
+static void amdgpu_virt_ras_add_bps(struct amdgpu_device *adev,
+ struct eeprom_table_record *bps, int pages) {
+ struct amdgpu_virt *virt = &adev->virt;
+ struct virt_ras_err_handler_data *data = virt->virt_eh_data;
+
+ if (!data)
+ return;
+
+ memcpy(&data->bps[data->count], bps, pages * sizeof(*data->bps));
+ data->count += pages;
+}
+
+static void amdgpu_virt_ras_reserve_bps(struct amdgpu_device *adev) {
+ struct amdgpu_virt *virt = &adev->virt;
+ struct virt_ras_err_handler_data *data = virt->virt_eh_data;
+ struct amdgpu_bo *bo = NULL;
+ uint64_t bp;
+ int i;
+
+ if (!data)
+ return;
+
+ for (i = data->last_reserved; i < data->count; i++) {
+ bp = data->bps[i].retired_page;
+
+ /* There are two cases of reserve error should be ignored:
+ * 1) a ras bad page has been allocated (used by someone);
+ * 2) a ras bad page has been reserved (duplicate error injection
+ * for one page);
+ */
+ if (amdgpu_bo_create_kernel_at(adev, bp << AMDGPU_GPU_PAGE_SHIFT,
+ AMDGPU_GPU_PAGE_SIZE, + AMDGPU_GEM_DOMAIN_VRAM, + &bo, NULL))
+ DRM_WARN("RAS WARN: reserve vram for retired page %llx fail\n", bp);
[Hawking]:
1. can we make this as debug message?
+
+ data->bps_bo[i] = bo;
+ data->last_reserved = i + 1;
+ bo = NULL;
+ }
+}
+
+static bool amdgpu_virt_check_bad_page(struct amdgpu_device *adev,
+ uint64_t retired_page)
+{
+ struct amdgpu_virt *virt = &adev->virt;
+ struct virt_ras_err_handler_data *data = virt->virt_eh_data;
+ int i;
+
+ if (!data)
+ return true;
+
+ for (i = 0; i < data->count; i++)
+ if (retired_page == data->bps[i].retired_page)
+ return true;
+
+ return false;
+}
+
+static void amdgpu_virt_add_bad_page(struct amdgpu_device *adev,
+ uint64_t bp_block_offset, uint32_t bp_block_size) {
+ struct eeprom_table_record bp;
+ uint64_t retired_page;
+ uint32_t bp_idx, bp_cnt;
+
+ if (bp_block_size) {
+ bp_cnt = bp_block_size / sizeof(uint64_t);
+ for (bp_idx = 0; bp_idx < bp_cnt; bp_idx++) {
+ retired_page = *(uint64_t *)(adev->fw_vram_usage.va +
+ bp_block_offset + bp_idx * sizeof(uint64_t));
+ bp.retired_page = retired_page;
+
+ if (amdgpu_virt_check_bad_page(adev, retired_page))
+ continue;
+
+ amdgpu_virt_ras_add_bps(adev, &bp, 1);
+
+ amdgpu_virt_ras_reserve_bps(adev);
+ }
+ }
+}
+
void amdgpu_virt_init_data_exchange(struct amdgpu_device *adev) {
uint32_t pf2vf_size = 0;
uint32_t checksum = 0;
uint32_t checkval;
char *str;
+ uint64_t bp_block_offset = 0;
+ uint32_t bp_block_size = 0;
adev->virt.fw_reserve.p_pf2vf = NULL; adev->virt.fw_reserve.p_vf2pf = NULL;
@@ -275,6 +428,17 @@ void amdgpu_virt_init_data_exchange(struct amdgpu_device *adev)
/* pf2vf message must be in 4K */
if (pf2vf_size > 0 && pf2vf_size < 4096) {
+ if (adev->virt.fw_reserve.p_pf2vf->version == 2) {
+ struct amdgim_pf2vf_info_v2 *pf2vf_v2 = NULL;
+
+ pf2vf_v2 = (struct amdgim_pf2vf_info_v2 *)adev->virt.fw_reserve.p_pf2vf;
+ bp_block_offset = ((uint64_t)pf2vf_v2->bp_block_offset_L & 0xFFFFFFFF) |
+ ((((uint64_t)pf2vf_v2->bp_block_offset_H) << 32) & 0xFFFFFFFF00000000);
+ bp_block_size = pf2vf_v2->bp_block_size;
+
+ amdgpu_virt_add_bad_page(adev, bp_block_offset, bp_block_size);
+ }
+
checkval = amdgpu_virt_fw_reserve_get_checksum(
adev->virt.fw_reserve.p_pf2vf, pf2vf_size,
adev->virt.fw_reserve.checksum_key, checksum); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
old mode 100644
new mode 100755
index b90e822cebd7..96d84f036e96
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +7 -1
@@ -143,19 +143,27 @@ struct amdgim_pf2vf_info_v2 {uint32_t vce_enc_max_pixels_count;/* 16x16 pixels/sec, codec independent */ uint32_t vce_enc_max_bandwidth; +/* Bad pages block position in BYTE */ +uint32_t bp_block_offset_L; +uint32_t bp_block_offset_H; +/* Bad pages block size in BYTE */ +uint32_t bp_block_size;/* MEC FW position in kb from the start of VF visible frame buffer */ -uint64_t mecfw_kboffset; +uint32_t mecfw_kboffset_L; +uint32_t mecfw_kboffset_H;
[Hawking]:
1. can you just explain the reason that split the position in lower 32bit and higher 32 bit respectively?
/* MEC FW size in KB */
uint32_t mecfw_ksize;
/* UVD FW position in kb from the start of VF visible frame buffer */
- uint64_t uvdfw_kboffset;
+ uint32_t uvdfw_kboffset_L;
+ uint32_t uvdfw_kboffset_H;
/* UVD FW size in KB */
uint32_t uvdfw_ksize;
/* VCE FW position in kb from the start of VF visible frame buffer */
- uint64_t vcefw_kboffset;
+ uint32_t vcefw_kboffset_L;
+ uint32_t vcefw_kboffset_H;
/* VCE FW size in KB */
uint32_t vcefw_ksize;
- uint32_t reserved[AMDGIM_GET_STRUCTURE_RESERVED_SIZE(256, 0, 0, (9 + sizeof(struct amd_sriov_msg_pf2vf_info_header)/sizeof(uint32_t)), 3)];
+ uint32_t reserved[AMDGIM_GET_STRUCTURE_RESERVED_SIZE(256, 0, 0, (18 +
+sizeof(struct amd_sriov_msg_pf2vf_info_header)/sizeof(uint32_t)), 0)];
} __aligned(4);
@@ -254,6 +262,17 @@ typedef struct amdgim_vf2pf_info_v2 amdgim_vf2pf_info ;
} \
} while (0)
+struct virt_ras_err_handler_data {
+ /* point to bad page records array */
+ struct eeprom_table_record *bps;
+ /* point to reserved bo array */
+ struct amdgpu_bo **bps_bo;
+ /* the count of entries */
+ int count;
+ /* last reserved entry's index + 1 */
+ int last_reserved;
+};
+
/* GPU virtualization */
struct amdgpu_virt {
uint32_t caps;
@@ -272,6 +291,7 @@ struct amdgpu_virt {
uint32_t reg_access_mode;
int req_init_data_ver;
bool tdr_debug;
+ struct virt_ras_err_handler_data *virt_eh_data;
};
#define amdgpu_sriov_enabled(adev) \
@@ -323,6 +343,8 @@ void amdgpu_virt_free_mm_table(struct amdgpu_device *adev); int amdgpu_virt_fw_reserve_get_checksum(void *obj, unsigned long obj_size,
unsigned int key,
unsigned int chksum);
+void amdgpu_release_virt_err_handler_data(struct amdgpu_device *adev);
+int amdgpu_virt_init_err_handler_data(struct amdgpu_device *adev);
void amdgpu_virt_init_data_exchange(struct amdgpu_device *adev); void amdgpu_detect_virtualization(struct amdgpu_device *adev);
--
2.25.1
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
[AMD Public Use]
Except Hawking's comments, we shall not change file mode in the patch as well.
mode change 100644 => 100755 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c mode change 100644 => 100755 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
Regards,
Guchun
-----Original Message-----
From: Zhang, Hawking <[email protected]>
Sent: Wednesday, June 3, 2020 11:26 PM
To: Yang, Stanley <[email protected]>; [email protected]
Cc: Chen, Guchun <[email protected]>; Liu, Monk <[email protected]>; Clements, John <[email protected]>; Zhou1, Tao <[email protected]>; Li, Dennis <[email protected]>; Yang, Stanley <[email protected]>
Subject: RE: [PATCH] drm/amdgpu: support reserve bad page for virt
[AMD Public Use]
Please check my comments inline.
Regards,
Hawking
-----Original Message-----
From: Stanley.Yang <[email protected]>
Sent: Wednesday, June 3, 2020 22:10
To: [email protected]
Cc: Zhang, Hawking <[email protected]>; Chen, Guchun <[email protected]>; Liu, Monk <[email protected]>; Clements, John <[email protected]>; Zhou1, Tao <[email protected]>; Li, Dennis <[email protected]>; Yang, Stanley <[email protected]>
Subject: [PATCH] drm/amdgpu: support reserve bad page for virt
--- drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 7 +- drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c | 164 +++++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h | 30 +++- 3 files changed, 196 insertions(+), 5 deletions(-) mode change 100644 => 100755 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c mode change 100644 => 100755 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +2 -1
@@ -2001,8 +2001,10 @@ static int amdgpu_device_ip_init(struct amdgpu_device *adev) } } -if (amdgpu_sriov_vf (adev)) +if (amdgpu_sriov_vf (adev)) { +amdgpu_virt_init_err_handler_data (adev);
[Hawking]:
1. It might be better to rename the function to amdgpu_virt_init_ras_err_handler_data
2. we shall apply either asic type check or more general data structure check before call into amdgpu_virt_init_ras_err_handler_data. Please check my comments in amdgpu_virt_init_ras_err_handler_data function
amdgpu_virt_init_data_exchange(adev);
+ }
r = amdgpu_ib_pool_init(adev);
if (r) {
@@ -2306,6 +2308,9 @@ static int amdgpu_device_ip_fini(struct amdgpu_device *adev) {
int i, r;
+ if (amdgpu_sriov_vf(adev))
+ amdgpu_release_virt_err_handler_data(adev);
[Hawking]:
1. It might be better to rename the function to amdgpu_virt_release_ras_err_handler_data
2. we shall apply either asic type check or more general data structure check In case it introduce regression on MI100 and V340L
+
amdgpu_ras_pre_fini(adev);
if (adev->gmc.xgmi.num_physical_nodes > 1) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
old mode 100644
new mode 100755
index f3b38c9e04ca..c1554562a2ce
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +1 -0
@@ -26,6 +26,7 @@#include <drm/drm_drv.h> #include "amdgpu.h" +#include "amdgpu_ras.h" bool amdgpu_virt_mmio_blocked(struct amdgpu_device *adev) { @@ -255,12 +256,164 @@ int amdgpu_virt_fw_reserve_get_checksum(void *obj, return ret;
}
+int amdgpu_virt_init_err_handler_data(struct amdgpu_device *adev) {
[Hawking]:
1. same as above, rename it to amdgpu_virt_init_ras_err_handler_data
+ struct amdgpu_virt *virt = &adev->virt;
+ struct virt_ras_err_handler_data **data = &virt->virt_eh_data;
+ /* GPU will be marked bad on host if bp count more then 10,
+ * so alloc 512 is enough.
+ */
+ unsigned int align_space = 512;
+ void *bps = NULL;
+ struct amdgpu_bo **bps_bo = NULL;
+
[Hawking]:
1. This will result to regression on MI100 SRIOV if we don't apply check. Or leverage general data structure for the purpose, thoughts?
+ *data = kmalloc(sizeof(struct virt_ras_err_handler_data), GFP_KERNEL);
[Hawking]:
1. rename the data structure to amdgpu_virt_ras_err_handler_data
+ if (!*data)
+ return -ENOMEM;
+
+ bps = kmalloc(align_space * sizeof((*data)->bps), GFP_KERNEL);
+ bps_bo = kmalloc(align_space * sizeof((*data)->bps_bo), GFP_KERNEL);
+
+ if (!bps || !bps_bo) {
+ kfree(bps);
+ kfree(bps_bo);
+ return -ENOMEM;
+ }
+
+ (*data)->bps = bps;
+ (*data)->bps_bo = bps_bo;
+ (*data)->count = 0;
+ (*data)->last_reserved = 0;
+ return 0;
+}
+
+static void amdgpu_virt_release_bp(struct amdgpu_device *adev) {
+ struct amdgpu_virt *virt = &adev->virt;
+ struct virt_ras_err_handler_data *data = virt->virt_eh_data;
[Hawking]:
1. same as above, please rename the virt_ras_err_handler_data to amdgpu_virt_ras_err_handler_data
+ struct amdgpu_bo *bo;
+ int i;
+
+ if (!data)
+ return;
+
+ for (i = data->last_reserved - 1; i >= 0; i--) {
+ bo = data->bps_bo[i];
+ amdgpu_bo_free_kernel(&bo, NULL, NULL);
+ data->bps_bo[i] = bo;
+ data->last_reserved = i;
+ }
+}
+
+void amdgpu_release_virt_err_handler_data(struct amdgpu_device *adev) {
[Hawking]:
1. same as above, rename the function to amdgpu_virt_release_ras_err_handler_data
+ struct amdgpu_virt *virt = &adev->virt;
+ struct virt_ras_err_handler_data *data = virt->virt_eh_data;
+
+ if (!data)
+ return;
+
+ amdgpu_virt_release_bp(adev);
+
+ kfree(data->bps);
+ kfree(data->bps_bo);
+ kfree(data);
+ virt->virt_eh_data = NULL;
+}
+
+static void amdgpu_virt_ras_add_bps(struct amdgpu_device *adev,
+ struct eeprom_table_record *bps, int pages) {
+ struct amdgpu_virt *virt = &adev->virt;
+ struct virt_ras_err_handler_data *data = virt->virt_eh_data;
+
+ if (!data)
+ return;
+
+ memcpy(&data->bps[data->count], bps, pages * sizeof(*data->bps));
+ data->count += pages;
+}
+
+static void amdgpu_virt_ras_reserve_bps(struct amdgpu_device *adev) {
+ struct amdgpu_virt *virt = &adev->virt;
+ struct virt_ras_err_handler_data *data = virt->virt_eh_data;
+ struct amdgpu_bo *bo = NULL;
+ uint64_t bp;
+ int i;
+
+ if (!data)
+ return;
+
+ for (i = data->last_reserved; i < data->count; i++) {
+ bp = data->bps[i].retired_page;
+
+ /* There are two cases of reserve error should be ignored:
+ * 1) a ras bad page has been allocated (used by someone);
+ * 2) a ras bad page has been reserved (duplicate error injection
+ * for one page);
+ */
+ if (amdgpu_bo_create_kernel_at(adev, bp << AMDGPU_GPU_PAGE_SHIFT,
+ AMDGPU_GPU_PAGE_SIZE, + AMDGPU_GEM_DOMAIN_VRAM, + &bo, NULL))
+ DRM_WARN("RAS WARN: reserve vram for retired page %llx fail\n", bp);
[Hawking]:
1. can we make this as debug message?
+
+ data->bps_bo[i] = bo;
+ data->last_reserved = i + 1;
+ bo = NULL;
+ }
+}
+
+static bool amdgpu_virt_check_bad_page(struct amdgpu_device *adev,
+ uint64_t retired_page)
+{
+ struct amdgpu_virt *virt = &adev->virt;
+ struct virt_ras_err_handler_data *data = virt->virt_eh_data;
+ int i;
+
+ if (!data)
+ return true;
+
+ for (i = 0; i < data->count; i++)
+ if (retired_page == data->bps[i].retired_page)
+ return true;
+
+ return false;
+}
+
+static void amdgpu_virt_add_bad_page(struct amdgpu_device *adev,
+ uint64_t bp_block_offset, uint32_t bp_block_size) {
+ struct eeprom_table_record bp;
+ uint64_t retired_page;
+ uint32_t bp_idx, bp_cnt;
+
+ if (bp_block_size) {
+ bp_cnt = bp_block_size / sizeof(uint64_t);
+ for (bp_idx = 0; bp_idx < bp_cnt; bp_idx++) {
+ retired_page = *(uint64_t *)(adev->fw_vram_usage.va +
+ bp_block_offset + bp_idx * sizeof(uint64_t));
+ bp.retired_page = retired_page;
+
+ if (amdgpu_virt_check_bad_page(adev, retired_page))
+ continue;
+
+ amdgpu_virt_ras_add_bps(adev, &bp, 1);
+
+ amdgpu_virt_ras_reserve_bps(adev);
+ }
+ }
+}
+
void amdgpu_virt_init_data_exchange(struct amdgpu_device *adev) {
uint32_t pf2vf_size = 0;
uint32_t checksum = 0;
uint32_t checkval;
char *str;
+ uint64_t bp_block_offset = 0;
+ uint32_t bp_block_size = 0;
adev->virt.fw_reserve.p_pf2vf = NULL; adev->virt.fw_reserve.p_vf2pf = NULL;
@@ -275,6 +428,17 @@ void amdgpu_virt_init_data_exchange(struct amdgpu_device *adev)
/* pf2vf message must be in 4K */
if (pf2vf_size > 0 && pf2vf_size < 4096) {
+ if (adev->virt.fw_reserve.p_pf2vf->version == 2) {
+ struct amdgim_pf2vf_info_v2 *pf2vf_v2 = NULL;
+
+ pf2vf_v2 = (struct amdgim_pf2vf_info_v2 *)adev->virt.fw_reserve.p_pf2vf;
+ bp_block_offset = ((uint64_t)pf2vf_v2->bp_block_offset_L & 0xFFFFFFFF) |
+ ((((uint64_t)pf2vf_v2->bp_block_offset_H) << 32) & 0xFFFFFFFF00000000);
+ bp_block_size = pf2vf_v2->bp_block_size;
+
+ amdgpu_virt_add_bad_page(adev, bp_block_offset, bp_block_size);
+ }
+
checkval = amdgpu_virt_fw_reserve_get_checksum(
adev->virt.fw_reserve.p_pf2vf, pf2vf_size,
adev->virt.fw_reserve.checksum_key, checksum); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
old mode 100644
new mode 100755
index b90e822cebd7..96d84f036e96
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +7 -1
@@ -143,19 +143,27 @@ struct amdgim_pf2vf_info_v2 {uint32_t vce_enc_max_pixels_count;/* 16x16 pixels/sec, codec independent */ uint32_t vce_enc_max_bandwidth; +/* Bad pages block position in BYTE */ +uint32_t bp_block_offset_L; +uint32_t bp_block_offset_H; +/* Bad pages block size in BYTE */ +uint32_t bp_block_size;/* MEC FW position in kb from the start of VF visible frame buffer */ -uint64_t mecfw_kboffset; +uint32_t mecfw_kboffset_L; +uint32_t mecfw_kboffset_H;
[Hawking]:
1. can you just explain the reason that split the position in lower 32bit and higher 32 bit respectively?
/* MEC FW size in KB */
uint32_t mecfw_ksize;
/* UVD FW position in kb from the start of VF visible frame buffer */
- uint64_t uvdfw_kboffset;
+ uint32_t uvdfw_kboffset_L;
+ uint32_t uvdfw_kboffset_H;
/* UVD FW size in KB */
uint32_t uvdfw_ksize;
/* VCE FW position in kb from the start of VF visible frame buffer */
- uint64_t vcefw_kboffset;
+ uint32_t vcefw_kboffset_L;
+ uint32_t vcefw_kboffset_H;
/* VCE FW size in KB */
uint32_t vcefw_ksize;
- uint32_t reserved[AMDGIM_GET_STRUCTURE_RESERVED_SIZE(256, 0, 0, (9 + sizeof(struct amd_sriov_msg_pf2vf_info_header)/sizeof(uint32_t)), 3)];
+ uint32_t reserved[AMDGIM_GET_STRUCTURE_RESERVED_SIZE(256, 0, 0, (18 +
+sizeof(struct amd_sriov_msg_pf2vf_info_header)/sizeof(uint32_t)), 0)];
} __aligned(4);
@@ -254,6 +262,17 @@ typedef struct amdgim_vf2pf_info_v2 amdgim_vf2pf_info ;
} \
} while (0)
+struct virt_ras_err_handler_data {
+ /* point to bad page records array */
+ struct eeprom_table_record *bps;
+ /* point to reserved bo array */
+ struct amdgpu_bo **bps_bo;
+ /* the count of entries */
+ int count;
+ /* last reserved entry's index + 1 */
+ int last_reserved;
+};
+
/* GPU virtualization */
struct amdgpu_virt {
uint32_t caps;
@@ -272,6 +291,7 @@ struct amdgpu_virt {
uint32_t reg_access_mode;
int req_init_data_ver;
bool tdr_debug;
+ struct virt_ras_err_handler_data *virt_eh_data;
};
#define amdgpu_sriov_enabled(adev) \
@@ -323,6 +343,8 @@ void amdgpu_virt_free_mm_table(struct amdgpu_device *adev); int amdgpu_virt_fw_reserve_get_checksum(void *obj, unsigned long obj_size,
unsigned int key,
unsigned int chksum);
+void amdgpu_release_virt_err_handler_data(struct amdgpu_device *adev);
+int amdgpu_virt_init_err_handler_data(struct amdgpu_device *adev);
void amdgpu_virt_init_data_exchange(struct amdgpu_device *adev); void amdgpu_detect_virtualization(struct amdgpu_device *adev);
--
2.25.1
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
[AMD Public Use]
Hi Hawking,
Thanks for your review, please see my replies inline.
Regards,
Stanley
Quoted 116 lines
-----Original Message-----
From: Zhang, Hawking <[email protected]>
Sent: Wednesday, June 3, 2020 11:26 PM
To: Yang, Stanley <[email protected]>; [email protected]
Cc: Chen, Guchun <[email protected]>; Liu, Monk <[email protected]>; Clements, John <[email protected]>; Zhou1, Tao <[email protected]>; Li, Dennis <[email protected]>; Yang, Stanley <[email protected]>
Subject: RE: [PATCH] drm/amdgpu: support reserve bad page for virt[AMD Public Use]
Please check my comments inline.
Regards,
Hawking-----Original Message-----
From: Stanley.Yang <[email protected]>
Sent: Wednesday, June 3, 2020 22:10
To: [email protected]
Cc: Zhang, Hawking <[email protected]>; Chen, Guchun <[email protected]>; Liu, Monk <[email protected]>; Clements, John <[email protected]>; Zhou1, Tao <[email protected]>; Li, Dennis <[email protected]>; Yang, Stanley <[email protected]>
Subject: [PATCH] drm/amdgpu: support reserve bad page for virtSigned-off-by Stanley.Yang <[email protected]>Change-Id Ia0ad9453ac3ac929f95c73cbee5b7a8fc42a9816--- drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 7 +- drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c | 164 +++++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h | 30 +++- 3 files changed, 196 insertions(+), 5 deletions(-) mode change 100644 => 100755 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c mode change 100644 => 100755
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.hdrivers/gpu/drm/amd/amdgpu/amdgpu_device.c +2
-1@@ -2001,8 +2001,10 @@ static int amdgpu_device_ip_init(struct amdgpu_device *adev) } } -if (amdgpu_sriov_vf (adev)) +if (amdgpu_sriov_vf (adev)) { +amdgpu_virt_init_err_handler_data (adev);[Hawking]:
1. It might be better to rename the function toamdgpu_virt_init_ras_err_handler_data
2. we shall apply either asic type check or more general data structure check before call intoamdgpu_virt_init_ras_err_handler_data. Please check my comments inamdgpu_virt_init_ras_err_handler_datafunctiondrivers/gpu/drm/amd/amdgpu/amdgpu_device.c +3
-0amdgpu_virt_init_data_exchange (adev); + } r =amdgpu_ib_pool_init (adev);if (r) {@@ -2306,6 +2308,9 @@ static int amdgpu_device_ip_fini(struct amdgpu_device *adev) {int i, r; +if (amdgpu_sriov_vf (adev)) +amdgpu_release_virt_err_handler_data (adev);[Hawking]:
1. It might be better to rename the function toamdgpu_virt_release_ras_err_handler_data
2. we shall apply either asic type check or more general data structure check In case it introduce regression on MI100 and V340Ldrivers/gpu/drm/amd/amdgpu/amdgpu_device.c +1
-0+amdgpu_ras_pre_fini (adev); if (adev->gmc.xgmi.num_physical_nodes > 1) diff --git
a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.cb/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
old mode 100644
new mode 100755
indexf3b38c9e04ca..c1554562a2cedrivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +1
-0@@ -26,6 +26,7 @@#include <drm/drm_drv.h> #include "amdgpu.h" +#include "amdgpu_ras.h" bool amdgpu_virt_mmio_blocked(struct amdgpu_device *adev) { @@ - 255,12 +256,164 @@ int amdgpu_virt_fw_reserve_get_checksum(void *obj,drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +1
-0return ret; } +int amdgpu_virt_init_err_handler_data (struct amdgpu_device *adev) {[Hawking]:
1. same as above, rename it toamdgpu_virt_init_ras_err_handler_datadrivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +9
-0+struct amdgpu_virt *virt = &adev->virt; +struct virt_ras_err_handler_data **data = &virt->virt_eh_data; +/* GPU will be marked bad on host if bp count more then 10, +* so alloc 512 is enough. +*/ +unsigned int align_space =512 ; +void *bps =NULL ; +struct amdgpu_bo **bps_bo =NULL ; +
Quoted 3 lines
[Hawking]:
1. This will result to regression on MI100 SRIOV if we don't apply check. Or leverage general data structure for the purpose, thoughts?
[Yang, Stanley]:
Thanks for reminding, I will add asic type check to call it.
Quoted 109 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +1
-0+ *data = kmalloc(sizeof(struct virt_ras_err_handler_data),
GFP_KERNEL);[Hawking]:
1. rename the data structure toamdgpu_virt_ras_err_handler_datadrivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +5
-0+if (!*data) +return -ENOMEM ; + + bps =kmalloc (align_space *sizeof ((*data)->bps),GFP_KERNEL ); + bps_bo = kmalloc(align_space * sizeof((*data)->bps_bo),
GFP_KERNEL);drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +17
-0+ +if (!bps || !bps_bo) { +kfree (bps); +kfree (bps_bo); +return -ENOMEM ; + } + + (*data)->bps = bps; + (*data)->bps_bo = bps_bo; + (*data)->count =0 ; + (*data)->last_reserved =0 ; +return 0 ; +} + +static void amdgpu_virt_release_bp (struct amdgpu_device *adev) { +struct amdgpu_virt *virt = &adev->virt; +struct virt_ras_err_handler_data *data = virt->virt_eh_data;[Hawking]:
1. same as above, please rename thevirt_ras_err_handler_datatoamdgpu_virt_ras_err_handler_datadrivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +15
-0+struct amdgpu_bo *bo; +int i; + +if (!data) +return ; + +for (i = data->last_reserved -1 ; i >=0 ; i--) { + bo = data->bps_bo[i]; +amdgpu_bo_free_kernel (&bo,NULL ,NULL ); + data->bps_bo[i] = bo; + data->last_reserved = i; + } +} + +void amdgpu_release_virt_err_handler_data (struct amdgpu_device *adev){
[Hawking]:
1. same as above, rename the function toamdgpu_virt_release_ras_err_handler_datadrivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +42
-0+struct amdgpu_virt *virt = &adev->virt; +struct virt_ras_err_handler_data *data = virt->virt_eh_data; + +if (!data) +return ; + +amdgpu_virt_release_bp (adev); + +kfree (data->bps); +kfree (data->bps_bo); +kfree (data); + virt->virt_eh_data =NULL ; +} + +static void amdgpu_virt_ras_add_bps (struct amdgpu_device *adev, +struct eeprom_table_record *bps,int pages) { +struct amdgpu_virt *virt = &adev->virt; +struct virt_ras_err_handler_data *data = virt->virt_eh_data; + +if (!data) +return ; + +memcpy (&data->bps[data->count], bps, pages *sizeof (*data->bps)); + data->count += pages; +} + +static void amdgpu_virt_ras_reserve_bps (struct amdgpu_device *adev) { +struct amdgpu_virt *virt = &adev->virt; +struct virt_ras_err_handler_data *data = virt->virt_eh_data; +struct amdgpu_bo *bo =NULL ; +uint64_t bp; +int i; + +if (!data) +return ; + +for (i = data->last_reserved; i < data->count; i++) { + bp = data->bps[i].retired_page; + +/* There are two cases of reserve error should be ignored: +* 1) a ras bad page has been allocated (used by someone); +* 2) a ras bad page has been reserved (duplicate error injection
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +3
-0+* for one page); +*/ + if (amdgpu_bo_create_kernel_at(adev, bp <<
AMDGPU_GPU_PAGE_SHIFT,drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +4
-0+ AMDGPU_GPU_PAGE_SIZE, +AMDGPU_GEM_DOMAIN_VRAM , + &bo, NULL)) + DRM_WARN("RAS WARN: reserve vram for retiredpage %llx fail\n", bp);
Quoted 2 lines
[Hawking]:
1. can we make this as debug message?
[Yang, Stanley] :
Sure, will modify it.
Quoted 106 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +34
-0+ + data->bps_bo[i] = bo; + data->last_reserved = i +1 ; + bo =NULL ; + } +} + +static bool amdgpu_virt_check_bad_page (struct amdgpu_device *adev, +uint64_t retired_page) +{ +struct amdgpu_virt *virt = &adev->virt; +struct virt_ras_err_handler_data *data = virt->virt_eh_data; +int i; + +if (!data) +return true; + +for (i =0 ; i < data->count; i++) +if (retired_page == data->bps[i].retired_page) +return true; + +return false; +} + +static void amdgpu_virt_add_bad_page (struct amdgpu_device *adev, +uint64_t bp_block_offset,uint32_t bp_block_size) { +struct eeprom_table_record bp; +uint64_t retired_page; +uint32_t bp_idx, bp_cnt; + +if (bp_block_size) { + bp_cnt = bp_block_size /sizeof (uint64_t ); +for (bp_idx =0 ; bp_idx < bp_cnt; bp_idx++) { + retired_page = *(uint64_t *)(adev-Quoted 1 line
fw_vram_usage.va +
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +1
-0+ bp_block_offset + bp_idx *
sizeof(uint64_t));drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +3
-0+ bp.retired_page = retired_page; + + if (amdgpu_virt_check_bad_page(adev,
retired_page))drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +16
-0+continue ; + +amdgpu_virt_ras_add_bps (adev, &bp,1 ); + +amdgpu_virt_ras_reserve_bps (adev); + } + } +} +void amdgpu_virt_init_data_exchange (struct amdgpu_device *adev) {uint32_t pf2vf_size =0 ;uint32_t checksum =0 ;uint32_t checkval;char *str; +uint64_t bp_block_offset =0 ; +uint32_t bp_block_size =0 ; adev->virt.fw_reserve.p_pf2vf =NULL ; adev->virt.fw_reserve.p_vf2pf =NULL ;@@ -275,6 +428,17 @@ void amdgpu_virt_init_data_exchange(struct amdgpu_device *adev)/* pf2vf message must be in 4K */ if (pf2vf_size >0 && pf2vf_size <4096 ) { +if (adev->virt.fw_reserve.p_pf2vf->version ==2 ) { +struct amdgim_pf2vf_info_v2 *pf2vf_v2 =NULL ; + + pf2vf_v2 = (struct amdgim_pf2vf_info_v2 *)adev->virt.fw_reserve.p_pf2vf; + bp_block_offset = ((uint64_t)pf2vf_v2-Quoted 1 line
bp_block_offset_L&0xFFFFFFFF) |drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +1
-0+ ((((uint64_t)pf2vf_v2-Quoted 1 line
bp_block_offset_H) << 32) &0xFFFFFFFF00000000);drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +3
-0+ bp_block_size = pf2vf_v2->bp_block_size; + + amdgpu_virt_add_bad_page(adev,
bp_block_offset,bp_block_size);drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +2
-0+ } + checkval = amdgpu_virt_fw_reserve_get_checksum( adev->virt.fw_reserve.p_pf2vf, pf2vf_size, adev->virt.fw_reserve.checksum_key,checksum); diff --git
a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.hb/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
old mode 100644
new mode 100755
indexb90e822cebd7..96d84f036e96drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +7
-1@@ -143,19 +143,27 @@ struct amdgim_pf2vf_info_v2 {uint32_t vce_enc_max_pixels_count;/* 16x16 pixels/sec, codec independent */ uint32_t vce_enc_max_bandwidth; +/* Bad pages block position in BYTE */ +uint32_t bp_block_offset_L; +uint32_t bp_block_offset_H; +/* Bad pages block size in BYTE */ +uint32_t bp_block_size;/* MEC FW position in kb from the start of VF visible frame buffer */ -uint64_t mecfw_kboffset; +uint32_t mecfw_kboffset_L; +uint32_t mecfw_kboffset_H;
Quoted 3 lines
[Hawking]:
1. can you just explain the reason that split the position in lower 32bit and higher 32 bit respectively?
[Yang, Stanley]:
The value is misalignment from the 64bit member in amdgim_pf2vf_info_v2 struct if use 64bit variable.
Quoted 62 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +4
-3/* MEC FW size in KB */ uint32_t mecfw_ksize;/* UVD FW position in kb from the start of VF visible frame buffer */ -uint64_t uvdfw_kboffset; +uint32_t uvdfw_kboffset_L; +uint32_t uvdfw_kboffset_H;/* UVD FW size in KB */ uint32_t uvdfw_ksize;/* VCE FW position in kb from the start of VF visible frame buffer */ -uint64_t vcefw_kboffset; +uint32_t vcefw_kboffset_L; +uint32_t vcefw_kboffset_H;/* VCE FW size in KB */ uint32_t vcefw_ksize; - uint32_t reserved[AMDGIM_GET_STRUCTURE_RESERVED_SIZE(256,0, 0, (9 +
sizeof(struct amd_sriov_msg_pf2vf_info_header)/sizeof(uint32_t)), 3)];drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +1
-0+ uint32_t reserved[AMDGIM_GET_STRUCTURE_RESERVED_SIZE(256,0, 0, (18 +
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +12
-0+sizeof(struct amd_sriov_msg_pf2vf_info_header)/sizeof(uint32_t)), 0)]; }__aligned(4) ;@@ -254,6 +262,17 @@ typedef struct amdgim_vf2pf_info_v2 amdgim_vf2pf_info ; } \ }while (0 ) +struct virt_ras_err_handler_data { +/* point to bad page records array */ +struct eeprom_table_record *bps; +/* point to reserved bo array */ +struct amdgpu_bo **bps_bo; +/* the count of entries */ +int count; +/* last reserved entry's index + 1 */ +int last_reserved; +}; +/* GPU virtualization */ struct amdgpu_virt {drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +1
-0uint32_t caps;@@ -272,6 +291,7 @@ struct amdgpu_virt {uint32_t reg_access_mode;int req_init_data_ver;bool tdr_debug; +struct virt_ras_err_handler_data *virt_eh_data; };#define amdgpu_sriov_enabled (adev) \@@ -323,6 +343,8 @@ void amdgpu_virt_free_mm_table(struct
amdgpu_device*adev); int amdgpu_virt_fw_reserve_get_checksum(void *obj, unsigned longobj_size,drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +1
-0unsigned int key, unsigned int chksum); +void amdgpu_release_virt_err_handler_data(struct amdgpu_device*adev);
+intamdgpu_virt_init_err_handler_data(struct amdgpu_device *adev);
voidamdgpu_virt_init_data_exchange(struct amdgpu_device *adev); voidamdgpu_detect_virtualization(struct amdgpu_device *adev);
Quoted 2 lines
--
2.25.1
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
[AMD Public Use]
Hi GuChun,
Thanks for reminding.
Regards,
Stanley
Quoted 424 lines
-----Original Message-----
From: Chen, Guchun <[email protected]>
Sent: Thursday, June 4, 2020 9:32 AM
To: Zhang, Hawking <[email protected]>; Yang, Stanley <[email protected]>; [email protected]
Cc: Liu, Monk <[email protected]>; Clements, John <[email protected]>; Zhou1, Tao <[email protected]>; Li, Dennis <[email protected]>; Yang, Stanley <[email protected]>
Subject: RE: [PATCH] drm/amdgpu: support reserve bad page for virt[AMD Public Use]
Except Hawking's comments, we shall not change file mode in the patch as well.
mode change 100644 => 100755
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
mode change 100644 => 100755drivers/gpu/drm/amd/amdgpu/amdgpu_virt.hRegards,
Guchun-----Original Message-----
From: Zhang, Hawking <[email protected]>
Sent: Wednesday, June 3, 2020 11:26 PM
To: Yang, Stanley <[email protected]>; [email protected]
Cc: Chen, Guchun <[email protected]>; Liu, Monk <[email protected]>; Clements, John <[email protected]>; Zhou1, Tao <[email protected]>; Li, Dennis <[email protected]>; Yang, Stanley <[email protected]>
Subject: RE: [PATCH] drm/amdgpu: support reserve bad page for virt[AMD Public Use]
Please check my comments inline.
Regards,
Hawking-----Original Message-----
From: Stanley.Yang <[email protected]>
Sent: Wednesday, June 3, 2020 22:10
To: [email protected]
Cc: Zhang, Hawking <[email protected]>; Chen, Guchun <[email protected]>; Liu, Monk <[email protected]>; Clements, John <[email protected]>; Zhou1, Tao <[email protected]>; Li, Dennis <[email protected]>; Yang, Stanley <[email protected]>
Subject: [PATCH] drm/amdgpu: support reserve bad page for virtSigned-off-by Stanley.Yang <[email protected]>Change-Id Ia0ad9453ac3ac929f95c73cbee5b7a8fc42a9816--- drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 7 +- drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c | 164 +++++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h | 30 +++- 3 files changed, 196 insertions(+), 5 deletions(-) mode change 100644 => 100755 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c mode change 100644 => 100755
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.hdrivers/gpu/drm/amd/amdgpu/amdgpu_device.c +2
-1@@ -2001,8 +2001,10 @@ static int amdgpu_device_ip_init(struct amdgpu_device *adev) } } -if (amdgpu_sriov_vf (adev)) +if (amdgpu_sriov_vf (adev)) { +amdgpu_virt_init_err_handler_data (adev);[Hawking]:
1. It might be better to rename the function toamdgpu_virt_init_ras_err_handler_data
2. we shall apply either asic type check or more general data structure check before call intoamdgpu_virt_init_ras_err_handler_data. Please check my comments inamdgpu_virt_init_ras_err_handler_datafunctiondrivers/gpu/drm/amd/amdgpu/amdgpu_device.c +3
-0amdgpu_virt_init_data_exchange (adev); + } r =amdgpu_ib_pool_init (adev);if (r) {@@ -2306,6 +2308,9 @@ static int amdgpu_device_ip_fini(struct amdgpu_device *adev) {int i, r; +if (amdgpu_sriov_vf (adev)) +amdgpu_release_virt_err_handler_data (adev);[Hawking]:
1. It might be better to rename the function toamdgpu_virt_release_ras_err_handler_data
2. we shall apply either asic type check or more general data structure check In case it introduce regression on MI100 and V340Ldrivers/gpu/drm/amd/amdgpu/amdgpu_device.c +1
-0+amdgpu_ras_pre_fini (adev); if (adev->gmc.xgmi.num_physical_nodes > 1) diff --git
a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.cb/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
old mode 100644
new mode 100755
indexf3b38c9e04ca..c1554562a2cedrivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +1
-0@@ -26,6 +26,7 @@#include <drm/drm_drv.h> #include "amdgpu.h" +#include "amdgpu_ras.h" bool amdgpu_virt_mmio_blocked(struct amdgpu_device *adev) { @@ - 255,12 +256,164 @@ int amdgpu_virt_fw_reserve_get_checksum(void *obj,drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +1
-0return ret; } +int amdgpu_virt_init_err_handler_data (struct amdgpu_device *adev) {[Hawking]:
1. same as above, rename it toamdgpu_virt_init_ras_err_handler_datadrivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +9
-0+struct amdgpu_virt *virt = &adev->virt; +struct virt_ras_err_handler_data **data = &virt->virt_eh_data; +/* GPU will be marked bad on host if bp count more then 10, +* so alloc 512 is enough. +*/ +unsigned int align_space =512 ; +void *bps =NULL ; +struct amdgpu_bo **bps_bo =NULL ; +[Hawking]:
1. This will result to regression on MI100 SRIOV if we don't apply check. Or leverage general data structure for the purpose, thoughts?drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +1
-0+ *data = kmalloc(sizeof(struct virt_ras_err_handler_data),
GFP_KERNEL);[Hawking]:
1. rename the data structure toamdgpu_virt_ras_err_handler_datadrivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +5
-0+if (!*data) +return -ENOMEM ; + + bps =kmalloc (align_space *sizeof ((*data)->bps),GFP_KERNEL ); + bps_bo = kmalloc(align_space * sizeof((*data)->bps_bo),
GFP_KERNEL);drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +17
-0+ +if (!bps || !bps_bo) { +kfree (bps); +kfree (bps_bo); +return -ENOMEM ; + } + + (*data)->bps = bps; + (*data)->bps_bo = bps_bo; + (*data)->count =0 ; + (*data)->last_reserved =0 ; +return 0 ; +} + +static void amdgpu_virt_release_bp (struct amdgpu_device *adev) { +struct amdgpu_virt *virt = &adev->virt; +struct virt_ras_err_handler_data *data = virt->virt_eh_data;[Hawking]:
1. same as above, please rename thevirt_ras_err_handler_datatoamdgpu_virt_ras_err_handler_datadrivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +15
-0+struct amdgpu_bo *bo; +int i; + +if (!data) +return ; + +for (i = data->last_reserved -1 ; i >=0 ; i--) { + bo = data->bps_bo[i]; +amdgpu_bo_free_kernel (&bo,NULL ,NULL ); + data->bps_bo[i] = bo; + data->last_reserved = i; + } +} + +void amdgpu_release_virt_err_handler_data (struct amdgpu_device *adev){
[Hawking]:
1. same as above, rename the function toamdgpu_virt_release_ras_err_handler_datadrivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +42
-0+struct amdgpu_virt *virt = &adev->virt; +struct virt_ras_err_handler_data *data = virt->virt_eh_data; + +if (!data) +return ; + +amdgpu_virt_release_bp (adev); + +kfree (data->bps); +kfree (data->bps_bo); +kfree (data); + virt->virt_eh_data =NULL ; +} + +static void amdgpu_virt_ras_add_bps (struct amdgpu_device *adev, +struct eeprom_table_record *bps,int pages) { +struct amdgpu_virt *virt = &adev->virt; +struct virt_ras_err_handler_data *data = virt->virt_eh_data; + +if (!data) +return ; + +memcpy (&data->bps[data->count], bps, pages *sizeof (*data->bps)); + data->count += pages; +} + +static void amdgpu_virt_ras_reserve_bps (struct amdgpu_device *adev) { +struct amdgpu_virt *virt = &adev->virt; +struct virt_ras_err_handler_data *data = virt->virt_eh_data; +struct amdgpu_bo *bo =NULL ; +uint64_t bp; +int i; + +if (!data) +return ; + +for (i = data->last_reserved; i < data->count; i++) { + bp = data->bps[i].retired_page; + +/* There are two cases of reserve error should be ignored: +* 1) a ras bad page has been allocated (used by someone); +* 2) a ras bad page has been reserved (duplicate error injection
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +3
-0+* for one page); +*/ + if (amdgpu_bo_create_kernel_at(adev, bp <<
AMDGPU_GPU_PAGE_SHIFT,drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +4
-0+ AMDGPU_GPU_PAGE_SIZE, +AMDGPU_GEM_DOMAIN_VRAM , + &bo, NULL)) + DRM_WARN("RAS WARN: reserve vram for retiredpage %llx fail\n", bp);
[Hawking]:
1. can we make this as debug message?drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +34
-0+ + data->bps_bo[i] = bo; + data->last_reserved = i +1 ; + bo =NULL ; + } +} + +static bool amdgpu_virt_check_bad_page (struct amdgpu_device *adev, +uint64_t retired_page) +{ +struct amdgpu_virt *virt = &adev->virt; +struct virt_ras_err_handler_data *data = virt->virt_eh_data; +int i; + +if (!data) +return true; + +for (i =0 ; i < data->count; i++) +if (retired_page == data->bps[i].retired_page) +return true; + +return false; +} + +static void amdgpu_virt_add_bad_page (struct amdgpu_device *adev, +uint64_t bp_block_offset,uint32_t bp_block_size) { +struct eeprom_table_record bp; +uint64_t retired_page; +uint32_t bp_idx, bp_cnt; + +if (bp_block_size) { + bp_cnt = bp_block_size /sizeof (uint64_t ); +for (bp_idx =0 ; bp_idx < bp_cnt; bp_idx++) { + retired_page = *(uint64_t *)(adev-Quoted 1 line
fw_vram_usage.va +
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +1
-0+ bp_block_offset + bp_idx *
sizeof(uint64_t));drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +3
-0+ bp.retired_page = retired_page; + + if (amdgpu_virt_check_bad_page(adev,
retired_page))drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +16
-0+continue ; + +amdgpu_virt_ras_add_bps (adev, &bp,1 ); + +amdgpu_virt_ras_reserve_bps (adev); + } + } +} +void amdgpu_virt_init_data_exchange (struct amdgpu_device *adev) {uint32_t pf2vf_size =0 ;uint32_t checksum =0 ;uint32_t checkval;char *str; +uint64_t bp_block_offset =0 ; +uint32_t bp_block_size =0 ; adev->virt.fw_reserve.p_pf2vf =NULL ; adev->virt.fw_reserve.p_vf2pf =NULL ;@@ -275,6 +428,17 @@ void amdgpu_virt_init_data_exchange(struct amdgpu_device *adev)/* pf2vf message must be in 4K */ if (pf2vf_size >0 && pf2vf_size <4096 ) { +if (adev->virt.fw_reserve.p_pf2vf->version ==2 ) { +struct amdgim_pf2vf_info_v2 *pf2vf_v2 =NULL ; + + pf2vf_v2 = (struct amdgim_pf2vf_info_v2 *)adev->virt.fw_reserve.p_pf2vf; + bp_block_offset = ((uint64_t)pf2vf_v2-Quoted 1 line
bp_block_offset_L&0xFFFFFFFF) |drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +1
-0+ ((((uint64_t)pf2vf_v2-Quoted 1 line
bp_block_offset_H) << 32) &0xFFFFFFFF00000000);drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +3
-0+ bp_block_size = pf2vf_v2->bp_block_size; + + amdgpu_virt_add_bad_page(adev,
bp_block_offset,bp_block_size);drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +2
-0+ } + checkval = amdgpu_virt_fw_reserve_get_checksum( adev->virt.fw_reserve.p_pf2vf, pf2vf_size, adev->virt.fw_reserve.checksum_key,checksum); diff --git
a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.hb/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
old mode 100644
new mode 100755
indexb90e822cebd7..96d84f036e96drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +7
-1@@ -143,19 +143,27 @@ struct amdgim_pf2vf_info_v2 {uint32_t vce_enc_max_pixels_count;/* 16x16 pixels/sec, codec independent */ uint32_t vce_enc_max_bandwidth; +/* Bad pages block position in BYTE */ +uint32_t bp_block_offset_L; +uint32_t bp_block_offset_H; +/* Bad pages block size in BYTE */ +uint32_t bp_block_size;/* MEC FW position in kb from the start of VF visible frame buffer */ -uint64_t mecfw_kboffset; +uint32_t mecfw_kboffset_L; +uint32_t mecfw_kboffset_H;[Hawking]:
1. can you just explain the reason that split the position in lower 32bit and higher 32 bit respectively?drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +4
-3/* MEC FW size in KB */ uint32_t mecfw_ksize;/* UVD FW position in kb from the start of VF visible frame buffer */ -uint64_t uvdfw_kboffset; +uint32_t uvdfw_kboffset_L; +uint32_t uvdfw_kboffset_H;/* UVD FW size in KB */ uint32_t uvdfw_ksize;/* VCE FW position in kb from the start of VF visible frame buffer */ -uint64_t vcefw_kboffset; +uint32_t vcefw_kboffset_L; +uint32_t vcefw_kboffset_H;/* VCE FW size in KB */ uint32_t vcefw_ksize; - uint32_t reserved[AMDGIM_GET_STRUCTURE_RESERVED_SIZE(256,0, 0, (9 +
sizeof(struct amd_sriov_msg_pf2vf_info_header)/sizeof(uint32_t)), 3)];drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +1
-0+ uint32_t reserved[AMDGIM_GET_STRUCTURE_RESERVED_SIZE(256,0, 0, (18 +
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +12
-0+sizeof(struct amd_sriov_msg_pf2vf_info_header)/sizeof(uint32_t)), 0)]; }__aligned(4) ;@@ -254,6 +262,17 @@ typedef struct amdgim_vf2pf_info_v2 amdgim_vf2pf_info ; } \ }while (0 ) +struct virt_ras_err_handler_data { +/* point to bad page records array */ +struct eeprom_table_record *bps; +/* point to reserved bo array */ +struct amdgpu_bo **bps_bo; +/* the count of entries */ +int count; +/* last reserved entry's index + 1 */ +int last_reserved; +}; +/* GPU virtualization */ struct amdgpu_virt {drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +1
-0uint32_t caps;@@ -272,6 +291,7 @@ struct amdgpu_virt {uint32_t reg_access_mode;int req_init_data_ver;bool tdr_debug; +struct virt_ras_err_handler_data *virt_eh_data; };#define amdgpu_sriov_enabled (adev) \@@ -323,6 +343,8 @@ void amdgpu_virt_free_mm_table(struct
amdgpu_device*adev); int amdgpu_virt_fw_reserve_get_checksum(void *obj, unsigned longobj_size,drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +1
-0unsigned int key, unsigned int chksum); +void amdgpu_release_virt_err_handler_data(struct amdgpu_device*adev);
+intamdgpu_virt_init_err_handler_data(struct amdgpu_device *adev);
voidamdgpu_virt_init_data_exchange(struct amdgpu_device *adev); voidamdgpu_detect_virtualization(struct amdgpu_device *adev);
Quoted 2 lines
--
2.25.1
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
[AMD Public Use]
Two comments inline
Quoted 28 lines
-----Original Message-----
From: Stanley.Yang <[email protected]>
Sent: 2020年6月3日 22:10
To: [email protected]
Cc: Zhang, Hawking <[email protected]>; Chen, Guchun <[email protected]>; Liu, Monk <[email protected]>; Clements, John <[email protected]>; Zhou1, Tao <[email protected]>; Li, Dennis <[email protected]>; Yang, Stanley <[email protected]>
Subject: [PATCH] drm/amdgpu: support reserve bad page for virtSigned-off-by Stanley.Yang <[email protected]>Change-Id Ia0ad9453ac3ac929f95c73cbee5b7a8fc42a9816--- drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 7 +- drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c | 164 +++++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h | 30 +++- 3 files changed, 196 insertions(+), 5 deletions(-) mode change 100644 => 100755 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c mode change 100644 => 100755
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.hdrivers/gpu/drm/amd/amdgpu/amdgpu_device.c +0
-0
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c static int amdgpu_device_ip_init(struct · line 2006
Quoted 7 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +2
-1@@ -2001,8 +2001,10 @@ static int amdgpu_device_ip_init(struct amdgpu_device *adev) } } -if (amdgpu_sriov_vf (adev)) +if (amdgpu_sriov_vf (adev)) { +amdgpu_virt_init_err_handler_data (adev);
[Tao] It's can be also called in amdgpu_virt_add_bad_page once to avoid asic type check, but either way is OK for me.
Quoted 135 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +1
-0amdgpu_virt_init_data_exchange (adev); + } r =amdgpu_ib_pool_init (adev);drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +3
-0if (r) {@@ -2306,6 +2308,9 @@ static int amdgpu_device_ip_fini(struct amdgpu_device *adev) {int i, r; +if (amdgpu_sriov_vf (adev)) +amdgpu_release_virt_err_handler_data (adev); +amdgpu_ras_pre_fini (adev); if (adev->gmc.xgmi.num_physical_nodes > 1) diff --git
a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.cb/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
old mode 100644
new mode 100755
indexf3b38c9e04ca..c1554562a2cedrivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +1
-0@@ -26,6 +26,7 @@#include <drm/drm_drv.h> #include "amdgpu.h" +#include "amdgpu_ras.h" bool amdgpu_virt_mmio_blocked(struct amdgpu_device *adev) { @@ - 255,12 +256,164 @@ int amdgpu_virt_fw_reserve_get_checksum(void *obj,drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +11
-0return ret; } +int amdgpu_virt_init_err_handler_data (struct amdgpu_device *adev) { +struct amdgpu_virt *virt = &adev->virt; +struct virt_ras_err_handler_data **data = &virt->virt_eh_data; +/* GPU will be marked bad on host if bp count more then 10, +* so alloc 512 is enough. +*/ +unsigned int align_space =512 ; +void *bps =NULL ; +struct amdgpu_bo **bps_bo =NULL ; + + *data = kmalloc(sizeof(struct virt_ras_err_handler_data),
GFP_KERNEL);drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +5
-0+if (!*data) +return -ENOMEM ; + + bps =kmalloc (align_space *sizeof ((*data)->bps),GFP_KERNEL ); + bps_bo = kmalloc(align_space * sizeof((*data)->bps_bo),
GFP_KERNEL);drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +32
-0+ +if (!bps || !bps_bo) { +kfree (bps); +kfree (bps_bo); +return -ENOMEM ; + } + + (*data)->bps = bps; + (*data)->bps_bo = bps_bo; + (*data)->count =0 ; + (*data)->last_reserved =0 ; +return 0 ; +} + +static void amdgpu_virt_release_bp (struct amdgpu_device *adev) { +struct amdgpu_virt *virt = &adev->virt; +struct virt_ras_err_handler_data *data = virt->virt_eh_data; +struct amdgpu_bo *bo; +int i; + +if (!data) +return ; + +for (i = data->last_reserved -1 ; i >=0 ; i--) { + bo = data->bps_bo[i]; +amdgpu_bo_free_kernel (&bo,NULL ,NULL ); + data->bps_bo[i] = bo; + data->last_reserved = i; + } +} + +void amdgpu_release_virt_err_handler_data (struct amdgpu_device *adev){
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +42
-0+struct amdgpu_virt *virt = &adev->virt; +struct virt_ras_err_handler_data *data = virt->virt_eh_data; + +if (!data) +return ; + +amdgpu_virt_release_bp (adev); + +kfree (data->bps); +kfree (data->bps_bo); +kfree (data); + virt->virt_eh_data =NULL ; +} + +static void amdgpu_virt_ras_add_bps (struct amdgpu_device *adev, +struct eeprom_table_record *bps,int pages) { +struct amdgpu_virt *virt = &adev->virt; +struct virt_ras_err_handler_data *data = virt->virt_eh_data; + +if (!data) +return ; + +memcpy (&data->bps[data->count], bps, pages *sizeof (*data->bps)); + data->count += pages; +} + +static void amdgpu_virt_ras_reserve_bps (struct amdgpu_device *adev) { +struct amdgpu_virt *virt = &adev->virt; +struct virt_ras_err_handler_data *data = virt->virt_eh_data; +struct amdgpu_bo *bo =NULL ; +uint64_t bp; +int i; + +if (!data) +return ; + +for (i = data->last_reserved; i < data->count; i++) { + bp = data->bps[i].retired_page; + +/* There are two cases of reserve error should be ignored: +* 1) a ras bad page has been allocated (used by someone); +* 2) a ras bad page has been reserved (duplicate error injection
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +3
-0+* for one page); +*/ + if (amdgpu_bo_create_kernel_at(adev, bp <<
AMDGPU_GPU_PAGE_SHIFT,drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +4
-0+ AMDGPU_GPU_PAGE_SIZE, +AMDGPU_GEM_DOMAIN_VRAM , + &bo, NULL)) + DRM_WARN("RAS WARN: reserve vram for retired
Quoted 1 line
page %llx fail\n", bp);
[Tao] It's better to distinguish the message from the one in bare mental RAS.
Quoted 162 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +34
-0+ + data->bps_bo[i] = bo; + data->last_reserved = i +1 ; + bo =NULL ; + } +} + +static bool amdgpu_virt_check_bad_page (struct amdgpu_device *adev, +uint64_t retired_page) +{ +struct amdgpu_virt *virt = &adev->virt; +struct virt_ras_err_handler_data *data = virt->virt_eh_data; +int i; + +if (!data) +return true; + +for (i =0 ; i < data->count; i++) +if (retired_page == data->bps[i].retired_page) +return true; + +return false; +} + +static void amdgpu_virt_add_bad_page (struct amdgpu_device *adev, +uint64_t bp_block_offset,uint32_t bp_block_size) { +struct eeprom_table_record bp; +uint64_t retired_page; +uint32_t bp_idx, bp_cnt; + +if (bp_block_size) { + bp_cnt = bp_block_size /sizeof (uint64_t ); +for (bp_idx =0 ; bp_idx < bp_cnt; bp_idx++) { + retired_page = *(uint64_t *)(adev-Quoted 1 line
fw_vram_usage.va +
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +1
-0+ bp_block_offset + bp_idx *
sizeof(uint64_t));drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +19
-0+ bp.retired_page = retired_page; + +if (amdgpu_virt_check_bad_page (adev, retired_page)) +continue ; + +amdgpu_virt_ras_add_bps (adev, &bp,1 ); + +amdgpu_virt_ras_reserve_bps (adev); + } + } +} +void amdgpu_virt_init_data_exchange (struct amdgpu_device *adev) {uint32_t pf2vf_size =0 ;uint32_t checksum =0 ;uint32_t checkval;char *str; +uint64_t bp_block_offset =0 ; +uint32_t bp_block_size =0 ; adev->virt.fw_reserve.p_pf2vf =NULL ; adev->virt.fw_reserve.p_vf2pf =NULL ;@@ -275,6 +428,17 @@ void amdgpu_virt_init_data_exchange(struct amdgpu_device *adev)/* pf2vf message must be in 4K */ if (pf2vf_size >0 && pf2vf_size <4096 ) { +if (adev->virt.fw_reserve.p_pf2vf->version ==2 ) { +struct amdgim_pf2vf_info_v2 *pf2vf_v2 =NULL ; + + pf2vf_v2 = (struct amdgim_pf2vf_info_v2 *)adev->virt.fw_reserve.p_pf2vf; + bp_block_offset = ((uint64_t)pf2vf_v2-Quoted 1 line
bp_block_offset_L&0xFFFFFFFF) |drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +1
-0+ ((((uint64_t)pf2vf_v2-Quoted 1 line
bp_block_offset_H) << 32) &0xFFFFFFFF00000000);drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +3
-0+ bp_block_size = pf2vf_v2->bp_block_size; + + amdgpu_virt_add_bad_page(adev,
bp_block_offset,bp_block_size);drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +2
-0+ } + checkval = amdgpu_virt_fw_reserve_get_checksum( adev->virt.fw_reserve.p_pf2vf, pf2vf_size, adev->virt.fw_reserve.checksum_key,checksum); diff --git
a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.hb/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
old mode 100644
new mode 100755
indexb90e822cebd7..96d84f036e96drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +12
-4@@ -143,19 +143,27 @@ struct amdgim_pf2vf_info_v2 {uint32_t vce_enc_max_pixels_count;/* 16x16 pixels/sec, codec independent */ uint32_t vce_enc_max_bandwidth; +/* Bad pages block position in BYTE */ +uint32_t bp_block_offset_L; +uint32_t bp_block_offset_H; +/* Bad pages block size in BYTE */ +uint32_t bp_block_size;/* MEC FW position in kb from the start of VF visible frame buffer */ -uint64_t mecfw_kboffset; +uint32_t mecfw_kboffset_L; +uint32_t mecfw_kboffset_H;/* MEC FW size in KB */ uint32_t mecfw_ksize;/* UVD FW position in kb from the start of VF visible frame buffer */ -uint64_t uvdfw_kboffset; +uint32_t uvdfw_kboffset_L; +uint32_t uvdfw_kboffset_H;/* UVD FW size in KB */ uint32_t uvdfw_ksize;/* VCE FW position in kb from the start of VF visible frame buffer */ -uint64_t vcefw_kboffset; +uint32_t vcefw_kboffset_L; +uint32_t vcefw_kboffset_H;/* VCE FW size in KB */ uint32_t vcefw_ksize; -uint32_t reserved[AMDGIM_GET_STRUCTURE_RESERVED_SIZE (256 ,0 , 0, (9 + sizeof(struct amd_sriov_msg_pf2vf_info_header)/sizeof(uint32_t)), 3)]; + uint32_t reserved[AMDGIM_GET_STRUCTURE_RESERVED_SIZE(256, 0,0, (18 +
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +12
-0+sizeof(struct amd_sriov_msg_pf2vf_info_header)/sizeof(uint32_t)), 0)]; }__aligned(4) ;@@ -254,6 +262,17 @@ typedef struct amdgim_vf2pf_info_v2 amdgim_vf2pf_info ; } \ }while (0 ) +struct virt_ras_err_handler_data { +/* point to bad page records array */ +struct eeprom_table_record *bps; +/* point to reserved bo array */ +struct amdgpu_bo **bps_bo; +/* the count of entries */ +int count; +/* last reserved entry's index + 1 */ +int last_reserved; +}; +/* GPU virtualization */ struct amdgpu_virt {drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +1
-0uint32_t caps;@@ -272,6 +291,7 @@ struct amdgpu_virt {uint32_t reg_access_mode;int req_init_data_ver;bool tdr_debug; +struct virt_ras_err_handler_data *virt_eh_data; };#define amdgpu_sriov_enabled (adev) \@@ -323,6 +343,8 @@ void amdgpu_virt_free_mm_table(struct
amdgpu_device*adev); int amdgpu_virt_fw_reserve_get_checksum(void *obj, unsigned longobj_size,drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +2
-0unsigned int key, unsigned int chksum); +void amdgpu_release_virt_err_handler_data (struct amdgpu_device *adev); +int amdgpu_virt_init_err_handler_data (struct amdgpu_device *adev);void amdgpu_virt_init_data_exchange (struct amdgpu_device *adev);void
amdgpu_detect_virtualization(struct amdgpu_device *adev);
Quoted 2 lines
--
2.25.1
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
[AMD Public Use]
Thanks tao, to call amdgpu_virt_init_err_handler_data In amdgpu_virt_add_bad_page once Is also a way, I will check whether has potential risk. And I'll make distinguish the message from the one in bare mental RAS when reserved page failed.
Regards,
Stanley
Quoted 365 lines
-----Original Message-----
From: Zhou1, Tao <[email protected]>
Sent: Thursday, June 4, 2020 12:16 PM
To: Yang, Stanley <[email protected]>; [email protected]
Cc: Zhang, Hawking <[email protected]>; Chen, Guchun <[email protected]>; Liu, Monk <[email protected]>; Clements, John <[email protected]>; Li, Dennis <[email protected]>; Yang, Stanley <[email protected]>
Subject: RE: [PATCH] drm/amdgpu: support reserve bad page for virt[AMD Public Use]
Two comments inline
Quoted 6 lines
-----Original Message-----
From: Stanley.Yang <[email protected]>
Sent: 2020年6月3日 22:10
To: [email protected]
Cc: Zhang, Hawking <[email protected]>; Chen, Guchun <[email protected]>; Liu, Monk <[email protected]>; Clements,John
Quoted 1 line
<[email protected]>; Zhou1, Tao <[email protected]>; Li,
Dennis
Quoted 22 lines
<[email protected]>; Yang, Stanley <[email protected]>
Subject: [PATCH] drm/amdgpu: support reserve bad page for virtSigned-off-by Stanley.Yang <[email protected]>Change-Id Ia0ad9453ac3ac929f95c73cbee5b7a8fc42a9816--- drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 7 +- drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c | 164 +++++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h | 30 +++- 3 files changed, 196 insertions(+), 5 deletions(-) mode change 100644 => 100755 drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c mode change 100644 => 100755
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.hdrivers/gpu/drm/amd/amdgpu/amdgpu_device.c +0
-0drivers/gpu/drm/amd/amdgpu/amdgpu_device.c static int amdgpu_device_ip_init(struct · line 2006
Quoted 7 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +2
-1@@ -2001,8 +2001,10 @@ static int amdgpu_device_ip_init(struct amdgpu_device *adev) } } -if (amdgpu_sriov_vf (adev)) +if (amdgpu_sriov_vf (adev)) { +amdgpu_virt_init_err_handler_data (adev);[Tao] It's can be also called in
amdgpu_virt_add_bad_pageonce to avoid asic type check, but either way is OK for me.Quoted 29 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +1
-0amdgpu_virt_init_data_exchange (adev); + } r =amdgpu_ib_pool_init (adev);drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +3
-0if (r) {@@ -2306,6 +2308,9 @@ static int amdgpu_device_ip_fini(struct amdgpu_device *adev) {int i, r; +if (amdgpu_sriov_vf (adev)) +amdgpu_release_virt_err_handler_data (adev); +amdgpu_ras_pre_fini (adev); if (adev->gmc.xgmi.num_physical_nodes > 1) diff --git
a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.cb/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
old mode 100644
new mode 100755
indexf3b38c9e04ca..c1554562a2cedrivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +1
-0@@ -26,6 +26,7 @@#include <drm/drm_drv.h> #include "amdgpu.h" +#include "amdgpu_ras.h" bool amdgpu_virt_mmio_blocked(struct amdgpu_device *adev) { @@ -Quoted 1 line
255,12 +256,164 @@ int amdgpu_virt_fw_reserve_get_checksum(void
*obj,
Quoted 43 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +11
-0return ret; } +int amdgpu_virt_init_err_handler_data (struct amdgpu_device *adev) { +struct amdgpu_virt *virt = &adev->virt; +struct virt_ras_err_handler_data **data = &virt->virt_eh_data; +/* GPU will be marked bad on host if bp count more then 10, +* so alloc 512 is enough. +*/ +unsigned int align_space =512 ; +void *bps =NULL ; +struct amdgpu_bo **bps_bo =NULL ; + + *data = kmalloc(sizeof(struct virt_ras_err_handler_data),
GFP_KERNEL);drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +5
-0+if (!*data) +return -ENOMEM ; + + bps =kmalloc (align_space *sizeof ((*data)->bps),GFP_KERNEL ); + bps_bo = kmalloc(align_space * sizeof((*data)->bps_bo),
GFP_KERNEL);drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +22
-0+ +if (!bps || !bps_bo) { +kfree (bps); +kfree (bps_bo); +return -ENOMEM ; + } + + (*data)->bps = bps; + (*data)->bps_bo = bps_bo; + (*data)->count =0 ; + (*data)->last_reserved =0 ; +return 0 ; +} + +static void amdgpu_virt_release_bp (struct amdgpu_device *adev) { +struct amdgpu_virt *virt = &adev->virt; +struct virt_ras_err_handler_data *data = virt->virt_eh_data; +struct amdgpu_bo *bo; +int i; + +if (!data) +return ;drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c int amdgpu_virt_fw_reserve_get_checksum(void *obj,
Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +10
-0+ +for (i = data->last_reserved -1 ; i >=0 ; i--) { + bo = data->bps_bo[i]; +amdgpu_bo_free_kernel (&bo,NULL ,NULL ); + data->bps_bo[i] = bo; + data->last_reserved = i; + } +} + +void amdgpu_release_virt_err_handler_data(struct amdgpu_device*adev)
Quoted 52 lines
{
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +42
-0+struct amdgpu_virt *virt = &adev->virt; +struct virt_ras_err_handler_data *data = virt->virt_eh_data; + +if (!data) +return ; + +amdgpu_virt_release_bp (adev); + +kfree (data->bps); +kfree (data->bps_bo); +kfree (data); + virt->virt_eh_data =NULL ; +} + +static void amdgpu_virt_ras_add_bps (struct amdgpu_device *adev, +struct eeprom_table_record *bps,int pages) { +struct amdgpu_virt *virt = &adev->virt; +struct virt_ras_err_handler_data *data = virt->virt_eh_data; + +if (!data) +return ; + +memcpy (&data->bps[data->count], bps, pages *sizeof (*data->bps)); + data->count += pages; +} + +static void amdgpu_virt_ras_reserve_bps (struct amdgpu_device *adev) { +struct amdgpu_virt *virt = &adev->virt; +struct virt_ras_err_handler_data *data = virt->virt_eh_data; +struct amdgpu_bo *bo =NULL ; +uint64_t bp; +int i; + +if (!data) +return ; + +for (i = data->last_reserved; i < data->count; i++) { + bp = data->bps[i].retired_page; + +/* There are two cases of reserve error should be ignored: +* 1) a ras bad page has been allocated (used by someone); +* 2) a ras bad page has been reserved (duplicate error injection
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +3
-0+* for one page); +*/ + if (amdgpu_bo_create_kernel_at(adev, bp <<
AMDGPU_GPU_PAGE_SHIFT,drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +4
-0+ AMDGPU_GPU_PAGE_SIZE, +AMDGPU_GEM_DOMAIN_VRAM , + &bo, NULL)) + DRM_WARN("RAS WARN: reserve vram for retiredQuoted 1 line
page %llx fail\n", bp);
[Tao] It's better to distinguish the message from the one in bare mental RAS.
Quoted 37 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +34
-0+ + data->bps_bo[i] = bo; + data->last_reserved = i +1 ; + bo =NULL ; + } +} + +static bool amdgpu_virt_check_bad_page (struct amdgpu_device *adev, +uint64_t retired_page) +{ +struct amdgpu_virt *virt = &adev->virt; +struct virt_ras_err_handler_data *data = virt->virt_eh_data; +int i; + +if (!data) +return true; + +for (i =0 ; i < data->count; i++) +if (retired_page == data->bps[i].retired_page) +return true; + +return false; +} + +static void amdgpu_virt_add_bad_page (struct amdgpu_device *adev, +uint64_t bp_block_offset,uint32_t bp_block_size) { +struct eeprom_table_record bp; +uint64_t retired_page; +uint32_t bp_idx, bp_cnt; + +if (bp_block_size) { + bp_cnt = bp_block_size /sizeof (uint64_t ); +for (bp_idx =0 ; bp_idx < bp_cnt; bp_idx++) { + retired_page = *(uint64_t *)(adev-Quoted 1 line
fw_vram_usage.va +
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +1
-0+ bp_block_offset + bp_idx *
sizeof(uint64_t));drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c int amdgpu_virt_fw_reserve_get_checksum(void *obj,
Quoted 3 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +3
-0+ bp.retired_page = retired_page; + + if (amdgpu_virt_check_bad_page(adev,
retired_page))Quoted 68 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +11
-0+continue ; + +amdgpu_virt_ras_add_bps (adev, &bp,1 ); + +amdgpu_virt_ras_reserve_bps (adev); + } + } +} +void amdgpu_virt_init_data_exchange (struct amdgpu_device *adev) {uint32_t pf2vf_size =0 ;uint32_t checksum =0 ;uint32_t checkval;char *str; +uint64_t bp_block_offset =0 ; +uint32_t bp_block_size =0 ; adev->virt.fw_reserve.p_pf2vf =NULL ; adev->virt.fw_reserve.p_vf2pf = NULL; @@ -275,6 +428,17 @@ voidamdgpu_virt_init_data_exchange(struct
amdgpu_device*adev)drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +2
-0/* pf2vf message must be in 4K */ if (pf2vf_size >0 && pf2vf_size <4096 ) { +if (adev->virt.fw_reserve.p_pf2vf->version ==2 ) { + struct amdgim_pf2vf_info_v2 *pf2vf_v2 =NULL;
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +2
-0+ + pf2vf_v2 = (struct amdgim_pf2vf_info_v2*)adev->virt.fw_reserve.p_pf2vf;
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +1
-0+ bp_block_offset = ((uint64_t)pf2vf_v2-Quoted 1 line
bp_block_offset_L&0xFFFFFFFF) |drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +1
-0+ ((((uint64_t)pf2vf_v2-Quoted 1 line
bp_block_offset_H) << 32) &0xFFFFFFFF00000000);drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +3
-0+ bp_block_size = pf2vf_v2->bp_block_size; + + amdgpu_virt_add_bad_page(adev,
bp_block_offset,bp_block_size);drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +2
-0+ } + checkval = amdgpu_virt_fw_reserve_get_checksum( adev->virt.fw_reserve.p_pf2vf, pf2vf_size, adev->virt.fw_reserve.checksum_key,checksum); diff --git
a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.hb/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
old mode 100644
new mode 100755
indexb90e822cebd7..96d84f036e96drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +8
-2@@ -143,19 +143,27 @@ struct amdgim_pf2vf_info_v2 {uint32_t vce_enc_max_pixels_count;/* 16x16 pixels/sec, codec independent */ uint32_t vce_enc_max_bandwidth; +/* Bad pages block position in BYTE */ +uint32_t bp_block_offset_L; +uint32_t bp_block_offset_H; +/* Bad pages block size in BYTE */ +uint32_t bp_block_size;/* MEC FW position in kb from the start of VF visible frame buffer */ -uint64_t mecfw_kboffset; +uint32_t mecfw_kboffset_L; +uint32_t mecfw_kboffset_H;/* MEC FW size in KB */ uint32_t mecfw_ksize;/* UVD FW position in kb from the start of VF visible frame buffer */ -uint64_t uvdfw_kboffset; +uint32_t uvdfw_kboffset_L;drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h struct amdgim_pf2vf_info_v2 { · line 158
Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +3
-2+uint32_t uvdfw_kboffset_H;/* UVD FW size in KB */ uint32_t uvdfw_ksize;/* VCE FW position in kb from the start of VF visible frame buffer */ -uint64_t vcefw_kboffset; +uint32_t vcefw_kboffset_L; +uint32_t vcefw_kboffset_H;/* VCE FW size in KB */ uint32_t vcefw_ksize; - uint32_t reserved[AMDGIM_GET_STRUCTURE_RESERVED_SIZE(256,0,
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
Quoted 3 lines
0, (9 +
sizeof(struct amd_sriov_msg_pf2vf_info_header)/sizeof(uint32_t)), 3)];drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +1
-0+ uint32_t reserved[AMDGIM_GET_STRUCTURE_RESERVED_SIZE(256,0,
Quoted 44 lines
0, (18 +
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +13
-0+sizeof(struct amd_sriov_msg_pf2vf_info_header)/sizeof(uint32_t)), +0)]; }__aligned(4) ;@@ -254,6 +262,17 @@ typedef struct amdgim_vf2pf_info_v2 amdgim_vf2pf_info ; } \ }while (0 ) +struct virt_ras_err_handler_data { +/* point to bad page records array */ +struct eeprom_table_record *bps; +/* point to reserved bo array */ +struct amdgpu_bo **bps_bo; +/* the count of entries */ +int count; +/* last reserved entry's index + 1 */ +int last_reserved; +}; +/* GPU virtualization */ struct amdgpu_virt {drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +1
-0uint32_t caps;@@ -272,6 +291,7 @@ struct amdgpu_virt {uint32_t reg_access_mode;int req_init_data_ver;bool tdr_debug; +struct virt_ras_err_handler_data *virt_eh_data; };#define amdgpu_sriov_enabled (adev) \@@ -323,6 +343,8 @@ void amdgpu_virt_free_mm_table(struct
amdgpu_device*adev); int amdgpu_virt_fw_reserve_get_checksum(void *obj, unsigned longobj_size,drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h +3
-0unsigned int key, unsigned int chksum); +void amdgpu_release_virt_err_handler_data (struct amdgpu_device +*adev);int amdgpu_virt_init_err_handler_data (struct amdgpu_device +*adev);void amdgpu_virt_init_data_exchange (struct amdgpu_device *adev);void
amdgpu_detect_virtualization(struct amdgpu_device *adev);
Quoted 2 lines
Quoted 2 lines
--
2.25.1
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx