drm/amdgpu: Update invalid PTE flag setting
Subsystems (2)
PATCH v4 1/1 drm/amdgpu: Update invalid PTE flag setting
Update the invalid PTE flag setting with TF enabled. This is to ensure, in addition to transitioning the retry fault to a no-retry fault, it also causes the wavefront to enter the trap handler. With the current setting, the fault only transitions to a no-retry fault. Additionally, have 2 sets of invalid PTE settings, one for TF enabled, the other for TF disabled. The setting with TF disabled, doesn't work with TF enabled.
---v1->v2:
- Update handling according to Christian's feedback.
v2->v3:
- Remove ASIC specific callback (Felix).
v3->v4:
- Add noretry flag to amdgpu->gmc. This allows to set ASIC specific flags.
drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h | 2 ++ drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h | 6 +++++ drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c | 31 +++++++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c | 1 + drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c | 1 + drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c | 1 + drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c | 1 + drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c | 1 + 9 files changed, 45 insertions(+), 1 deletion(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h +2 -0
@@ -331,6 +331,8 @@ struct amdgpu_gmc {u64 VM_CONTEXT_PAGE_TABLE_END_ADDR_LO32 [16 ];u64 VM_CONTEXT_PAGE_TABLE_END_ADDR_HI32 [16 ];u64 MC_VM_MX_L1_TLB_CNTL ; + +u64 noretry_flags; };#define amdgpu_gmc_flush_gpu_tlb (adev, vmid, vmhub, type) ((adev)->gmc.gmc_funcs->flush_gpu_tlb((adev), (vmid), (vmhub), (type)))
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +1 -1
@@ -2604,7 +2604,7 @@ bool amdgpu_vm_handle_fault(struct amdgpu_device *adev, u32 pasid,/* Intentionally setting invalid PTE flag * combination to force a no-retry-fault */ - flags =AMDGPU_PTE_SNOOPED |AMDGPU_PTE_PRT ; + flags =AMDGPU_VM_NORETRY_FLAGS ; value =0 ; }else if (amdgpu_vm_fault_stop ==AMDGPU_VM_FAULT_STOP_NEVER ) {/* Redirect the access to the dummy page */
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h +6 -0
@@ -84,7 +84,13 @@ struct amdgpu_mem_stats;/* PDE Block Fragment Size for VEGA10 */ #define AMDGPU_PDE_BFS (a) ((uint64_t)a << 59) +/* Flag combination to set no-retry with TF disabled */ +#define AMDGPU_VM_NORETRY_FLAGS (AMDGPU_PTE_EXECUTABLE | AMDGPU_PDE_PTE | \ + AMDGPU_PTE_TF) +/* Flag combination to set no-retry with TF enabled */ +#define AMDGPU_VM_NORETRY_FLAGS_TF (AMDGPU_PTE_VALID | AMDGPU_PTE_SYSTEM | \ + AMDGPU_PTE_PRT)/* For GFX9 */ #define AMDGPU_PTE_MTYPE_VG10 (a) ((uint64_t)(a) << 57)#define AMDGPU_PTE_MTYPE_VG10_MASK AMDGPU_PTE_MTYPE_VG10(3ULL)
drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c +31 -0
@@ -778,6 +778,27 @@ int amdgpu_vm_pde_update(struct amdgpu_vm_update_params *params, 1, 0, flags); } +/** +* amdgpu_vm_pte_update_noretry_flags - Update PTE no-retry flags +* +* @adev - amdgpu_device pointer +* @flags: pointer to PTE flags +* +* Update PTE no-retry flags when TF is enabled. +*/ +static void amdgpu_vm_pte_update_noretry_flags (struct amdgpu_device *adev, +uint64_t *flags) +{ +/* +* Update no-retry flags with the corresponding TF +* no-retry combination. +*/ +if ((*flags &AMDGPU_VM_NORETRY_FLAGS ) ==AMDGPU_VM_NORETRY_FLAGS ) { + *flags &= ~AMDGPU_VM_NORETRY_FLAGS ; + *flags |= adev->gmc.noretry_flags; + } +} +/* * amdgpu_vm_pte_update_flags - figure out flags for PTE updates *
@@ -804,6 +825,16 @@ static void amdgpu_vm_pte_update_flags(struct amdgpu_vm_update_params *params, flags |=AMDGPU_PTE_EXECUTABLE ; } +/* +* Update no-retry flags to use the no-retry flag combination +* with TF enabled. The AMDGPU_VM_NORETRY_FLAGS flag combination +* does not work when TF is enabled. So, replace them with +* AMDGPU_VM_NORETRY_FLAGS_TF flag combination which works for +* all cases. +*/ +if (level ==AMDGPU_VM_PTB ) +amdgpu_vm_pte_update_noretry_flags (adev, &flags); +/* APUs mapping system memory may need different MTYPEs on different * NUMA nodes. Only do this for contiguous ranges that can be assumed * to be on the same NUMA node.
drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c +1 -0
@@ -751,6 +751,7 @@ static int gmc_v10_0_early_init(void *handle) adev->gmc.private_aperture_start =0x1000000000000000ULL ; adev->gmc.private_aperture_end = adev->gmc.private_aperture_start + (4ULL <<30 ) -1 ; + adev->gmc.noretry_flags =AMDGPU_VM_NORETRY_FLAGS_TF ;return 0 ; }
drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c +1 -0
@@ -651,6 +651,7 @@ static int gmc_v11_0_early_init(void *handle) adev->gmc.private_aperture_start =0x1000000000000000ULL ; adev->gmc.private_aperture_end = adev->gmc.private_aperture_start + (4ULL <<30 ) -1 ; + adev->gmc.noretry_flags =AMDGPU_VM_NORETRY_FLAGS_TF ;return 0 ; }
drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c +1 -0
@@ -942,6 +942,7 @@ static int gmc_v7_0_early_init(void *handle) adev->gmc.shared_aperture_end +1 ; adev->gmc.private_aperture_end = adev->gmc.private_aperture_start + (4ULL <<30 ) -1 ; + adev->gmc.noretry_flags =AMDGPU_VM_NORETRY_FLAGS_TF ;return 0 ; }
drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c +1 -0
@@ -1056,6 +1056,7 @@ static int gmc_v8_0_early_init(void *handle) adev->gmc.shared_aperture_end +1 ; adev->gmc.private_aperture_end = adev->gmc.private_aperture_start + (4ULL <<30 ) -1 ; + adev->gmc.noretry_flags =AMDGPU_VM_NORETRY_FLAGS_TF ;return 0 ; }
drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +1 -0
@@ -1622,6 +1622,7 @@ static int gmc_v9_0_early_init(void *handle) adev->gmc.private_aperture_start =0x1000000000000000ULL ; adev->gmc.private_aperture_end = adev->gmc.private_aperture_start + (4ULL <<30 ) -1 ; + adev->gmc.noretry_flags =AMDGPU_VM_NORETRY_FLAGS_TF ;return 0 ; }
Signature · 1 line
2.35.1
Quoted 10 lines · Mukul Joshi
Update the invalid PTE flag setting with TF enabled. This is to ensure, in addition to transitioning the retry fault to a no-retry fault, it also causes the wavefront to enter the trap handler. With the current setting, the fault only transitions to a no-retry fault. Additionally, have 2 sets of invalid PTE settings, one for TF enabled, the other for TF disabled. The setting with TF disabled, doesn't work with TF enabled.
Signed-off-by Mukul Joshi <[email protected]>
Quoted 175 lines
---
v1->v2:
- Update handling according to Christian's feedback.
v2->v3:
- Remove ASIC specific callback (Felix).
v3->v4:
- Add noretry flag toamdgpu->gmc. This allows to set ASIC specific flags.drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h | 2 ++ drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h | 6 +++++ drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c | 31 +++++++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c | 1 + drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c | 1 + drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c | 1 + drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c | 1 + drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c | 1 + 9 files changed, 45 insertions(+), 1 deletion(-)drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h +2
-0@@ -331,6 +331,8 @@ struct amdgpu_gmc {u64 VM_CONTEXT_PAGE_TABLE_END_ADDR_LO32 [16 ];u64 VM_CONTEXT_PAGE_TABLE_END_ADDR_HI32 [16 ];u64 MC_VM_MX_L1_TLB_CNTL ; + +u64 noretry_flags; };#define amdgpu_gmc_flush_gpu_tlb (adev, vmid, vmhub, type) ((adev)->gmc.gmc_funcs->flush_gpu_tlb((adev), (vmid), (vmhub), (type)))drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +1
-1@@ -2604,7 +2604,7 @@ bool amdgpu_vm_handle_fault(struct amdgpu_device *adev, u32 pasid,/* Intentionally setting invalid PTE flag * combination to force a no-retry-fault */ - flags =AMDGPU_PTE_SNOOPED |AMDGPU_PTE_PRT ; + flags =AMDGPU_VM_NORETRY_FLAGS ; value =0 ; }else if (amdgpu_vm_fault_stop ==AMDGPU_VM_FAULT_STOP_NEVER ) {/* Redirect the access to the dummy page */ drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h +6
-0@@ -84,7 +84,13 @@ struct amdgpu_mem_stats;/* PDE Block Fragment Size for VEGA10 */ #define AMDGPU_PDE_BFS (a) ((uint64_t)a << 59) +/* Flag combination to set no-retry with TF disabled */ +#define AMDGPU_VM_NORETRY_FLAGS (AMDGPU_PTE_EXECUTABLE | AMDGPU_PDE_PTE | \ + AMDGPU_PTE_TF) +/* Flag combination to set no-retry with TF enabled */ +#define AMDGPU_VM_NORETRY_FLAGS_TF (AMDGPU_PTE_VALID | AMDGPU_PTE_SYSTEM | \ + AMDGPU_PTE_PRT)/* For GFX9 */ #define AMDGPU_PTE_MTYPE_VG10 (a) ((uint64_t)(a) << 57)#define AMDGPU_PTE_MTYPE_VG10_MASK AMDGPU_PTE_MTYPE_VG10(3ULL)drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c +31
-0@@ -778,6 +778,27 @@ int amdgpu_vm_pde_update(struct amdgpu_vm_update_params *params, 1, 0, flags); } +/** +* amdgpu_vm_pte_update_noretry_flags - Update PTE no-retry flags +* +* @adev - amdgpu_device pointer +* @flags: pointer to PTE flags +* +* Update PTE no-retry flags when TF is enabled. +*/ +static void amdgpu_vm_pte_update_noretry_flags (struct amdgpu_device *adev, +uint64_t *flags) +{ +/* +* Update no-retry flags with the corresponding TF +* no-retry combination. +*/ +if ((*flags &AMDGPU_VM_NORETRY_FLAGS ) ==AMDGPU_VM_NORETRY_FLAGS ) { + *flags &= ~AMDGPU_VM_NORETRY_FLAGS ; + *flags |= adev->gmc.noretry_flags; + } +} +/* * amdgpu_vm_pte_update_flags - figure out flags for PTE updates * @@ -804,6 +825,16 @@ static void amdgpu_vm_pte_update_flags(struct amdgpu_vm_update_params *params, flags |=AMDGPU_PTE_EXECUTABLE ; } +/* +* Update no-retry flags to use the no-retry flag combination +* with TF enabled. The AMDGPU_VM_NORETRY_FLAGS flag combination +* does not work when TF is enabled. So, replace them with +* AMDGPU_VM_NORETRY_FLAGS_TF flag combination which works for +* all cases. +*/ +if (level ==AMDGPU_VM_PTB ) +amdgpu_vm_pte_update_noretry_flags (adev, &flags); +/* APUs mapping system memory may need different MTYPEs on different * NUMA nodes. Only do this for contiguous ranges that can be assumed * to be on the same NUMA node. drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c +1
-0@@ -751,6 +751,7 @@ static int gmc_v10_0_early_init(void *handle) adev->gmc.private_aperture_start =0x1000000000000000ULL ; adev->gmc.private_aperture_end = adev->gmc.private_aperture_start + (4ULL <<30 ) -1 ; + adev->gmc.noretry_flags =AMDGPU_VM_NORETRY_FLAGS_TF ;return 0 ; }drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c +1
-0@@ -651,6 +651,7 @@ static int gmc_v11_0_early_init(void *handle) adev->gmc.private_aperture_start =0x1000000000000000ULL ; adev->gmc.private_aperture_end = adev->gmc.private_aperture_start + (4ULL <<30 ) -1 ; + adev->gmc.noretry_flags =AMDGPU_VM_NORETRY_FLAGS_TF ;return 0 ; }drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c +1
-0@@ -942,6 +942,7 @@ static int gmc_v7_0_early_init(void *handle) adev->gmc.shared_aperture_end +1 ; adev->gmc.private_aperture_end = adev->gmc.private_aperture_start + (4ULL <<30 ) -1 ; + adev->gmc.noretry_flags =AMDGPU_VM_NORETRY_FLAGS_TF ;return 0 ; }drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c +1
-0@@ -1056,6 +1056,7 @@ static int gmc_v8_0_early_init(void *handle) adev->gmc.shared_aperture_end +1 ; adev->gmc.private_aperture_end = adev->gmc.private_aperture_start + (4ULL <<30 ) -1 ; + adev->gmc.noretry_flags =AMDGPU_VM_NORETRY_FLAGS_TF ;return 0 ; }drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +1
-0@@ -1622,6 +1622,7 @@ static int gmc_v9_0_early_init(void *handle) adev->gmc.private_aperture_start =0x1000000000000000ULL ; adev->gmc.private_aperture_end = adev->gmc.private_aperture_start + (4ULL <<30 ) -1 ; + adev->gmc.noretry_flags =AMDGPU_VM_NORETRY_FLAGS_TF ;return 0 ; }
Am 19.06.23 um 19:38 schrieb Mukul Joshi:
Quoted 10 lines
Update the invalid PTE flag setting with TF enabled. This is to ensure, in addition to transitioning the retry fault to a no-retry fault, it also causes the wavefront to enter the trap handler. With the current setting, the fault only transitions to a no-retry fault. Additionally, have 2 sets of invalid PTE settings, one for TF enabled, the other for TF disabled. The setting with TF disabled, doesn't work with TF enabled.
Signed-off-by Mukul Joshi <[email protected]>
Quoted 175 lines
---
v1->v2:
- Update handling according to Christian's feedback.
v2->v3:
- Remove ASIC specific callback (Felix).
v3->v4:
- Add noretry flag toamdgpu->gmc. This allows to set ASIC specific flags.drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h | 2 ++ drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 2 +- drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h | 6 +++++ drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c | 31 +++++++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c | 1 + drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c | 1 + drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c | 1 + drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c | 1 + drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c | 1 + 9 files changed, 45 insertions(+), 1 deletion(-)drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h +2
-0@@ -331,6 +331,8 @@ struct amdgpu_gmc {u64 VM_CONTEXT_PAGE_TABLE_END_ADDR_LO32 [16 ];u64 VM_CONTEXT_PAGE_TABLE_END_ADDR_HI32 [16 ];u64 MC_VM_MX_L1_TLB_CNTL ; + +u64 noretry_flags; };#define amdgpu_gmc_flush_gpu_tlb (adev, vmid, vmhub, type) ((adev)->gmc.gmc_funcs->flush_gpu_tlb((adev), (vmid), (vmhub), (type)))drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +1
-1@@ -2604,7 +2604,7 @@ bool amdgpu_vm_handle_fault(struct amdgpu_device *adev, u32 pasid,/* Intentionally setting invalid PTE flag * combination to force a no-retry-fault */ - flags =AMDGPU_PTE_SNOOPED |AMDGPU_PTE_PRT ; + flags =AMDGPU_VM_NORETRY_FLAGS ; value =0 ; }else if (amdgpu_vm_fault_stop ==AMDGPU_VM_FAULT_STOP_NEVER ) {/* Redirect the access to the dummy page */ drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h +6
-0@@ -84,7 +84,13 @@ struct amdgpu_mem_stats;/* PDE Block Fragment Size for VEGA10 */ #define AMDGPU_PDE_BFS (a) ((uint64_t)a << 59) +/* Flag combination to set no-retry with TF disabled */ +#define AMDGPU_VM_NORETRY_FLAGS (AMDGPU_PTE_EXECUTABLE | AMDGPU_PDE_PTE | \ + AMDGPU_PTE_TF) +/* Flag combination to set no-retry with TF enabled */ +#define AMDGPU_VM_NORETRY_FLAGS_TF (AMDGPU_PTE_VALID | AMDGPU_PTE_SYSTEM | \ + AMDGPU_PTE_PRT)/* For GFX9 */ #define AMDGPU_PTE_MTYPE_VG10 (a) ((uint64_t)(a) << 57)#define AMDGPU_PTE_MTYPE_VG10_MASK AMDGPU_PTE_MTYPE_VG10(3ULL)drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c +31
-0@@ -778,6 +778,27 @@ int amdgpu_vm_pde_update(struct amdgpu_vm_update_params *params, 1, 0, flags); } +/** +* amdgpu_vm_pte_update_noretry_flags - Update PTE no-retry flags +* +* @adev - amdgpu_device pointer +* @flags: pointer to PTE flags +* +* Update PTE no-retry flags when TF is enabled. +*/ +static void amdgpu_vm_pte_update_noretry_flags (struct amdgpu_device *adev, +uint64_t *flags) +{ +/* +* Update no-retry flags with the corresponding TF +* no-retry combination. +*/ +if ((*flags &AMDGPU_VM_NORETRY_FLAGS ) ==AMDGPU_VM_NORETRY_FLAGS ) { + *flags &= ~AMDGPU_VM_NORETRY_FLAGS ; + *flags |= adev->gmc.noretry_flags; + } +} +/* * amdgpu_vm_pte_update_flags - figure out flags for PTE updates * @@ -804,6 +825,16 @@ static void amdgpu_vm_pte_update_flags(struct amdgpu_vm_update_params *params, flags |=AMDGPU_PTE_EXECUTABLE ; } +/* +* Update no-retry flags to use the no-retry flag combination +* with TF enabled. The AMDGPU_VM_NORETRY_FLAGS flag combination +* does not work when TF is enabled. So, replace them with +* AMDGPU_VM_NORETRY_FLAGS_TF flag combination which works for +* all cases. +*/ +if (level ==AMDGPU_VM_PTB ) +amdgpu_vm_pte_update_noretry_flags (adev, &flags); +/* APUs mapping system memory may need different MTYPEs on different * NUMA nodes. Only do this for contiguous ranges that can be assumed * to be on the same NUMA node. drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c +1
-0@@ -751,6 +751,7 @@ static int gmc_v10_0_early_init(void *handle) adev->gmc.private_aperture_start =0x1000000000000000ULL ; adev->gmc.private_aperture_end = adev->gmc.private_aperture_start + (4ULL <<30 ) -1 ; + adev->gmc.noretry_flags =AMDGPU_VM_NORETRY_FLAGS_TF ;return 0 ; }drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c +1
-0@@ -651,6 +651,7 @@ static int gmc_v11_0_early_init(void *handle) adev->gmc.private_aperture_start =0x1000000000000000ULL ; adev->gmc.private_aperture_end = adev->gmc.private_aperture_start + (4ULL <<30 ) -1 ; + adev->gmc.noretry_flags =AMDGPU_VM_NORETRY_FLAGS_TF ;return 0 ; }drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c +1
-0@@ -942,6 +942,7 @@ static int gmc_v7_0_early_init(void *handle) adev->gmc.shared_aperture_end +1 ; adev->gmc.private_aperture_end = adev->gmc.private_aperture_start + (4ULL <<30 ) -1 ; + adev->gmc.noretry_flags =AMDGPU_VM_NORETRY_FLAGS_TF ;return 0 ; }drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c +1
-0@@ -1056,6 +1056,7 @@ static int gmc_v8_0_early_init(void *handle) adev->gmc.shared_aperture_end +1 ; adev->gmc.private_aperture_end = adev->gmc.private_aperture_start + (4ULL <<30 ) -1 ; + adev->gmc.noretry_flags =AMDGPU_VM_NORETRY_FLAGS_TF ;return 0 ; }drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +1
-0@@ -1622,6 +1622,7 @@ static int gmc_v9_0_early_init(void *handle) adev->gmc.private_aperture_start =0x1000000000000000ULL ; adev->gmc.private_aperture_end = adev->gmc.private_aperture_start + (4ULL <<30 ) -1 ; + adev->gmc.noretry_flags =AMDGPU_VM_NORETRY_FLAGS_TF ;return 0 ; }