drm/amdgpu: Avoid reclaim fs while eviction lock
Subsystems (3)
PATCH 1/7 drm/amdgpu: Avoid reclaim fs while eviction lock
[Why]
Avoid reclaim filesystem while eviction lock is held called from MMU notifier.
[How]
Setting PF_MEMALLOC_NOFS flags while eviction mutex is locked. Using memalloc_nofs_save / memalloc_nofs_restore API.
--- drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 40 +++++++++++++++++++++----- drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h | 6 +++- 2 files changed, 38 insertions(+), 8 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +33 -7
@@ -82,6 +82,32 @@ struct amdgpu_prt_cb {struct dma_fence_cb cb; }; +/** +* vm eviction_lock can be taken in MMU notifiers. Make sure no reclaim-FS +* happens while holding this lock anywhere to prevent deadlocks when +* an MMU notifier runs in reclaim-FS context. +*/ +static inline void amdgpu_vm_eviction_lock (struct amdgpu_vm *vm) +{ +mutex_lock (&vm->eviction_lock); + vm->saved_flags =memalloc_nofs_save (); +} + +static inline int amdgpu_vm_eviction_trylock (struct amdgpu_vm *vm) +{ +if (mutex_trylock (&vm->eviction_lock)) { + vm->saved_flags =memalloc_nofs_save (); +return 1 ; + } +return 0 ; +} + +static inline void amdgpu_vm_eviction_unlock (struct amdgpu_vm *vm) +{ +memalloc_nofs_restore (vm->saved_flags); +mutex_unlock (&vm->eviction_lock); +} +/** * amdgpu_vm_level_shift - return the addr shift for each level *
@@ -678,9 +704,9 @@ int amdgpu_vm_validate_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm, } } -mutex_lock (&vm->eviction_lock); +amdgpu_vm_eviction_lock (vm); vm->evicting = false; -mutex_unlock (&vm->eviction_lock); +amdgpu_vm_eviction_unlock (vm);return 0 ; }
@@ -1559,7 +1585,7 @@ static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,if (!(flags &AMDGPU_PTE_VALID )) owner =AMDGPU_FENCE_OWNER_KFD ; -mutex_lock (&vm->eviction_lock); +amdgpu_vm_eviction_lock (vm);if (vm->evicting) { r = -EBUSY ; goto error_unlock;
@@ -1576,7 +1602,7 @@ static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev, r = vm->update_funcs->commit (¶ms, fence); error_unlock: -mutex_unlock (&vm->eviction_lock); +amdgpu_vm_eviction_unlock (vm);return r; }
@@ -2537,18 +2563,18 @@ bool amdgpu_vm_evictable(struct amdgpu_bo *bo)return false;/* Try to block ongoing updates */ -if (!mutex_trylock (&bo_base->vm->eviction_lock)) +if (!amdgpu_vm_eviction_trylock (bo_base->vm))return false;/* Don't evict VM page tables while they are updated */ if (!dma_fence_is_signaled (bo_base->vm->last_direct) || !dma_fence_is_signaled (bo_base->vm->last_delayed)) { -mutex_unlock (&bo_base->vm->eviction_lock); +amdgpu_vm_eviction_unlock (bo_base->vm);return false; } bo_base->vm->evicting = true; -mutex_unlock (&bo_base->vm->eviction_lock); +amdgpu_vm_eviction_unlock (bo_base->vm);return true; }
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h +5 -1
@@ -30,6 +30,7 @@#include <drm/gpu_scheduler.h> #include <drm/drm_file.h> #include <drm/ttm/ttm_bo_driver.h> +#include <linux/sched/mm.h> #include "amdgpu_sync.h" #include "amdgpu_ring.h"
@@ -242,9 +243,12 @@ struct amdgpu_vm {/* tree of virtual addresses mapped */ struct rb_root_cached va; -/* Lock to prevent eviction while we are updating page tables */ +/* Lock to prevent eviction while we are updating page tables +* use vm_eviction_lock/unlock(vm) +*/ struct mutex eviction_lock;bool evicting; +unsigned int saved_flags;/* BOs who needs a validation */ struct list_head evicted;
Signature · 6 lines
2.17.1 _______________________________________________ amd-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/amd-gfx
PATCH 2/7 drm/amdgpu: export function to flush TLB via pasid
This can be used directly from amdgpu and amdkfd to invalidate TLB through pasid.
It supports gmc v7, v8, v9 and v10.
--- drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h | 6 ++ drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c | 87 ++++++++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c | 33 +++++++++ drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c | 34 ++++++++++ drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c | 89 +++++++++++++++++++++++++ 5 files changed, 249 insertions(+)
drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h +6 -0
@@ -92,6 +92,9 @@ struct amdgpu_gmc_funcs {/* flush the vm tlb via mmio */ void (*flush_gpu_tlb)(struct amdgpu_device *adev,uint32_t vmid,uint32_t vmhub,uint32_t flush_type); +/* flush the vm tlb via pasid */ +int (*flush_gpu_tlb_pasid)(struct amdgpu_device *adev,uint16_t pasid, +uint32_t flush_type,bool all_hub);/* flush the vm tlb via ring */ uint64_t (*emit_flush_gpu_tlb)(struct amdgpu_ring *ring,unsigned vmid,uint64_t pd_addr);
@@ -216,6 +219,9 @@ struct amdgpu_gmc { };#define amdgpu_gmc_flush_gpu_tlb (adev, vmid, vmhub, type) ((adev)->gmc.gmc_funcs->flush_gpu_tlb((adev), (vmid), (vmhub), (type))) +#define amdgpu_gmc_flush_gpu_tlb_pasid (adev, pasid, type, allhub) \ + ((adev)->gmc.gmc_funcs->flush_gpu_tlb_pasid \ + ((adev), (pasid), (type), (allhub)))#define amdgpu_gmc_emit_flush_gpu_tlb (r, vmid, addr) (r)->adev->gmc.gmc_funcs->emit_flush_gpu_tlb((r), (vmid), (addr))#define amdgpu_gmc_emit_pasid_mapping (r, vmid, pasid) (r)->adev->gmc.gmc_funcs->emit_pasid_mapping((r), (vmid), (pasid))#define amdgpu_gmc_map_mtype (adev, flags) (adev)->gmc.gmc_funcs->map_mtype((adev),(flags))
drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c +87 -0
117 lines not on this page · raw message
drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c +33 -0
@@ -418,6 +418,38 @@ static int gmc_v7_0_mc_init(struct amdgpu_device *adev)return 0 ; } +/** +* gmc_v7_0_flush_gpu_tlb_pasid - tlb flush via pasid +* +* @adev: amdgpu_device pointer +* @pasid: pasid to be flush +* +* Flush the TLB for the requested pasid. +*/ +static int gmc_v7_0_flush_gpu_tlb_pasid (struct amdgpu_device *adev, +uint16_t pasid,uint32_t flush_type, +bool all_hub) +{ +int vmid; +unsigned int tmp; + +if (adev->in_gpu_reset) +return -EIO ; + +for (vmid =1 ; vmid <16 ; vmid++) { + + tmp =RREG32 (mmATC_VMID0_PASID_MAPPING + vmid); +if ((tmp &ATC_VMID0_PASID_MAPPING__VALID_MASK ) && + (tmp &ATC_VMID0_PASID_MAPPING__PASID_MASK ) == pasid) { +WREG32 (mmVM_INVALIDATE_REQUEST,1 << vmid); +RREG32 (mmVM_INVALIDATE_RESPONSE); +break ; + } + } + +return 0 ; +} +/* * GART * VMID 0 is the physical GPU addresses as used by the kernel.
@@ -1333,6 +1365,7 @@ static const struct amd_ip_funcs gmc_v7_0_ip_funcs = {static const struct amdgpu_gmc_funcs gmc_v7_0_gmc_funcs = { .flush_gpu_tlb = gmc_v7_0_flush_gpu_tlb, + .flush_gpu_tlb_pasid = gmc_v7_0_flush_gpu_tlb_pasid, .emit_flush_gpu_tlb = gmc_v7_0_emit_flush_gpu_tlb, .emit_pasid_mapping = gmc_v7_0_emit_pasid_mapping, .set_prt = gmc_v7_0_set_prt,
drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c +34 -0
@@ -620,6 +620,39 @@ static int gmc_v8_0_mc_init(struct amdgpu_device *adev)return 0 ; } +/** +* gmc_v8_0_flush_gpu_tlb_pasid - tlb flush via pasid +* +* @adev: amdgpu_device pointer +* @pasid: pasid to be flush +* +* Flush the TLB for the requested pasid. +*/ +static int gmc_v8_0_flush_gpu_tlb_pasid (struct amdgpu_device *adev, +uint16_t pasid,uint32_t flush_type, +bool all_hub) +{ +int vmid; +unsigned int tmp; + +if (adev->in_gpu_reset) +return -EIO ; + +for (vmid =1 ; vmid <16 ; vmid++) { + + tmp =RREG32 (mmATC_VMID0_PASID_MAPPING + vmid); +if ((tmp &ATC_VMID0_PASID_MAPPING__VALID_MASK ) && + (tmp &ATC_VMID0_PASID_MAPPING__PASID_MASK ) == pasid) { +WREG32 (mmVM_INVALIDATE_REQUEST,1 << vmid); +RREG32 (mmVM_INVALIDATE_RESPONSE); +break ; + } + } + +return 0 ; + +} +/* * GART * VMID 0 is the physical GPU addresses as used by the kernel.
@@ -1700,6 +1733,7 @@ static const struct amd_ip_funcs gmc_v8_0_ip_funcs = {static const struct amdgpu_gmc_funcs gmc_v8_0_gmc_funcs = { .flush_gpu_tlb = gmc_v8_0_flush_gpu_tlb, + .flush_gpu_tlb_pasid = gmc_v8_0_flush_gpu_tlb_pasid, .emit_flush_gpu_tlb = gmc_v8_0_emit_flush_gpu_tlb, .emit_pasid_mapping = gmc_v8_0_emit_pasid_mapping, .set_prt = gmc_v8_0_set_prt,
drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +89 -0
117 lines not on this page · raw message
Signature · 6 lines
2.17.1 _______________________________________________ amd-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/amd-gfx
PATCH 3/7 drm/amdgpu: GPU TLB flush API moved to amdgpu_amdkfd
In reply to Alex Sierra[Why]
TLB flush method has been deprecated using kfd2kgd interface. This implementation is now on the amdgpu_amdkfd API.
[How]
TLB flush functions now implemented in amdgpu_amdkfd.
--- drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c | 32 ++++++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h | 2 ++ drivers/gpu/drm/amd/amdkfd/kfd_process.c | 8 ++++-- 3 files changed, 39 insertions(+), 3 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +32 -0
@@ -628,6 +628,38 @@ bool amdgpu_amdkfd_is_kfd_vmid(struct amdgpu_device *adev, u32 vmid)return false; } +int amdgpu_amdkfd_flush_gpu_tlb_vmid (struct kgd_dev *kgd,uint16_t vmid) +{ +struct amdgpu_device *adev = (struct amdgpu_device *)kgd; + +if (adev->family ==AMDGPU_FAMILY_AI ) { +int i; + +for (i =0 ; i < adev->num_vmhubs; i++) +amdgpu_gmc_flush_gpu_tlb (adev, vmid, i,0 ); + }else { +amdgpu_gmc_flush_gpu_tlb (adev, vmid,AMDGPU_GFXHUB_0 ,0 ); + } + +return 0 ; +} + +int amdgpu_amdkfd_flush_gpu_tlb_pasid (struct kgd_dev *kgd,uint16_t pasid) +{ +struct amdgpu_device *adev = (struct amdgpu_device *)kgd; +uint32_t flush_type =0 ; +bool all_hub = false; + +if (adev->gmc.xgmi.num_physical_nodes && + adev->asic_type ==CHIP_VEGA20 ) + flush_type =2 ; + +if (adev->family ==AMDGPU_FAMILY_AI ) + all_hub = true; + +return amdgpu_gmc_flush_gpu_tlb_pasid (adev, pasid, flush_type, all_hub); +} +bool amdgpu_amdkfd_have_atomics_support (struct kgd_dev *kgd) {struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h +2 -0
@@ -136,6 +136,8 @@ int amdgpu_amdkfd_submit_ib(struct kgd_dev *kgd, enum kgd_engine_type engine, uint32_t *ib_cmd, uint32_t ib_len);void amdgpu_amdkfd_set_compute_idle (struct kgd_dev *kgd,bool idle);bool amdgpu_amdkfd_have_atomics_support (struct kgd_dev *kgd); +int amdgpu_amdkfd_flush_gpu_tlb_vmid (struct kgd_dev *kgd,uint16_t vmid); +int amdgpu_amdkfd_flush_gpu_tlb_pasid (struct kgd_dev *kgd,uint16_t pasid);bool amdgpu_amdkfd_is_kfd_vmid (struct amdgpu_device *adev,u32 vmid);
drivers/gpu/drm/amd/amdkfd/kfd_process.c +5 -3
@@ -32,6 +32,7 @@#include <linux/mman.h> #include <linux/file.h> #include "amdgpu_amdkfd.h" +#include "amdgpu.h" struct mm_struct ;
@@ -1152,16 +1153,17 @@ int kfd_reserved_mem_mmap(struct kfd_dev *dev, struct kfd_process *process,void kfd_flush_tlb (struct kfd_process_device *pdd) {struct kfd_dev *dev = pdd->dev; -const struct kfd2kgd_calls *f2g = dev->kfd2kgd;if (dev->dqm->sched_policy ==KFD_SCHED_POLICY_NO_HWS ) {/* Nothing to flush until a VMID is assigned, which * only happens when the first queue is created. */ if (pdd->qpd.vmid) - f2g->invalidate_tlbs_vmid (dev->kgd, pdd->qpd.vmid); +amdgpu_amdkfd_flush_gpu_tlb_vmid (dev->kgd, + pdd->qpd.vmid); }else { - f2g->invalidate_tlbs (dev->kgd, pdd->process->pasid); +amdgpu_amdkfd_flush_gpu_tlb_pasid (dev->kgd, + pdd->process->pasid); } }
Signature · 6 lines
2.17.1 _______________________________________________ amd-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/amd-gfx
PATCH 4/7 drm/amdgpu: flush TLB functions removal from kfd2kgd interface
In reply to Alex Sierra[Why]
kfd2kgd interface will be deprecated. This removal only covers TLB invalidation for now. They have been replaced in amdgpu_amdkfd API.
[How]
TLB invalidate functions removed from the different amdkfd_gfx_v* versions.
--- .../drm/amd/amdgpu/amdgpu_amdkfd_arcturus.c | 2 - .../drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.c | 67 ------------- .../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c | 41 -------- .../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c | 41 -------- .../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c | 96 ------------------- .../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.h | 2 - .../gpu/drm/amd/include/kgd_kfd_interface.h | 2 - 7 files changed, 251 deletions(-)
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_arcturus.c +0 -2
@@ -317,7 +317,5 @@ const struct kfd2kgd_calls arcturus_kfd2kgd = { kgd_gfx_v9_get_atc_vmid_pasid_mapping_info, .get_tile_config = kgd_gfx_v9_get_tile_config, .set_vm_context_page_table_base = kgd_set_vm_context_page_table_base, - .invalidate_tlbs = kgd_gfx_v9_invalidate_tlbs, - .invalidate_tlbs_vmid = kgd_gfx_v9_invalidate_tlbs_vmid, .get_hive_id = amdgpu_amdkfd_get_hive_id, };
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.c +0 -67
78 lines not on this page · raw message
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c +0 -41
52 lines not on this page · raw message
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c +0 -41
51 lines not on this page · raw message
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c +0 -96
107 lines not on this page · raw message
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.h +0 -2
@@ -57,7 +57,5 @@ uint32_t kgd_gfx_v9_address_watch_get_offset(struct kgd_dev *kgd,bool kgd_gfx_v9_get_atc_vmid_pasid_mapping_info (struct kgd_dev *kgd,uint8_t vmid,uint16_t *p_pasid); -int kgd_gfx_v9_invalidate_tlbs (struct kgd_dev *kgd,uint16_t pasid); -int kgd_gfx_v9_invalidate_tlbs_vmid (struct kgd_dev *kgd,uint16_t vmid);int kgd_gfx_v9_get_tile_config (struct kgd_dev *kgd,struct tile_config *config);
drivers/gpu/drm/amd/include/kgd_kfd_interface.h +0 -2
@@ -307,8 +307,6 @@ struct kfd2kgd_calls {void (*set_vm_context_page_table_base)(struct kgd_dev *kgd,uint32_t vmid,uint64_t page_table_base); -int (*invalidate_tlbs)(struct kgd_dev *kgd,uint16_t pasid); -int (*invalidate_tlbs_vmid)(struct kgd_dev *kgd,uint16_t vmid);uint32_t (*read_vmid_from_vmfault_reg)(struct kgd_dev *kgd);uint64_t (*get_hive_id)(struct kgd_dev *kgd);
Signature · 6 lines
2.17.1 _______________________________________________ amd-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/amd-gfx
PATCH 5/7 drm/amdgpu: kiq pm4 function implementation for gfx_v9
In reply to Alex SierraFunctions implemented from kiq_pm4_funcs struct members for gfx_v9 version.
--- drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c | 115 ++++++++++++++++++++++++++ 1 file changed, 115 insertions(+)
drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c +115 -0
@@ -739,6 +739,120 @@ static int gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev,static int gfx_v9_0_ras_error_inject (struct amdgpu_device *adev,void *inject_if); +static void gfx_v9_0_kiq_set_resources (struct amdgpu_ring *kiq_ring, +uint64_t queue_mask) +{ +amdgpu_ring_write (kiq_ring,PACKET3 (PACKET3_SET_RESOURCES ,6 )); +amdgpu_ring_write (kiq_ring, +PACKET3_SET_RESOURCES_VMID_MASK (0 ) | +/* vmid_mask:0* queue_type:0 (KIQ) */ +PACKET3_SET_RESOURCES_QUEUE_TYPE (0 )); +amdgpu_ring_write (kiq_ring, +lower_32_bits (queue_mask));/* queue mask lo */ +amdgpu_ring_write (kiq_ring, +upper_32_bits (queue_mask));/* queue mask hi */ +amdgpu_ring_write (kiq_ring,0 );/* gws mask lo */ +amdgpu_ring_write (kiq_ring,0 );/* gws mask hi */ +amdgpu_ring_write (kiq_ring,0 );/* oac mask */ +amdgpu_ring_write (kiq_ring,0 );/* gds heap base:0, gds heap size:0 */ +} + +static void gfx_v9_0_kiq_map_queues (struct amdgpu_ring *kiq_ring, +struct amdgpu_ring *ring) +{ +struct amdgpu_device *adev = kiq_ring->adev; +uint64_t mqd_addr =amdgpu_bo_gpu_offset (ring->mqd_obj); +uint64_t wptr_addr = adev->wb.gpu_addr + (ring->wptr_offs *4 ); +uint32_t eng_sel = ring->funcs->type ==AMDGPU_RING_TYPE_GFX ?4 :0 ; + +amdgpu_ring_write (kiq_ring,PACKET3 (PACKET3_MAP_QUEUES ,5 )); +/* Q_sel:0, vmid:0, vidmem: 1, engine:0, num_Q:1*/ +amdgpu_ring_write (kiq_ring,/* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */ +PACKET3_MAP_QUEUES_QUEUE_SEL (0 ) |/* Queue_Sel */ +PACKET3_MAP_QUEUES_VMID (0 ) |/* VMID */ +PACKET3_MAP_QUEUES_QUEUE (ring->queue) | +PACKET3_MAP_QUEUES_PIPE (ring->pipe) | +PACKET3_MAP_QUEUES_ME ((ring->me ==1 ?0 :1 )) | +/*queue_type: normal compute queue */ +PACKET3_MAP_QUEUES_QUEUE_TYPE (0 ) | +/* alloc format: all_on_one_pipe */ +PACKET3_MAP_QUEUES_ALLOC_FORMAT (0 ) | +PACKET3_MAP_QUEUES_ENGINE_SEL (eng_sel) | +/* num_queues: must be 1 */ +PACKET3_MAP_QUEUES_NUM_QUEUES (1 )); +amdgpu_ring_write (kiq_ring, +PACKET3_MAP_QUEUES_DOORBELL_OFFSET (ring->doorbell_index)); +amdgpu_ring_write (kiq_ring,lower_32_bits (mqd_addr)); +amdgpu_ring_write (kiq_ring,upper_32_bits (mqd_addr)); +amdgpu_ring_write (kiq_ring,lower_32_bits (wptr_addr)); +amdgpu_ring_write (kiq_ring,upper_32_bits (wptr_addr)); +} + +static void gfx_v9_0_kiq_unmap_queues (struct amdgpu_ring *kiq_ring, +struct amdgpu_ring *ring, +enum amdgpu_unmap_queues_action action, +u64 gpu_addr,u64 seq) +{ +uint32_t eng_sel = ring->funcs->type ==AMDGPU_RING_TYPE_GFX ?4 :0 ; + +amdgpu_ring_write (kiq_ring,PACKET3 (PACKET3_UNMAP_QUEUES ,4 )); +amdgpu_ring_write (kiq_ring,/* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */ +PACKET3_UNMAP_QUEUES_ACTION (action) | +PACKET3_UNMAP_QUEUES_QUEUE_SEL (0 ) | +PACKET3_UNMAP_QUEUES_ENGINE_SEL (eng_sel) | +PACKET3_UNMAP_QUEUES_NUM_QUEUES (1 )); +amdgpu_ring_write (kiq_ring, +PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0 (ring->doorbell_index)); + +if (action ==PREEMPT_QUEUES_NO_UNMAP ) { +amdgpu_ring_write (kiq_ring,lower_32_bits (gpu_addr)); +amdgpu_ring_write (kiq_ring,upper_32_bits (gpu_addr)); +amdgpu_ring_write (kiq_ring, seq); + }else { +amdgpu_ring_write (kiq_ring,0 ); +amdgpu_ring_write (kiq_ring,0 ); +amdgpu_ring_write (kiq_ring,0 ); + } +} + +static void gfx_v9_0_kiq_query_status (struct amdgpu_ring *kiq_ring, +struct amdgpu_ring *ring, +u64 addr, +u64 seq) +{ +uint32_t eng_sel = ring->funcs->type ==AMDGPU_RING_TYPE_GFX ?4 :0 ; + +amdgpu_ring_write (kiq_ring,PACKET3 (PACKET3_QUERY_STATUS ,5 )); +amdgpu_ring_write (kiq_ring, +PACKET3_QUERY_STATUS_CONTEXT_ID (0 ) | +PACKET3_QUERY_STATUS_INTERRUPT_SEL (0 ) | +PACKET3_QUERY_STATUS_COMMAND (2 )); +/* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */ +amdgpu_ring_write (kiq_ring, +PACKET3_QUERY_STATUS_DOORBELL_OFFSET (ring->doorbell_index) | +PACKET3_QUERY_STATUS_ENG_SEL (eng_sel)); +amdgpu_ring_write (kiq_ring,lower_32_bits (addr)); +amdgpu_ring_write (kiq_ring,upper_32_bits (addr)); +amdgpu_ring_write (kiq_ring,lower_32_bits (seq)); +amdgpu_ring_write (kiq_ring,upper_32_bits (seq)); +} + +static const struct kiq_pm4_funcs gfx_v9_0_kiq_pm4_funcs = { + .kiq_set_resources = gfx_v9_0_kiq_set_resources, + .kiq_map_queues = gfx_v9_0_kiq_map_queues, + .kiq_unmap_queues = gfx_v9_0_kiq_unmap_queues, + .kiq_query_status = gfx_v9_0_kiq_query_status, + .set_resources_size =8 , + .map_queues_size =7 , + .unmap_queues_size =6 , + .query_status_size =7 , +}; + +static void gfx_v9_0_set_kiq_pm4_funcs (struct amdgpu_device *adev) +{ + adev->gfx.kiq.pmf = &gfx_v9_0_kiq_pm4_funcs; +} +static void gfx_v9_0_init_golden_registers (struct amdgpu_device *adev) {switch (adev->asic_type) {
@@ -4241,6 +4355,7 @@ static int gfx_v9_0_early_init(void *handle)else adev->gfx.num_gfx_rings =GFX9_NUM_GFX_RINGS ; adev->gfx.num_compute_rings =AMDGPU_MAX_COMPUTE_RINGS ; +gfx_v9_0_set_kiq_pm4_funcs (adev);gfx_v9_0_set_ring_funcs (adev);gfx_v9_0_set_irq_funcs (adev);gfx_v9_0_set_gds_init (adev);
Signature · 6 lines
2.17.1 _______________________________________________ amd-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/amd-gfx
PATCH 6/7 drm/amdgpu: implement tlbs invalidate on gfx9 gfx10
In reply to Alex Sierratlbs invalidate pointer function added to kiq_pm4_funcs struct. This way, tlb flush can be done through kiq member. TLBs invalidatation implemented for gfx9 and gfx10.
--- drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h | 4 +++ drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c | 33 +++++++++++++++++++++++++ drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c | 32 ++++++++++++++++++++++++ 3 files changed, 69 insertions(+)
drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h +4 -0
@@ -76,11 +76,15 @@ struct kiq_pm4_funcs {struct amdgpu_ring *ring, u64 addr, u64 seq); +int (*kiq_invalidate_tlbs)(struct amdgpu_ring *kiq_ring, +uint16_t pasid,uint32_t flush_type, +bool all_hub);/* Packet sizes */ int set_resources_size;int map_queues_size;int unmap_queues_size;int query_status_size; +int invalidate_tlbs_size; };struct amdgpu_kiq {
drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c +33 -0
@@ -40,6 +40,7 @@#include "ivsrcid/gfx/irqsrcs_gfx_10_1.h" #include "soc15.h" +#include "soc15d.h" #include "soc15_common.h" #include "clearstate_gfx10.h" #include "v10_structs.h"
@@ -346,15 +347,47 @@ static void gfx10_kiq_query_status(struct amdgpu_ring *kiq_ring,amdgpu_ring_write (kiq_ring,upper_32_bits (seq)); } +static int gfx10_kiq_invalidate_tlbs (struct amdgpu_ring *kiq_ring, +uint16_t pasid,uint32_t flush_type, +bool all_hub) +{ +signed long r; +uint32_t seq; +struct amdgpu_device *adev = kiq_ring->adev; +struct amdgpu_kiq *kiq = &adev->gfx.kiq; + +spin_lock (&adev->gfx.kiq.ring_lock); +amdgpu_ring_alloc (kiq_ring, kiq->pmf->invalidate_tlbs_size); +amdgpu_ring_write (kiq_ring,PACKET3 (PACKET3_INVALIDATE_TLBS ,0 )); +amdgpu_ring_write (kiq_ring, +PACKET3_INVALIDATE_TLBS_DST_SEL (1 ) | +PACKET3_INVALIDATE_TLBS_ALL_HUB (all_hub) | +PACKET3_INVALIDATE_TLBS_PASID (pasid) | +PACKET3_INVALIDATE_TLBS_FLUSH_TYPE (flush_type)); +amdgpu_fence_emit_polling (kiq_ring, &seq); +amdgpu_ring_commit (kiq_ring); +spin_unlock (&adev->gfx.kiq.ring_lock); + + r =amdgpu_fence_wait_polling (kiq_ring, seq, adev->usec_timeout); +if (r <1 ) { +DRM_ERROR ("wait for kiq fence error: %ld.\n" , r); +return -ETIME ; + } + +return 0 ; +} +static const struct kiq_pm4_funcs gfx_v10_0_kiq_pm4_funcs = { .kiq_set_resources = gfx10_kiq_set_resources, .kiq_map_queues = gfx10_kiq_map_queues, .kiq_unmap_queues = gfx10_kiq_unmap_queues, .kiq_query_status = gfx10_kiq_query_status, + .kiq_invalidate_tlbs = gfx10_kiq_invalidate_tlbs, .set_resources_size =8 , .map_queues_size =7 , .unmap_queues_size =6 , .query_status_size =7 , + .invalidate_tlbs_size =12 , };static void gfx_v10_0_set_kiq_pm4_funcs (struct amdgpu_device *adev)
drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c +32 -0
@@ -837,15 +837,47 @@ static void gfx_v9_0_kiq_query_status(struct amdgpu_ring *kiq_ring,amdgpu_ring_write (kiq_ring,upper_32_bits (seq)); } +static int gfx_v9_0_kiq_invalidate_tlbs (struct amdgpu_ring *kiq_ring, +uint16_t pasid,uint32_t flush_type, +bool all_hub) +{ +signed long r; +uint32_t seq; +struct amdgpu_device *adev = kiq_ring->adev; +struct amdgpu_kiq *kiq = &adev->gfx.kiq; + +spin_lock (&adev->gfx.kiq.ring_lock); +amdgpu_ring_alloc (kiq_ring, kiq->pmf->invalidate_tlbs_size); +amdgpu_ring_write (kiq_ring,PACKET3 (PACKET3_INVALIDATE_TLBS ,0 )); +amdgpu_ring_write (kiq_ring, +PACKET3_INVALIDATE_TLBS_DST_SEL (1 ) | +PACKET3_INVALIDATE_TLBS_ALL_HUB (all_hub) | +PACKET3_INVALIDATE_TLBS_PASID (pasid) | +PACKET3_INVALIDATE_TLBS_FLUSH_TYPE (flush_type)); +amdgpu_fence_emit_polling (kiq_ring, &seq); +amdgpu_ring_commit (kiq_ring); +spin_unlock (&adev->gfx.kiq.ring_lock); + + r =amdgpu_fence_wait_polling (kiq_ring, seq, adev->usec_timeout); +if (r <1 ) { +DRM_ERROR ("wait for kiq fence error: %ld.\n" , r); +return -ETIME ; + } + +return 0 ; +} +static const struct kiq_pm4_funcs gfx_v9_0_kiq_pm4_funcs = { .kiq_set_resources = gfx_v9_0_kiq_set_resources, .kiq_map_queues = gfx_v9_0_kiq_map_queues, .kiq_unmap_queues = gfx_v9_0_kiq_unmap_queues, .kiq_query_status = gfx_v9_0_kiq_query_status, + .kiq_invalidate_tlbs = gfx_v9_0_kiq_invalidate_tlbs, .set_resources_size =8 , .map_queues_size =7 , .unmap_queues_size =6 , .query_status_size =7 , + .invalidate_tlbs_size =12 , };static void gfx_v9_0_set_kiq_pm4_funcs (struct amdgpu_device *adev)
Signature · 6 lines
2.17.1 _______________________________________________ amd-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/amd-gfx
PATCH 7/7 drm/amdgpu: tlbs invalidate replaced by kiq_pm4_funcs
In reply to Alex SierraTLBs invalidation was removed from gmc_v9 and gmc_v10 files. This implementation has been moved to kiq_pm4_funcs calls.
--- drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c | 32 ++------------------------ drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c | 32 ++------------------------ 2 files changed, 4 insertions(+), 60 deletions(-)
drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c +2 -30
@@ -250,35 +250,6 @@ static bool gmc_v10_0_get_atc_vmid_pasid_mapping_info(return !!(value &ATC_VMID0_PASID_MAPPING__VALID_MASK ); } -static int gmc_v10_0_invalidate_tlbs_with_kiq (struct amdgpu_device *adev, -uint16_t pasid,uint32_t flush_type, -bool all_hub) -{ -signed long r; -uint32_t seq; -struct amdgpu_ring *ring = &adev->gfx.kiq.ring; - -spin_lock (&adev->gfx.kiq.ring_lock); -amdgpu_ring_alloc (ring,12 );/* fence + invalidate_tlbs package*/ -amdgpu_ring_write (ring,PACKET3 (PACKET3_INVALIDATE_TLBS ,0 )); -amdgpu_ring_write (ring, -PACKET3_INVALIDATE_TLBS_DST_SEL (1 ) | -PACKET3_INVALIDATE_TLBS_ALL_HUB (all_hub) | -PACKET3_INVALIDATE_TLBS_PASID (pasid) | -PACKET3_INVALIDATE_TLBS_FLUSH_TYPE (flush_type)); -amdgpu_fence_emit_polling (ring, &seq); -amdgpu_ring_commit (ring); -spin_unlock (&adev->gfx.kiq.ring_lock); - - r =amdgpu_fence_wait_polling (ring, seq, adev->usec_timeout); -if (r <1 ) { -DRM_ERROR ("wait for kiq fence error: %ld.\n" , r); -return -ETIME ; - } - -return 0 ; -} -/* * GART * VMID 0 is the physical GPU addresses as used by the kernel.
@@ -441,9 +412,10 @@ static int gmc_v10_0_flush_gpu_tlb_pasid(struct amdgpu_device *adev,uint16_t queried_pasid;bool ret;struct amdgpu_ring *ring = &adev->gfx.kiq.ring; +struct amdgpu_kiq *kiq = &adev->gfx.kiq;if (amdgpu_emu_mode ==0 && ring->sched.ready) -return gmc_v10_0_invalidate_tlbs_with_kiq (adev, +return kiq->pmf->kiq_invalidate_tlbs (ring, pasid, flush_type, all_hub);for (vmid =1 ; vmid <16 ; vmid++) {
drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +2 -30
@@ -455,35 +455,6 @@ static bool gmc_v9_0_get_atc_vmid_pasid_mapping_info(struct amdgpu_device *adev,return !!(value &ATC_VMID0_PASID_MAPPING__VALID_MASK ); } -static int gmc_v9_0_invalidate_tlbs_with_kiq (struct amdgpu_device *adev, -uint16_t pasid,uint32_t flush_type, -bool all_hub) -{ -signed long r; -uint32_t seq; -struct amdgpu_ring *ring = &adev->gfx.kiq.ring; - -spin_lock (&adev->gfx.kiq.ring_lock); -amdgpu_ring_alloc (ring,12 );/* fence + invalidate_tlbs package*/ -amdgpu_ring_write (ring,PACKET3 (PACKET3_INVALIDATE_TLBS ,0 )); -amdgpu_ring_write (ring, -PACKET3_INVALIDATE_TLBS_DST_SEL (1 ) | -PACKET3_INVALIDATE_TLBS_ALL_HUB (all_hub) | -PACKET3_INVALIDATE_TLBS_PASID (pasid) | -PACKET3_INVALIDATE_TLBS_FLUSH_TYPE (flush_type)); -amdgpu_fence_emit_polling (ring, &seq); -amdgpu_ring_commit (ring); -spin_unlock (&adev->gfx.kiq.ring_lock); - - r =amdgpu_fence_wait_polling (ring, seq, adev->usec_timeout); -if (r <1 ) { -DRM_ERROR ("wait for kiq fence error: %ld.\n" , r); -return -ETIME ; - } - -return 0 ; -} -/* * GART * VMID 0 is the physical GPU addresses as used by the kernel.
@@ -598,12 +569,13 @@ static int gmc_v9_0_flush_gpu_tlb_pasid(struct amdgpu_device *adev,uint16_t queried_pasid;bool ret;struct amdgpu_ring *ring = &adev->gfx.kiq.ring; +struct amdgpu_kiq *kiq = &adev->gfx.kiq;if (adev->in_gpu_reset)return -EIO ;if (ring->sched.ready) -return gmc_v9_0_invalidate_tlbs_with_kiq (adev, +return kiq->pmf->kiq_invalidate_tlbs (ring, pasid, flush_type, all_hub);for (vmid =1 ; vmid <16 ; vmid++) {
Signature · 6 lines
2.17.1 _______________________________________________ amd-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/amd-gfx
Re: [PATCH 7/7] drm/amdgpu: tlbs invalidate replaced by kiq_pm4_funcs
The end result looks good, but you should consider re-ordering this patch set.
That would clean things up and make review much easier.
Adding code first and then removing it later on in the same patch set is not a good practice.
Regards,
Christian.
Am 10.01.20 um 01:04 schrieb Alex Sierra:
Quoted 107 lines
TLBs invalidation was removed from
gmc_v9andgmc_v10files. This implementation has been moved tokiq_pm4_funcscalls.Change-Id I66463e2e5fdf7611ef4a802eb517ae0d269987c3Signed-off-by Alex Sierra <[email protected]>--- drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c | 32 ++------------------------ drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c | 32 ++------------------------ 2 files changed, 4 insertions(+), 60 deletions(-)drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c +2
-30@@ -250,35 +250,6 @@ static bool gmc_v10_0_get_atc_vmid_pasid_mapping_info(return !!(value &ATC_VMID0_PASID_MAPPING__VALID_MASK ); } -static int gmc_v10_0_invalidate_tlbs_with_kiq (struct amdgpu_device *adev, -uint16_t pasid,uint32_t flush_type, -bool all_hub) -{ -signed long r; -uint32_t seq; -struct amdgpu_ring *ring = &adev->gfx.kiq.ring; - -spin_lock (&adev->gfx.kiq.ring_lock); -amdgpu_ring_alloc (ring,12 );/* fence + invalidate_tlbs package*/ -amdgpu_ring_write (ring,PACKET3 (PACKET3_INVALIDATE_TLBS ,0 )); -amdgpu_ring_write (ring, -PACKET3_INVALIDATE_TLBS_DST_SEL (1 ) | -PACKET3_INVALIDATE_TLBS_ALL_HUB (all_hub) | -PACKET3_INVALIDATE_TLBS_PASID (pasid) | -PACKET3_INVALIDATE_TLBS_FLUSH_TYPE (flush_type)); -amdgpu_fence_emit_polling (ring, &seq); -amdgpu_ring_commit (ring); -spin_unlock (&adev->gfx.kiq.ring_lock); - - r =amdgpu_fence_wait_polling (ring, seq, adev->usec_timeout); -if (r <1 ) { -DRM_ERROR ("wait for kiq fence error: %ld.\n" , r); -return -ETIME ; - } - -return 0 ; -} -/* * GART * VMID 0 is the physical GPU addresses as used by the kernel. @@ -441,9 +412,10 @@ static int gmc_v10_0_flush_gpu_tlb_pasid(struct amdgpu_device *adev,uint16_t queried_pasid;bool ret;struct amdgpu_ring *ring = &adev->gfx.kiq.ring; +struct amdgpu_kiq *kiq = &adev->gfx.kiq;if (amdgpu_emu_mode ==0 && ring->sched.ready) -return gmc_v10_0_invalidate_tlbs_with_kiq (adev, +return kiq->pmf->kiq_invalidate_tlbs (ring, pasid, flush_type, all_hub);for (vmid =1 ; vmid <16 ; vmid++) {drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +1
-29@@ -455,35 +455,6 @@ static bool gmc_v9_0_get_atc_vmid_pasid_mapping_info(struct amdgpu_device *adev,return !!(value &ATC_VMID0_PASID_MAPPING__VALID_MASK ); } -static int gmc_v9_0_invalidate_tlbs_with_kiq (struct amdgpu_device *adev, -uint16_t pasid,uint32_t flush_type, -bool all_hub) -{ -signed long r; -uint32_t seq; -struct amdgpu_ring *ring = &adev->gfx.kiq.ring; - -spin_lock (&adev->gfx.kiq.ring_lock); -amdgpu_ring_alloc (ring,12 );/* fence + invalidate_tlbs package*/ -amdgpu_ring_write (ring,PACKET3 (PACKET3_INVALIDATE_TLBS ,0 )); -amdgpu_ring_write (ring, -PACKET3_INVALIDATE_TLBS_DST_SEL (1 ) | -PACKET3_INVALIDATE_TLBS_ALL_HUB (all_hub) | -PACKET3_INVALIDATE_TLBS_PASID (pasid) | -PACKET3_INVALIDATE_TLBS_FLUSH_TYPE (flush_type)); -amdgpu_fence_emit_polling (ring, &seq); -amdgpu_ring_commit (ring); -spin_unlock (&adev->gfx.kiq.ring_lock); - - r =amdgpu_fence_wait_polling (ring, seq, adev->usec_timeout); -if (r <1 ) { -DRM_ERROR ("wait for kiq fence error: %ld.\n" , r); -return -ETIME ; - } - -return 0 ; -} -/* * GART * VMID 0 is the physical GPU addresses as used by the kernel. @@ -598,12 +569,13 @@ static int gmc_v9_0_flush_gpu_tlb_pasid(struct amdgpu_device *adev,uint16_t queried_pasid;bool ret;struct amdgpu_ring *ring = &adev->gfx.kiq.ring; +struct amdgpu_kiq *kiq = &adev->gfx.kiq;
drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c static int gmc_v9_0_flush_gpu_tlb_pasid(struct amdgpu_device *adev, · line 581
Quoted 10 lines
drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +1
-1if (adev->in_gpu_reset)return -EIO ;if (ring->sched.ready) -return gmc_v9_0_invalidate_tlbs_with_kiq (adev, +return kiq->pmf->kiq_invalidate_tlbs (ring, pasid, flush_type, all_hub);for (vmid =1 ; vmid <16 ; vmid++) {
_______________________________________________
amd-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/amd-gfx