Keep device awake during GT TLB invalidations
Subsystems (2)
Matthew Brost
maintainerv2 cover Keep device awake during GT TLB invalidations
Issue was raised in [1], attempt to fix this by holding PM ref when GT TLB invalidations are in flight. An alternative solution could be hold a PM ref when any H2G are inflight which expect a G2H.
Including two patches from [2] as they a prerequisite for this fix.
[1] https://patchwork.freedesktop.org/series/136145/
[2] https://patchwork.freedesktop.org/series/135809/
v2: Fix CI failure
Matthew Brost (3):
drm/xe: Add xe_gt_tlb_invalidation_fence_init helper drm/xe: Drop xe_gt_tlb_invalidation_wait
drm/xe: Hold a PM ref when GT TLB invalidations are inflight
drivers/gpu/drm/xe/xe_gt_tlb_invalidation.c | 201 ++++++++++-------- drivers/gpu/drm/xe/xe_gt_tlb_invalidation.h | 12 +- .../gpu/drm/xe/xe_gt_tlb_invalidation_types.h | 4 + drivers/gpu/drm/xe/xe_pt.c | 26 +-- drivers/gpu/drm/xe/xe_vm.c | 30 +-- 5 files changed, 142 insertions(+), 131 deletions(-)
Signature · 1 line
2.34.1
Matthew Brost
maintainerPATCH v2 1/3 drm/xe: Add xe_gt_tlb_invalidation_fence_init helper
Other layers should not be touching struct xe_gt_tlb_invalidation_fence directly, add helper for initialization.
v2:
- Add dma_fence_get and list init to xe_gt_tlb_invalidation_fence_init
--- drivers/gpu/drm/xe/xe_gt_tlb_invalidation.c | 36 +++++++++++++++++++++ drivers/gpu/drm/xe/xe_gt_tlb_invalidation.h | 3 ++ drivers/gpu/drm/xe/xe_pt.c | 26 +-------------- 3 files changed, 40 insertions(+), 25 deletions(-)
drivers/gpu/drm/xe/xe_gt_tlb_invalidation.c +36 -0
@@ -508,3 +508,39 @@ int xe_guc_tlb_invalidation_done_handler(struct xe_guc *guc, u32 *msg, u32 len)return 0 ; } + +static const char * +invalidation_fence_get_driver_name (struct dma_fence *dma_fence) +{ +return "xe" ; +} + +static const char * +invalidation_fence_get_timeline_name (struct dma_fence *dma_fence) +{ +return "invalidation_fence" ; +} + +static const struct dma_fence_ops invalidation_fence_ops = { + .get_driver_name = invalidation_fence_get_driver_name, + .get_timeline_name = invalidation_fence_get_timeline_name, +}; + +/** +* xe_gt_tlb_invalidation_fence_init - Initialize TLB invalidation fence +* @gt: GT +* @fence: TLB invalidation fence to initialize +* +* Initialize TLB invalidation fence for use +*/ +void xe_gt_tlb_invalidation_fence_init (struct xe_gt *gt, +struct xe_gt_tlb_invalidation_fence *fence) +{ +spin_lock_irq (>->tlb_invalidation.lock); +dma_fence_init (&fence->base, &invalidation_fence_ops, + >->tlb_invalidation.lock, +dma_fence_context_alloc (1 ),1 ); +spin_unlock_irq (>->tlb_invalidation.lock); +INIT_LIST_HEAD (&fence->link); +dma_fence_get (&fence->base); +}
drivers/gpu/drm/xe/xe_gt_tlb_invalidation.h +3 -0
@@ -26,4 +26,7 @@ int xe_gt_tlb_invalidation_range(struct xe_gt *gt,int xe_gt_tlb_invalidation_wait (struct xe_gt *gt,int seqno);int xe_guc_tlb_invalidation_done_handler (struct xe_guc *guc,u32 *msg,u32 len); +void xe_gt_tlb_invalidation_fence_init (struct xe_gt *gt, +struct xe_gt_tlb_invalidation_fence *fence); +#endif /* _XE_GT_TLB_INVALIDATION_ */
drivers/gpu/drm/xe/xe_pt.c +1 -25
@@ -1317,23 +1317,6 @@ struct invalidation_fence {u32 asid; }; -static const char * -invalidation_fence_get_driver_name (struct dma_fence *dma_fence) -{ -return "xe" ; -} - -static const char * -invalidation_fence_get_timeline_name (struct dma_fence *dma_fence) -{ -return "invalidation_fence" ; -} - -static const struct dma_fence_ops invalidation_fence_ops = { - .get_driver_name = invalidation_fence_get_driver_name, - .get_timeline_name = invalidation_fence_get_timeline_name, -}; -static void invalidation_fence_cb (struct dma_fence *fence,struct dma_fence_cb *cb) {
@@ -1372,15 +1355,8 @@ static void invalidation_fence_init(struct xe_gt *gt,trace_xe_gt_tlb_invalidation_fence_create (gt_to_xe (gt), &ifence->base); -spin_lock_irq (>->tlb_invalidation.lock); -dma_fence_init (&ifence->base.base, &invalidation_fence_ops, - >->tlb_invalidation.lock, -dma_fence_context_alloc (1 ),1 ); -spin_unlock_irq (>->tlb_invalidation.lock); - -INIT_LIST_HEAD (&ifence->base.link); +xe_gt_tlb_invalidation_fence_init (gt, &ifence->base); -dma_fence_get (&ifence->base.base);/* Ref for caller */ ifence->fence = fence; ifence->gt = gt; ifence->start = start;
Signature · 1 line
2.34.1
Matthew Brost
maintainerPATCH v2 2/3 drm/xe: Drop xe_gt_tlb_invalidation_wait
In reply to Matthew BrostHaving two methods to wait on GT TLB invalidations is not ideal. Remove xe_gt_tlb_invalidation_wait and only use GT TLB invalidation fences.
In addition to two methods being less than ideal, once GT TLB invalidations are coalesced the seqno cannot be assigned during xe_gt_tlb_invalidation_ggtt/range. Thus xe_gt_tlb_invalidation_wait would not have a seqno to wait one. A fence however can be armed and later signaled.
v3:
- Add explaination about coalescing to commit message
v4:
- Don't put dma fence if defined on stack (CI)
--- drivers/gpu/drm/xe/xe_gt_tlb_invalidation.c | 117 +++++++------------- drivers/gpu/drm/xe/xe_gt_tlb_invalidation.h | 10 +- drivers/gpu/drm/xe/xe_pt.c | 2 +- drivers/gpu/drm/xe/xe_vm.c | 28 ++--- 4 files changed, 65 insertions(+), 92 deletions(-)
drivers/gpu/drm/xe/xe_gt_tlb_invalidation.c +41 -76
@@ -17,6 +17,8 @@#include "xe_trace.h" #include "regs/xe_guc_regs.h" +#define FENCE_STACK_BIT DMA_FENCE_FLAG_USER_BITS +/* * TLB inval depends on pending commands in the CT queue and then the real * invalidation time. Double up the time to process full CT queue
@@ -90,9 +92,12 @@ int xe_gt_tlb_invalidation_init(struct xe_gt *gt)static void __invalidation_fence_signal (struct xe_device *xe,struct xe_gt_tlb_invalidation_fence *fence) { +bool stack =test_bit (FENCE_STACK_BIT , &fence->base.flags); +trace_xe_gt_tlb_invalidation_fence_signal (xe, fence);dma_fence_signal (&fence->base); -dma_fence_put (&fence->base); +if (!stack) +dma_fence_put (&fence->base); } static void
@@ -111,7 +116,6 @@ invalidation_fence_signal(struct xe_device *xe, struct xe_gt_tlb_invalidation_fevoid xe_gt_tlb_invalidation_reset (struct xe_gt *gt) {struct xe_gt_tlb_invalidation_fence *fence, *next; -struct xe_guc *guc = >->uc.guc;int pending_seqno;/*
@@ -134,7 +138,6 @@ void xe_gt_tlb_invalidation_reset(struct xe_gt *gt)else pending_seqno = gt->tlb_invalidation.seqno -1 ;WRITE_ONCE (gt->tlb_invalidation.seqno_recv, pending_seqno); -wake_up_all (&guc->ct.wq);list_for_each_entry_safe (fence, next, >->tlb_invalidation.pending_fences, link)
@@ -165,6 +168,8 @@ static int send_tlb_invalidation(struct xe_guc *guc,int seqno;int ret; +xe_gt_assert (gt, fence); +/* * XXX: The seqno algorithm relies on TLB invalidation being processed * in order which they currently are, if that changes the algorithm will
@@ -173,10 +178,8 @@ static int send_tlb_invalidation(struct xe_guc *guc,mutex_lock (&guc->ct.lock); seqno = gt->tlb_invalidation.seqno; -if (fence) { - fence->seqno = seqno; -trace_xe_gt_tlb_invalidation_fence_send (xe, fence); - } + fence->seqno = seqno; +trace_xe_gt_tlb_invalidation_fence_send (xe, fence); action[1 ] = seqno; ret =xe_guc_ct_send_locked (&guc->ct, action, len,G2H_LEN_DW_TLB_INVALIDATE ,1 );
@@ -209,7 +212,6 @@ static int send_tlb_invalidation(struct xe_guc *guc,TLB_INVALIDATION_SEQNO_MAX ;if (!gt->tlb_invalidation.seqno) gt->tlb_invalidation.seqno =1 ; - ret = seqno; }mutex_unlock (&guc->ct.lock);
@@ -223,14 +225,16 @@ static int send_tlb_invalidation(struct xe_guc *guc,/** * xe_gt_tlb_invalidation_guc - Issue a TLB invalidation on this GT for the GuC * @gt: graphics tile +* @fence: invalidation fence which will be signal on TLB invalidation +* completion * * Issue a TLB invalidation for the GuC. Completion of TLB is asynchronous and -* caller can use seqno + xe_gt_tlb_invalidation_wait to wait for completion. +* caller can use the invalidation fence to wait for completion. * -* Return: Seqno which can be passed to xe_gt_tlb_invalidation_wait on success, -* negative error code on error. +* Return: 0 on success, negative error code on error */ -static int xe_gt_tlb_invalidation_guc (struct xe_gt *gt) +static int xe_gt_tlb_invalidation_guc (struct xe_gt *gt, +struct xe_gt_tlb_invalidation_fence *fence) {u32 action[] = {XE_GUC_ACTION_TLB_INVALIDATION ,
@@ -238,7 +242,7 @@ static int xe_gt_tlb_invalidation_guc(struct xe_gt *gt) MAKE_INVAL_OP(XE_GUC_TLB_INVAL_GUC), }; -return send_tlb_invalidation (>->uc.guc,NULL , action, +return send_tlb_invalidation (>->uc.guc, fence, action,ARRAY_SIZE (action)); }
@@ -257,13 +261,15 @@ int xe_gt_tlb_invalidation_ggtt(struct xe_gt *gt)if (xe_guc_ct_enabled (>->uc.guc.ct) && gt->uc.guc.submission_state.enabled) { -int seqno; +struct xe_gt_tlb_invalidation_fence fence; +int ret; - seqno =xe_gt_tlb_invalidation_guc (gt); -if (seqno <=0 ) -return seqno; +xe_gt_tlb_invalidation_fence_init (gt, &fence, true); + ret =xe_gt_tlb_invalidation_guc (gt, &fence); +if (ret <0 ) +return ret; -xe_gt_tlb_invalidation_wait (gt, seqno); +xe_gt_tlb_invalidation_fence_wait (&fence); }else if (xe_device_uc_enabled (xe) && !xe_device_wedged (xe)) {if (IS_SRIOV_VF (xe))return 0 ;
@@ -290,18 +296,16 @@ int xe_gt_tlb_invalidation_ggtt(struct xe_gt *gt)* * @gt: graphics tile * @fence: invalidation fence which will be signal on TLB invalidation -* completion, can be NULL +* completion * @start: start address * @end: end address * @asid: address space id * * Issue a range based TLB invalidation if supported, if not fallback to a full -* TLB invalidation. Completion of TLB is asynchronous and caller can either use -* the invalidation fence or seqno + xe_gt_tlb_invalidation_wait to wait for -* completion. +* TLB invalidation. Completion of TLB is asynchronous and caller can use +* the invalidation fence to wait for completion. * -* Return: Seqno which can be passed to xe_gt_tlb_invalidation_wait on success, -* negative error code on error. +* Return: Negative error code on error, 0 on success */ int xe_gt_tlb_invalidation_range(struct xe_gt *gt, struct xe_gt_tlb_invalidation_fence *fence,
@@ -312,11 +316,11 @@ int xe_gt_tlb_invalidation_range(struct xe_gt *gt,u32 action[MAX_TLB_INVALIDATION_LEN ];int len =0 ; +xe_gt_assert (gt, fence); +/* Execlists not supported */ if (gt_to_xe (gt)->info.force_execlist) { -if (fence) -__invalidation_fence_signal (xe, fence); - +__invalidation_fence_signal (xe, fence);return 0 ; }
@@ -382,12 +386,10 @@ int xe_gt_tlb_invalidation_range(struct xe_gt *gt,* @vma: VMA to invalidate * * Issue a range based TLB invalidation if supported, if not fallback to a full -* TLB invalidation. Completion of TLB is asynchronous and caller can either use -* the invalidation fence or seqno + xe_gt_tlb_invalidation_wait to wait for -* completion. +* TLB invalidation. Completion of TLB is asynchronous and caller can use +* the invalidation fence to wait for completion. * -* Return: Seqno which can be passed to xe_gt_tlb_invalidation_wait on success, -* negative error code on error. +* Return: Negative error code on error, 0 on success */ int xe_gt_tlb_invalidation_vma(struct xe_gt *gt, struct xe_gt_tlb_invalidation_fence *fence,
@@ -400,43 +402,6 @@ int xe_gt_tlb_invalidation_vma(struct xe_gt *gt, xe_vma_vm(vma)->usm.asid); } -/** -* xe_gt_tlb_invalidation_wait - Wait for TLB to complete -* @gt: graphics tile -* @seqno: seqno to wait which was returned from xe_gt_tlb_invalidation -* -* Wait for tlb_timeout_jiffies() for a TLB invalidation to complete. -* -* Return: 0 on success, -ETIME on TLB invalidation timeout -*/ -int xe_gt_tlb_invalidation_wait (struct xe_gt *gt,int seqno) -{ -struct xe_guc *guc = >->uc.guc; -int ret; - -/* Execlists not supported */ -if (gt_to_xe (gt)->info.force_execlist) -return 0 ; - -/* -* XXX: See above, this algorithm only works if seqno are always in -* order -*/ - ret =wait_event_timeout (guc->ct.wq, -tlb_invalidation_seqno_past (gt, seqno), -tlb_timeout_jiffies (gt)); -if (!ret) { -struct drm_printer p =xe_gt_err_printer (gt); - -xe_gt_err (gt,"TLB invalidation time'd out, seqno=%d, recv=%d\n" , - seqno, gt->tlb_invalidation.seqno_recv); -xe_guc_ct_print (&guc->ct, &p, true); -return -ETIME ; - } - -return 0 ; -} -/** * xe_guc_tlb_invalidation_done_handler - TLB invalidation done handler * @guc: guc
@@ -480,12 +445,7 @@ int xe_guc_tlb_invalidation_done_handler(struct xe_guc *guc, u32 *msg, u32 len)return 0 ; } -/* -* wake_up_all() and wait_event_timeout() already have the correct -* barriers. -*/ WRITE_ONCE (gt->tlb_invalidation.seqno_recv, msg[0 ]); -wake_up_all (&guc->ct.wq);list_for_each_entry_safe (fence, next, >->tlb_invalidation.pending_fences, link) {
@@ -530,11 +490,13 @@ static const struct dma_fence_ops invalidation_fence_ops = {* xe_gt_tlb_invalidation_fence_init - Initialize TLB invalidation fence * @gt: GT * @fence: TLB invalidation fence to initialize +* @stack: fence is stack variable * * Initialize TLB invalidation fence for use */ void xe_gt_tlb_invalidation_fence_init (struct xe_gt *gt, -struct xe_gt_tlb_invalidation_fence *fence) +struct xe_gt_tlb_invalidation_fence *fence, +bool stack) {spin_lock_irq (>->tlb_invalidation.lock); dma_fence_init(&fence->base, &invalidation_fence_ops,
@@ -542,5 +504,8 @@ void xe_gt_tlb_invalidation_fence_init(struct xe_gt *gt, dma_fence_context_alloc(1), 1);spin_unlock_irq (>->tlb_invalidation.lock);INIT_LIST_HEAD (&fence->link); -dma_fence_get (&fence->base); +if (stack) +set_bit (FENCE_STACK_BIT , &fence->base.flags); +else +dma_fence_get (&fence->base); }
drivers/gpu/drm/xe/xe_gt_tlb_invalidation.h +8 -2
@@ -23,10 +23,16 @@ int xe_gt_tlb_invalidation_vma(struct xe_gt *gt,int xe_gt_tlb_invalidation_range (struct xe_gt *gt,struct xe_gt_tlb_invalidation_fence *fence,u64 start,u64 end,u32 asid); -int xe_gt_tlb_invalidation_wait (struct xe_gt *gt,int seqno);int xe_guc_tlb_invalidation_done_handler (struct xe_guc *guc,u32 *msg,u32 len);void xe_gt_tlb_invalidation_fence_init (struct xe_gt *gt, -struct xe_gt_tlb_invalidation_fence *fence); +struct xe_gt_tlb_invalidation_fence *fence, +bool stack); + +static inline void +xe_gt_tlb_invalidation_fence_wait (struct xe_gt_tlb_invalidation_fence *fence) +{ +dma_fence_wait (&fence->base, false); +}#endif /* _XE_GT_TLB_INVALIDATION_ */
drivers/gpu/drm/xe/xe_pt.c +1 -1
@@ -1355,7 +1355,7 @@ static void invalidation_fence_init(struct xe_gt *gt,trace_xe_gt_tlb_invalidation_fence_create (gt_to_xe (gt), &ifence->base); -xe_gt_tlb_invalidation_fence_init (gt, &ifence->base); +xe_gt_tlb_invalidation_fence_init (gt, &ifence->base, false); ifence->fence = fence; ifence->gt = gt;
drivers/gpu/drm/xe/xe_vm.c +15 -13
@@ -3180,8 +3180,8 @@ int xe_vm_invalidate_vma(struct xe_vma *vma) {struct xe_device *xe =xe_vma_vm (vma)->xe;struct xe_tile *tile; +struct xe_gt_tlb_invalidation_fence fence[XE_MAX_TILES_PER_DEVICE ];u32 tile_needs_invalidate =0 ; -int seqno[XE_MAX_TILES_PER_DEVICE ];u8 id;int ret;
@@ -3208,29 +3208,31 @@ int xe_vm_invalidate_vma(struct xe_vma *vma)for_each_tile (tile, xe, id) {if (xe_pt_zap_ptes (tile, vma)) { - tile_needs_invalidate |=BIT (id);xe_device_wmb (xe); +xe_gt_tlb_invalidation_fence_init (tile->primary_gt, + &fence[id], true); +/* * FIXME: We potentially need to invalidate multiple * GTs within the tile */ - seqno[id] =xe_gt_tlb_invalidation_vma (tile->primary_gt,NULL , vma); -if (seqno[id] <0 ) -return seqno[id]; - } - } - -for_each_tile (tile, xe, id) { -if (tile_needs_invalidate &BIT (id)) { - ret =xe_gt_tlb_invalidation_wait (tile->primary_gt, seqno[id]); + ret =xe_gt_tlb_invalidation_vma (tile->primary_gt, + &fence[id], vma);if (ret <0 ) -return ret; + goto wait; + + tile_needs_invalidate |=BIT (id); } } +wait: +for_each_tile (tile, xe, id) +if (tile_needs_invalidate &BIT (id)) +xe_gt_tlb_invalidation_fence_wait (&fence[id]); + vma->tile_invalidated = vma->tile_mask; -return 0 ; +return ret; }struct xe_vm_snapshot {
Signature · 1 line
2.34.1
Matthew Brost
maintainerPATCH v2 3/3 drm/xe: Hold a PM ref when GT TLB invalidations are inflight
In reply to Matthew BrostAvoid GT TLB invalidation timeouts by holding a PM ref when invalidations are inflight.
v2:
- Drop PM ref before signaling fence (CI)
--- drivers/gpu/drm/xe/xe_gt_tlb_invalidation.c | 62 ++++++++++++------- drivers/gpu/drm/xe/xe_gt_tlb_invalidation.h | 1 + .../gpu/drm/xe/xe_gt_tlb_invalidation_types.h | 4 ++ drivers/gpu/drm/xe/xe_vm.c | 4 +- 4 files changed, 47 insertions(+), 24 deletions(-)
drivers/gpu/drm/xe/xe_gt_tlb_invalidation.c +39 -23
@@ -13,6 +13,7 @@#include "xe_guc.h" #include "xe_guc_ct.h" #include "xe_mmio.h" +#include "xe_pm.h" #include "xe_sriov.h" #include "xe_trace.h" #include "regs/xe_guc_regs.h"
@@ -35,6 +36,24 @@ static long tlb_timeout_jiffies(struct xe_gt *gt)return hw_tlb_timeout +2 * delay; } +static void +__invalidation_fence_signal (struct xe_device *xe,struct xe_gt_tlb_invalidation_fence *fence) +{ +bool stack =test_bit (FENCE_STACK_BIT , &fence->base.flags); + +trace_xe_gt_tlb_invalidation_fence_signal (xe, fence); +xe_gt_tlb_invalidation_fence_fini (fence); +dma_fence_signal (&fence->base); +if (!stack) +dma_fence_put (&fence->base); +} + +static void +invalidation_fence_signal (struct xe_device *xe,struct xe_gt_tlb_invalidation_fence *fence) +{ +list_del (&fence->link); +__invalidation_fence_signal (xe, fence); +}static void xe_gt_tlb_fence_timeout (struct work_struct *work) {
@@ -56,10 +75,8 @@ static void xe_gt_tlb_fence_timeout(struct work_struct *work)xe_gt_err (gt,"TLB invalidation fence timeout, seqno=%d recv=%d" , fence->seqno, gt->tlb_invalidation.seqno_recv); -list_del (&fence->link); fence->base.error = -ETIME ; -dma_fence_signal (&fence->base); -dma_fence_put (&fence->base); +invalidation_fence_signal (xe, fence); }if (!list_empty (>->tlb_invalidation.pending_fences)) queue_delayed_work(system_wq,
@@ -89,24 +106,6 @@ int xe_gt_tlb_invalidation_init(struct xe_gt *gt)return 0 ; } -static void -__invalidation_fence_signal (struct xe_device *xe,struct xe_gt_tlb_invalidation_fence *fence) -{ -bool stack =test_bit (FENCE_STACK_BIT , &fence->base.flags); - -trace_xe_gt_tlb_invalidation_fence_signal (xe, fence); -dma_fence_signal (&fence->base); -if (!stack) -dma_fence_put (&fence->base); -} - -static void -invalidation_fence_signal (struct xe_device *xe,struct xe_gt_tlb_invalidation_fence *fence) -{ -list_del (&fence->link); -__invalidation_fence_signal (xe, fence); -} -/** * xe_gt_tlb_invalidation_reset - Initialize GT TLB invalidation reset * @gt: graphics tile
@@ -266,8 +265,10 @@ int xe_gt_tlb_invalidation_ggtt(struct xe_gt *gt)xe_gt_tlb_invalidation_fence_init (gt, &fence, true); ret =xe_gt_tlb_invalidation_guc (gt, &fence); -if (ret <0 ) +if (ret <0 ) { +xe_gt_tlb_invalidation_fence_fini (&fence);return ret; + }xe_gt_tlb_invalidation_fence_wait (&fence); }else if (xe_device_uc_enabled (xe) && !xe_device_wedged (xe)) {
@@ -492,12 +493,15 @@ static const struct dma_fence_ops invalidation_fence_ops = {* @fence: TLB invalidation fence to initialize * @stack: fence is stack variable * -* Initialize TLB invalidation fence for use +* Initialize TLB invalidation fence for use. xe_gt_tlb_invalidation_fence_fini +* must be called if fence is not signaled. */ void xe_gt_tlb_invalidation_fence_init (struct xe_gt *gt,struct xe_gt_tlb_invalidation_fence *fence,bool stack) { +xe_pm_runtime_get_noresume (gt_to_xe (gt)); +spin_lock_irq (>->tlb_invalidation.lock); dma_fence_init(&fence->base, &invalidation_fence_ops, >->tlb_invalidation.lock,
@@ -508,4 +512,16 @@ void xe_gt_tlb_invalidation_fence_init(struct xe_gt *gt,set_bit (FENCE_STACK_BIT , &fence->base.flags); else dma_fence_get(&fence->base); + fence->gt = gt; +} + +/** +* xe_gt_tlb_invalidation_fence_fini - Finalize TLB invalidation fence +* @fence: TLB invalidation fence to finalize +* +* Drop PM ref which fence took durinig init. +*/ +void xe_gt_tlb_invalidation_fence_fini (struct xe_gt_tlb_invalidation_fence *fence) +{ +xe_pm_runtime_put (gt_to_xe (fence->gt)); }
drivers/gpu/drm/xe/xe_gt_tlb_invalidation.h +1 -0
@@ -28,6 +28,7 @@ int xe_guc_tlb_invalidation_done_handler(struct xe_guc *guc, u32 *msg, u32 len);void xe_gt_tlb_invalidation_fence_init (struct xe_gt *gt,struct xe_gt_tlb_invalidation_fence *fence,bool stack); +void xe_gt_tlb_invalidation_fence_fini (struct xe_gt_tlb_invalidation_fence *fence);static inline void xe_gt_tlb_invalidation_fence_wait (struct xe_gt_tlb_invalidation_fence *fence)
drivers/gpu/drm/xe/xe_gt_tlb_invalidation_types.h +4 -0
@@ -8,6 +8,8 @@#include <linux/dma-fence.h> +struct xe_gt ; +/** * struct xe_gt_tlb_invalidation_fence - XE GT TLB invalidation fence *
@@ -17,6 +19,8 @@struct xe_gt_tlb_invalidation_fence {/** @base: dma fence base */ struct dma_fence base; +/** @gt: GT which fence belong to */ +struct xe_gt *gt;/** @link: link into list of pending tlb fences */ struct list_head link;/** @seqno: seqno of TLB invalidation to signal fence one */
drivers/gpu/drm/xe/xe_vm.c +3 -1
@@ -3218,8 +3218,10 @@ int xe_vm_invalidate_vma(struct xe_vma *vma)*/ ret =xe_gt_tlb_invalidation_vma (tile->primary_gt, &fence[id], vma); -if (ret <0 ) +if (ret <0 ) { +xe_gt_tlb_invalidation_fence_fini (&fence[id]); goto wait; + } tile_needs_invalidate |=BIT (id); }
Signature · 1 line
2.34.1