Keep device awake TLB invalidations and G2H in flight
Subsystems (2)
Matthew Brost
maintainerv5 cover Keep device awake TLB invalidations and G2H in flight
Combine [1], [2] to fix PM issues
[1] https://patchwork.freedesktop.org/series/136226/
[2] https://patchwork.freedesktop.org/series/136219/
Matthew Brost (4):
drm/xe: Add xe_gt_tlb_invalidation_fence_init helper drm/xe: Drop xe_gt_tlb_invalidation_wait
drm/xe: Hold a PM ref when GT TLB invalidations are inflight drm/xe: Build PM into GuC CT layer
drivers/gpu/drm/xe/xe_device.c | 4 + drivers/gpu/drm/xe/xe_gt_tlb_invalidation.c | 201 ++++++++++-------- drivers/gpu/drm/xe/xe_gt_tlb_invalidation.h | 12 +- .../gpu/drm/xe/xe_gt_tlb_invalidation_types.h | 4 + drivers/gpu/drm/xe/xe_guc_ct.c | 10 +- drivers/gpu/drm/xe/xe_guc_submit.c | 4 + drivers/gpu/drm/xe/xe_pt.c | 26 +-- drivers/gpu/drm/xe/xe_vm.c | 32 +-- 8 files changed, 160 insertions(+), 133 deletions(-)
Signature · 1 line
2.34.1
Matthew Brost
maintainerPATCH v5 1/4 drm/xe: Add xe_gt_tlb_invalidation_fence_init helper
Other layers should not be touching struct xe_gt_tlb_invalidation_fence directly, add helper for initialization.
v2:
- Add dma_fence_get and list init to xe_gt_tlb_invalidation_fence_init
--- drivers/gpu/drm/xe/xe_gt_tlb_invalidation.c | 36 +++++++++++++++++++++ drivers/gpu/drm/xe/xe_gt_tlb_invalidation.h | 3 ++ drivers/gpu/drm/xe/xe_pt.c | 26 +-------------- 3 files changed, 40 insertions(+), 25 deletions(-)
drivers/gpu/drm/xe/xe_gt_tlb_invalidation.c +36 -0
@@ -508,3 +508,39 @@ int xe_guc_tlb_invalidation_done_handler(struct xe_guc *guc, u32 *msg, u32 len)return 0 ; } + +static const char * +invalidation_fence_get_driver_name (struct dma_fence *dma_fence) +{ +return "xe" ; +} + +static const char * +invalidation_fence_get_timeline_name (struct dma_fence *dma_fence) +{ +return "invalidation_fence" ; +} + +static const struct dma_fence_ops invalidation_fence_ops = { + .get_driver_name = invalidation_fence_get_driver_name, + .get_timeline_name = invalidation_fence_get_timeline_name, +}; + +/** +* xe_gt_tlb_invalidation_fence_init - Initialize TLB invalidation fence +* @gt: GT +* @fence: TLB invalidation fence to initialize +* +* Initialize TLB invalidation fence for use +*/ +void xe_gt_tlb_invalidation_fence_init (struct xe_gt *gt, +struct xe_gt_tlb_invalidation_fence *fence) +{ +spin_lock_irq (>->tlb_invalidation.lock); +dma_fence_init (&fence->base, &invalidation_fence_ops, + >->tlb_invalidation.lock, +dma_fence_context_alloc (1 ),1 ); +spin_unlock_irq (>->tlb_invalidation.lock); +INIT_LIST_HEAD (&fence->link); +dma_fence_get (&fence->base); +}
drivers/gpu/drm/xe/xe_gt_tlb_invalidation.h +3 -0
@@ -26,4 +26,7 @@ int xe_gt_tlb_invalidation_range(struct xe_gt *gt,int xe_gt_tlb_invalidation_wait (struct xe_gt *gt,int seqno);int xe_guc_tlb_invalidation_done_handler (struct xe_guc *guc,u32 *msg,u32 len); +void xe_gt_tlb_invalidation_fence_init (struct xe_gt *gt, +struct xe_gt_tlb_invalidation_fence *fence); +#endif /* _XE_GT_TLB_INVALIDATION_ */
drivers/gpu/drm/xe/xe_pt.c +1 -25
@@ -1317,23 +1317,6 @@ struct invalidation_fence {u32 asid; }; -static const char * -invalidation_fence_get_driver_name (struct dma_fence *dma_fence) -{ -return "xe" ; -} - -static const char * -invalidation_fence_get_timeline_name (struct dma_fence *dma_fence) -{ -return "invalidation_fence" ; -} - -static const struct dma_fence_ops invalidation_fence_ops = { - .get_driver_name = invalidation_fence_get_driver_name, - .get_timeline_name = invalidation_fence_get_timeline_name, -}; -static void invalidation_fence_cb (struct dma_fence *fence,struct dma_fence_cb *cb) {
@@ -1372,15 +1355,8 @@ static void invalidation_fence_init(struct xe_gt *gt,trace_xe_gt_tlb_invalidation_fence_create (gt_to_xe (gt), &ifence->base); -spin_lock_irq (>->tlb_invalidation.lock); -dma_fence_init (&ifence->base.base, &invalidation_fence_ops, - >->tlb_invalidation.lock, -dma_fence_context_alloc (1 ),1 ); -spin_unlock_irq (>->tlb_invalidation.lock); - -INIT_LIST_HEAD (&ifence->base.link); +xe_gt_tlb_invalidation_fence_init (gt, &ifence->base); -dma_fence_get (&ifence->base.base);/* Ref for caller */ ifence->fence = fence; ifence->gt = gt; ifence->start = start;
Signature · 1 line
2.34.1
Matthew Brost
maintainerPATCH v5 2/4 drm/xe: Drop xe_gt_tlb_invalidation_wait
In reply to Matthew BrostHaving two methods to wait on GT TLB invalidations is not ideal. Remove xe_gt_tlb_invalidation_wait and only use GT TLB invalidation fences.
In addition to two methods being less than ideal, once GT TLB invalidations are coalesced the seqno cannot be assigned during xe_gt_tlb_invalidation_ggtt/range. Thus xe_gt_tlb_invalidation_wait would not have a seqno to wait one. A fence however can be armed and later signaled.
v3:
- Add explaination about coalescing to commit message
v4:
- Don't put dma fence if defined on stack (CI)
v5:
- Initialize ret to zero (CI)
v6:
- Use invalidation_fence_signal helper in tlb timeout (Matthew Auld)
--- drivers/gpu/drm/xe/xe_gt_tlb_invalidation.c | 148 ++++++++------------ drivers/gpu/drm/xe/xe_gt_tlb_invalidation.h | 10 +- drivers/gpu/drm/xe/xe_pt.c | 2 +- drivers/gpu/drm/xe/xe_vm.c | 30 ++-- 4 files changed, 80 insertions(+), 110 deletions(-)
drivers/gpu/drm/xe/xe_gt_tlb_invalidation.c +55 -93
@@ -17,6 +17,8 @@#include "xe_trace.h" #include "regs/xe_guc_regs.h" +#define FENCE_STACK_BIT DMA_FENCE_FLAG_USER_BITS +/* * TLB inval depends on pending commands in the CT queue and then the real * invalidation time. Double up the time to process full CT queue
@@ -33,6 +35,23 @@ static long tlb_timeout_jiffies(struct xe_gt *gt)return hw_tlb_timeout +2 * delay; } +static void +__invalidation_fence_signal (struct xe_device *xe,struct xe_gt_tlb_invalidation_fence *fence) +{ +bool stack =test_bit (FENCE_STACK_BIT , &fence->base.flags); + +trace_xe_gt_tlb_invalidation_fence_signal (xe, fence); +dma_fence_signal (&fence->base); +if (!stack) +dma_fence_put (&fence->base); +} + +static void +invalidation_fence_signal (struct xe_device *xe,struct xe_gt_tlb_invalidation_fence *fence) +{ +list_del (&fence->link); +__invalidation_fence_signal (xe, fence); +}static void xe_gt_tlb_fence_timeout (struct work_struct *work) {
@@ -54,10 +73,8 @@ static void xe_gt_tlb_fence_timeout(struct work_struct *work)xe_gt_err (gt,"TLB invalidation fence timeout, seqno=%d recv=%d" , fence->seqno, gt->tlb_invalidation.seqno_recv); -list_del (&fence->link); fence->base.error = -ETIME ; -dma_fence_signal (&fence->base); -dma_fence_put (&fence->base); +invalidation_fence_signal (xe, fence); }if (!list_empty (>->tlb_invalidation.pending_fences)) queue_delayed_work(system_wq,
@@ -87,21 +104,6 @@ int xe_gt_tlb_invalidation_init(struct xe_gt *gt)return 0 ; } -static void -__invalidation_fence_signal (struct xe_device *xe,struct xe_gt_tlb_invalidation_fence *fence) -{ -trace_xe_gt_tlb_invalidation_fence_signal (xe, fence); -dma_fence_signal (&fence->base); -dma_fence_put (&fence->base); -} - -static void -invalidation_fence_signal (struct xe_device *xe,struct xe_gt_tlb_invalidation_fence *fence) -{ -list_del (&fence->link); -__invalidation_fence_signal (xe, fence); -} -/** * xe_gt_tlb_invalidation_reset - Initialize GT TLB invalidation reset * @gt: graphics tile
@@ -111,7 +113,6 @@ invalidation_fence_signal(struct xe_device *xe, struct xe_gt_tlb_invalidation_fevoid xe_gt_tlb_invalidation_reset (struct xe_gt *gt) {struct xe_gt_tlb_invalidation_fence *fence, *next; -struct xe_guc *guc = >->uc.guc;int pending_seqno;/*
@@ -134,7 +135,6 @@ void xe_gt_tlb_invalidation_reset(struct xe_gt *gt)else pending_seqno = gt->tlb_invalidation.seqno -1 ;WRITE_ONCE (gt->tlb_invalidation.seqno_recv, pending_seqno); -wake_up_all (&guc->ct.wq);list_for_each_entry_safe (fence, next, >->tlb_invalidation.pending_fences, link)
@@ -165,6 +165,8 @@ static int send_tlb_invalidation(struct xe_guc *guc,int seqno;int ret; +xe_gt_assert (gt, fence); +/* * XXX: The seqno algorithm relies on TLB invalidation being processed * in order which they currently are, if that changes the algorithm will
@@ -173,10 +175,8 @@ static int send_tlb_invalidation(struct xe_guc *guc,mutex_lock (&guc->ct.lock); seqno = gt->tlb_invalidation.seqno; -if (fence) { - fence->seqno = seqno; -trace_xe_gt_tlb_invalidation_fence_send (xe, fence); - } + fence->seqno = seqno; +trace_xe_gt_tlb_invalidation_fence_send (xe, fence); action[1 ] = seqno; ret =xe_guc_ct_send_locked (&guc->ct, action, len,G2H_LEN_DW_TLB_INVALIDATE ,1 );
@@ -209,7 +209,6 @@ static int send_tlb_invalidation(struct xe_guc *guc,TLB_INVALIDATION_SEQNO_MAX ;if (!gt->tlb_invalidation.seqno) gt->tlb_invalidation.seqno =1 ; - ret = seqno; }mutex_unlock (&guc->ct.lock);
@@ -223,14 +222,16 @@ static int send_tlb_invalidation(struct xe_guc *guc,/** * xe_gt_tlb_invalidation_guc - Issue a TLB invalidation on this GT for the GuC * @gt: graphics tile +* @fence: invalidation fence which will be signal on TLB invalidation +* completion * * Issue a TLB invalidation for the GuC. Completion of TLB is asynchronous and -* caller can use seqno + xe_gt_tlb_invalidation_wait to wait for completion. +* caller can use the invalidation fence to wait for completion. * -* Return: Seqno which can be passed to xe_gt_tlb_invalidation_wait on success, -* negative error code on error. +* Return: 0 on success, negative error code on error */ -static int xe_gt_tlb_invalidation_guc (struct xe_gt *gt) +static int xe_gt_tlb_invalidation_guc (struct xe_gt *gt, +struct xe_gt_tlb_invalidation_fence *fence) {u32 action[] = {XE_GUC_ACTION_TLB_INVALIDATION ,
@@ -238,7 +239,7 @@ static int xe_gt_tlb_invalidation_guc(struct xe_gt *gt) MAKE_INVAL_OP(XE_GUC_TLB_INVAL_GUC), }; -return send_tlb_invalidation (>->uc.guc,NULL , action, +return send_tlb_invalidation (>->uc.guc, fence, action,ARRAY_SIZE (action)); }
@@ -257,13 +258,15 @@ int xe_gt_tlb_invalidation_ggtt(struct xe_gt *gt)if (xe_guc_ct_enabled (>->uc.guc.ct) && gt->uc.guc.submission_state.enabled) { -int seqno; +struct xe_gt_tlb_invalidation_fence fence; +int ret; - seqno =xe_gt_tlb_invalidation_guc (gt); -if (seqno <=0 ) -return seqno; +xe_gt_tlb_invalidation_fence_init (gt, &fence, true); + ret =xe_gt_tlb_invalidation_guc (gt, &fence); +if (ret <0 ) +return ret; -xe_gt_tlb_invalidation_wait (gt, seqno); +xe_gt_tlb_invalidation_fence_wait (&fence); }else if (xe_device_uc_enabled (xe) && !xe_device_wedged (xe)) {if (IS_SRIOV_VF (xe))return 0 ;
@@ -290,18 +293,16 @@ int xe_gt_tlb_invalidation_ggtt(struct xe_gt *gt)* * @gt: graphics tile * @fence: invalidation fence which will be signal on TLB invalidation -* completion, can be NULL +* completion * @start: start address * @end: end address * @asid: address space id * * Issue a range based TLB invalidation if supported, if not fallback to a full -* TLB invalidation. Completion of TLB is asynchronous and caller can either use -* the invalidation fence or seqno + xe_gt_tlb_invalidation_wait to wait for -* completion. +* TLB invalidation. Completion of TLB is asynchronous and caller can use +* the invalidation fence to wait for completion. * -* Return: Seqno which can be passed to xe_gt_tlb_invalidation_wait on success, -* negative error code on error. +* Return: Negative error code on error, 0 on success */ int xe_gt_tlb_invalidation_range(struct xe_gt *gt, struct xe_gt_tlb_invalidation_fence *fence,
@@ -312,11 +313,11 @@ int xe_gt_tlb_invalidation_range(struct xe_gt *gt,u32 action[MAX_TLB_INVALIDATION_LEN ];int len =0 ; +xe_gt_assert (gt, fence); +/* Execlists not supported */ if (gt_to_xe (gt)->info.force_execlist) { -if (fence) -__invalidation_fence_signal (xe, fence); - +__invalidation_fence_signal (xe, fence);return 0 ; }
@@ -382,12 +383,10 @@ int xe_gt_tlb_invalidation_range(struct xe_gt *gt,* @vma: VMA to invalidate * * Issue a range based TLB invalidation if supported, if not fallback to a full -* TLB invalidation. Completion of TLB is asynchronous and caller can either use -* the invalidation fence or seqno + xe_gt_tlb_invalidation_wait to wait for -* completion. +* TLB invalidation. Completion of TLB is asynchronous and caller can use +* the invalidation fence to wait for completion. * -* Return: Seqno which can be passed to xe_gt_tlb_invalidation_wait on success, -* negative error code on error. +* Return: Negative error code on error, 0 on success */ int xe_gt_tlb_invalidation_vma(struct xe_gt *gt, struct xe_gt_tlb_invalidation_fence *fence,
@@ -400,43 +399,6 @@ int xe_gt_tlb_invalidation_vma(struct xe_gt *gt, xe_vma_vm(vma)->usm.asid); } -/** -* xe_gt_tlb_invalidation_wait - Wait for TLB to complete -* @gt: graphics tile -* @seqno: seqno to wait which was returned from xe_gt_tlb_invalidation -* -* Wait for tlb_timeout_jiffies() for a TLB invalidation to complete. -* -* Return: 0 on success, -ETIME on TLB invalidation timeout -*/ -int xe_gt_tlb_invalidation_wait (struct xe_gt *gt,int seqno) -{ -struct xe_guc *guc = >->uc.guc; -int ret; - -/* Execlists not supported */ -if (gt_to_xe (gt)->info.force_execlist) -return 0 ; - -/* -* XXX: See above, this algorithm only works if seqno are always in -* order -*/ - ret =wait_event_timeout (guc->ct.wq, -tlb_invalidation_seqno_past (gt, seqno), -tlb_timeout_jiffies (gt)); -if (!ret) { -struct drm_printer p =xe_gt_err_printer (gt); - -xe_gt_err (gt,"TLB invalidation time'd out, seqno=%d, recv=%d\n" , - seqno, gt->tlb_invalidation.seqno_recv); -xe_guc_ct_print (&guc->ct, &p, true); -return -ETIME ; - } - -return 0 ; -} -/** * xe_guc_tlb_invalidation_done_handler - TLB invalidation done handler * @guc: guc
@@ -480,12 +442,7 @@ int xe_guc_tlb_invalidation_done_handler(struct xe_guc *guc, u32 *msg, u32 len)return 0 ; } -/* -* wake_up_all() and wait_event_timeout() already have the correct -* barriers. -*/ WRITE_ONCE (gt->tlb_invalidation.seqno_recv, msg[0 ]); -wake_up_all (&guc->ct.wq);list_for_each_entry_safe (fence, next, >->tlb_invalidation.pending_fences, link) {
@@ -530,11 +487,13 @@ static const struct dma_fence_ops invalidation_fence_ops = {* xe_gt_tlb_invalidation_fence_init - Initialize TLB invalidation fence * @gt: GT * @fence: TLB invalidation fence to initialize +* @stack: fence is stack variable * * Initialize TLB invalidation fence for use */ void xe_gt_tlb_invalidation_fence_init (struct xe_gt *gt, -struct xe_gt_tlb_invalidation_fence *fence) +struct xe_gt_tlb_invalidation_fence *fence, +bool stack) {spin_lock_irq (>->tlb_invalidation.lock); dma_fence_init(&fence->base, &invalidation_fence_ops,
@@ -542,5 +501,8 @@ void xe_gt_tlb_invalidation_fence_init(struct xe_gt *gt, dma_fence_context_alloc(1), 1);spin_unlock_irq (>->tlb_invalidation.lock);INIT_LIST_HEAD (&fence->link); -dma_fence_get (&fence->base); +if (stack) +set_bit (FENCE_STACK_BIT , &fence->base.flags); +else +dma_fence_get (&fence->base); }
drivers/gpu/drm/xe/xe_gt_tlb_invalidation.h +8 -2
@@ -23,10 +23,16 @@ int xe_gt_tlb_invalidation_vma(struct xe_gt *gt,int xe_gt_tlb_invalidation_range (struct xe_gt *gt,struct xe_gt_tlb_invalidation_fence *fence,u64 start,u64 end,u32 asid); -int xe_gt_tlb_invalidation_wait (struct xe_gt *gt,int seqno);int xe_guc_tlb_invalidation_done_handler (struct xe_guc *guc,u32 *msg,u32 len);void xe_gt_tlb_invalidation_fence_init (struct xe_gt *gt, -struct xe_gt_tlb_invalidation_fence *fence); +struct xe_gt_tlb_invalidation_fence *fence, +bool stack); + +static inline void +xe_gt_tlb_invalidation_fence_wait (struct xe_gt_tlb_invalidation_fence *fence) +{ +dma_fence_wait (&fence->base, false); +}#endif /* _XE_GT_TLB_INVALIDATION_ */
drivers/gpu/drm/xe/xe_pt.c +1 -1
@@ -1355,7 +1355,7 @@ static void invalidation_fence_init(struct xe_gt *gt,trace_xe_gt_tlb_invalidation_fence_create (gt_to_xe (gt), &ifence->base); -xe_gt_tlb_invalidation_fence_init (gt, &ifence->base); +xe_gt_tlb_invalidation_fence_init (gt, &ifence->base, false); ifence->fence = fence; ifence->gt = gt;
drivers/gpu/drm/xe/xe_vm.c +16 -14
@@ -3184,10 +3184,10 @@ int xe_vm_invalidate_vma(struct xe_vma *vma) {struct xe_device *xe =xe_vma_vm (vma)->xe;struct xe_tile *tile; +struct xe_gt_tlb_invalidation_fence fence[XE_MAX_TILES_PER_DEVICE ];u32 tile_needs_invalidate =0 ; -int seqno[XE_MAX_TILES_PER_DEVICE ];u8 id; -int ret; +int ret =0 ;xe_assert (xe, !xe_vma_is_null (vma));trace_xe_vma_invalidate (vma);
@@ -3212,29 +3212,31 @@ int xe_vm_invalidate_vma(struct xe_vma *vma)for_each_tile (tile, xe, id) {if (xe_pt_zap_ptes (tile, vma)) { - tile_needs_invalidate |=BIT (id);xe_device_wmb (xe); +xe_gt_tlb_invalidation_fence_init (tile->primary_gt, + &fence[id], true); +/* * FIXME: We potentially need to invalidate multiple * GTs within the tile */ - seqno[id] =xe_gt_tlb_invalidation_vma (tile->primary_gt,NULL , vma); -if (seqno[id] <0 ) -return seqno[id]; - } - } - -for_each_tile (tile, xe, id) { -if (tile_needs_invalidate &BIT (id)) { - ret =xe_gt_tlb_invalidation_wait (tile->primary_gt, seqno[id]); + ret =xe_gt_tlb_invalidation_vma (tile->primary_gt, + &fence[id], vma);if (ret <0 ) -return ret; + goto wait; + + tile_needs_invalidate |=BIT (id); } } +wait: +for_each_tile (tile, xe, id) +if (tile_needs_invalidate &BIT (id)) +xe_gt_tlb_invalidation_fence_wait (&fence[id]); + vma->tile_invalidated = vma->tile_mask; -return 0 ; +return ret; }struct xe_vm_snapshot {
Signature · 1 line
2.34.1
Matthew Brost
maintainerPATCH v5 3/4 drm/xe: Hold a PM ref when GT TLB invalidations are inflight
In reply to Matthew BrostAvoid GT TLB invalidation timeouts by holding a PM ref when invalidations are inflight.
v2:
- Drop PM ref before signaling fence (CI)
v3:
- Move invalidation_fence_signal helper in tlb timeout to previous patch (Matthew Auld)
--- drivers/gpu/drm/xe/xe_gt_tlb_invalidation.c | 23 +++++++++++++++++-- drivers/gpu/drm/xe/xe_gt_tlb_invalidation.h | 1 + .../gpu/drm/xe/xe_gt_tlb_invalidation_types.h | 4 ++++ drivers/gpu/drm/xe/xe_vm.c | 4 +++- 4 files changed, 29 insertions(+), 3 deletions(-)
drivers/gpu/drm/xe/xe_gt_tlb_invalidation.c +21 -2
@@ -13,6 +13,7 @@#include "xe_guc.h" #include "xe_guc_ct.h" #include "xe_mmio.h" +#include "xe_pm.h" #include "xe_sriov.h" #include "xe_trace.h" #include "regs/xe_guc_regs.h"
@@ -41,6 +42,7 @@ __invalidation_fence_signal(struct xe_device *xe, struct xe_gt_tlb_invalidation_bool stack =test_bit (FENCE_STACK_BIT , &fence->base.flags);trace_xe_gt_tlb_invalidation_fence_signal (xe, fence); +xe_gt_tlb_invalidation_fence_fini (fence);dma_fence_signal (&fence->base);if (!stack)dma_fence_put (&fence->base);
@@ -263,8 +265,10 @@ int xe_gt_tlb_invalidation_ggtt(struct xe_gt *gt)xe_gt_tlb_invalidation_fence_init (gt, &fence, true); ret =xe_gt_tlb_invalidation_guc (gt, &fence); -if (ret <0 ) +if (ret <0 ) { +xe_gt_tlb_invalidation_fence_fini (&fence);return ret; + }xe_gt_tlb_invalidation_fence_wait (&fence); }else if (xe_device_uc_enabled (xe) && !xe_device_wedged (xe)) {
@@ -489,12 +493,15 @@ static const struct dma_fence_ops invalidation_fence_ops = {* @fence: TLB invalidation fence to initialize * @stack: fence is stack variable * -* Initialize TLB invalidation fence for use +* Initialize TLB invalidation fence for use. xe_gt_tlb_invalidation_fence_fini +* must be called if fence is not signaled. */ void xe_gt_tlb_invalidation_fence_init (struct xe_gt *gt,struct xe_gt_tlb_invalidation_fence *fence,bool stack) { +xe_pm_runtime_get_noresume (gt_to_xe (gt)); +spin_lock_irq (>->tlb_invalidation.lock); dma_fence_init(&fence->base, &invalidation_fence_ops, >->tlb_invalidation.lock,
@@ -505,4 +512,16 @@ void xe_gt_tlb_invalidation_fence_init(struct xe_gt *gt,set_bit (FENCE_STACK_BIT , &fence->base.flags); else dma_fence_get(&fence->base); + fence->gt = gt; +} + +/** +* xe_gt_tlb_invalidation_fence_fini - Finalize TLB invalidation fence +* @fence: TLB invalidation fence to finalize +* +* Drop PM ref which fence took durinig init. +*/ +void xe_gt_tlb_invalidation_fence_fini (struct xe_gt_tlb_invalidation_fence *fence) +{ +xe_pm_runtime_put (gt_to_xe (fence->gt)); }
drivers/gpu/drm/xe/xe_gt_tlb_invalidation.h +1 -0
@@ -28,6 +28,7 @@ int xe_guc_tlb_invalidation_done_handler(struct xe_guc *guc, u32 *msg, u32 len);void xe_gt_tlb_invalidation_fence_init (struct xe_gt *gt,struct xe_gt_tlb_invalidation_fence *fence,bool stack); +void xe_gt_tlb_invalidation_fence_fini (struct xe_gt_tlb_invalidation_fence *fence);static inline void xe_gt_tlb_invalidation_fence_wait (struct xe_gt_tlb_invalidation_fence *fence)
drivers/gpu/drm/xe/xe_gt_tlb_invalidation_types.h +4 -0
@@ -8,6 +8,8 @@#include <linux/dma-fence.h> +struct xe_gt ; +/** * struct xe_gt_tlb_invalidation_fence - XE GT TLB invalidation fence *
@@ -17,6 +19,8 @@struct xe_gt_tlb_invalidation_fence {/** @base: dma fence base */ struct dma_fence base; +/** @gt: GT which fence belong to */ +struct xe_gt *gt;/** @link: link into list of pending tlb fences */ struct list_head link;/** @seqno: seqno of TLB invalidation to signal fence one */
drivers/gpu/drm/xe/xe_vm.c +3 -1
@@ -3222,8 +3222,10 @@ int xe_vm_invalidate_vma(struct xe_vma *vma)*/ ret =xe_gt_tlb_invalidation_vma (tile->primary_gt, &fence[id], vma); -if (ret <0 ) +if (ret <0 ) { +xe_gt_tlb_invalidation_fence_fini (&fence[id]); goto wait; + } tile_needs_invalidate |=BIT (id); }
Signature · 1 line
2.34.1
Matthew Brost
maintainerPATCH v5 4/4 drm/xe: Build PM into GuC CT layer
In reply to Matthew BrostTake PM ref when any G2H are outstanding, drop when none are outstanding.
To safely ensure we have PM ref when in the GuC CT layer, a PM ref needs to be held when scheduler messages are pending too.
v2:
- Add outer PM protections to xe_file_close (CI)
v3:
- Only take PM ref 0->1 and drop on 1->0 (Matthew Auld)
v4:
- Add assert to G2H increment function
v5:
- Rebase
--- drivers/gpu/drm/xe/xe_device.c | 4 ++++ drivers/gpu/drm/xe/xe_guc_ct.c | 10 +++++++++- drivers/gpu/drm/xe/xe_guc_submit.c | 4 ++++ 3 files changed, 17 insertions(+), 1 deletion(-)
drivers/gpu/drm/xe/xe_device.c +4 -0
@@ -146,6 +146,8 @@ static void xe_file_close(struct drm_device *dev, struct drm_file *file)struct xe_exec_queue *q;unsigned long idx; +xe_pm_runtime_get (xe); +/* * No need for exec_queue.lock here as there is no contention for it * when FD is closing as IOCTLs presumably can't be modifying the
@@ -162,6 +164,8 @@ static void xe_file_close(struct drm_device *dev, struct drm_file *file)mutex_unlock (&xef->vm.lock);xe_file_put (xef); + +xe_pm_runtime_put (xe); }static const struct drm_ioctl_desc xe_ioctls[] = {
drivers/gpu/drm/xe/xe_guc_ct.c +9 -1
@@ -327,6 +327,8 @@ static void xe_guc_ct_set_state(struct xe_guc_ct *ct,xe_gt_assert (ct_to_gt (ct), ct->g2h_outstanding ==0 || state ==XE_GUC_CT_STATE_STOPPED ); +if (ct->g2h_outstanding) +xe_pm_runtime_put (ct_to_xe (ct)); ct->g2h_outstanding =0 ; ct->state = state;
@@ -495,10 +497,15 @@ static void h2g_reserve_space(struct xe_guc_ct *ct, u32 cmd_len)static void __g2h_reserve_space (struct xe_guc_ct *ct,u32 g2h_len,u32 num_g2h) {xe_gt_assert (ct_to_gt (ct), g2h_len <= ct->ctbs.g2h.info.space); +xe_gt_assert (ct_to_gt (ct), (!g2h_len && !num_g2h) || + (g2h_len && num_g2h));if (g2h_len) {lockdep_assert_held (&ct->fast_lock); +if (!ct->g2h_outstanding) +xe_pm_runtime_get_noresume (ct_to_xe (ct)); + ct->ctbs.g2h.info.space -= g2h_len; ct->g2h_outstanding += num_g2h; }
@@ -511,7 +518,8 @@ static void __g2h_release_space(struct xe_guc_ct *ct, u32 g2h_len) ct->ctbs.g2h.info.size - ct->ctbs.g2h.info.resv_space); ct->ctbs.g2h.info.space += g2h_len; - --ct->g2h_outstanding; +if (!--ct->g2h_outstanding) +xe_pm_runtime_put (ct_to_xe (ct)); }static void g2h_release_space (struct xe_guc_ct *ct,u32 g2h_len)
drivers/gpu/drm/xe/xe_guc_submit.c +4 -0
@@ -1413,6 +1413,8 @@ static void guc_exec_queue_process_msg(struct xe_sched_msg *msg)default :XE_WARN_ON ("Unknown message type" ); } + +xe_pm_runtime_put (guc_to_xe (exec_queue_to_guc (msg->private_data))); }static const struct drm_sched_backend_ops drm_sched_ops = {
@@ -1503,6 +1505,8 @@ static void guc_exec_queue_kill(struct xe_exec_queue *q)static void guc_exec_queue_add_msg (struct xe_exec_queue *q,struct xe_sched_msg *msg,u32 opcode) { +xe_pm_runtime_get_noresume (guc_to_xe (exec_queue_to_guc (q))); +INIT_LIST_HEAD (&msg->link); msg->opcode = opcode; msg->private_data = q;
Signature · 1 line
2.34.1
Patchwork
CI✓ CI.Patch_applied: success for Keep device awake TLB invalidations and G2H in flight (rev2)
In reply to Matthew Brost== Series Details ==
== Series Details ==
Series: Keep device awake TLB invalidations and G2H in flight (rev2)
URL : https://patchwork.freedesktop.org/series/136256/
State : success
== Summary ==
=== Applying kernel patches on branch 'drm-tip' with base: ===
Base commit: eb6045a759ea drm-tip: 2024y-07m-19d-11h-07m-10s UTC integration manifest === git am output follows ===
Applying: drm/xe: Add xe_gt_tlb_invalidation_fence_init helper
Applying: drm/xe: Drop xe_gt_tlb_invalidation_wait
Applying: drm/xe: Hold a PM ref when GT TLB invalidations are inflight
Applying: drm/xe: Build PM into GuC CT layer
✓ CI.checkpatch: success for Keep device awake TLB invalidations and G2H in flight (rev2)
In reply to Matthew Brost== Series Details ==
== Series Details ==
Series: Keep device awake TLB invalidations and G2H in flight (rev2)
URL : https://patchwork.freedesktop.org/series/136256/
State : success
== Summary ==
+ KERNEL=/kernel
+ git clone https://gitlab.freedesktop.org/drm/maintainer-tools mt Cloning into 'mt'...
warning: redirecting to https://gitlab.freedesktop.org/drm/maintainer-tools.git/
+ git -C mt rev-list -n1 origin/master 5ce3e132caaa5b45e5e50201b574a097d130967c
+ cd /kernel
+ git config --global --add safe.directory /kernel
+ git log -n1
commit 25e2d94e6396189ca52580e89f926506403ab9ef
Author: Matthew Brost <[email protected]> Date: Fri Jul 19 09:50:12 2024 -0700
drm/xe: Build PM into GuC CT layer
Take PM ref when any G2H are outstanding, drop when none are outstanding.
To safely ensure we have PM ref when in the GuC CT layer, a PM ref needs to be held when scheduler messages are pending too.
v2:
- Add outer PM protections to xe_file_close (CI)
v3:
- Only take PM ref 0->1 and drop on 1->0 (Matthew Auld)
v4:
- Add assert to G2H increment function
v5:
- Rebase
+ /mt/dim checkpatch eb6045a759ea13e8d159bdaea423e904b9e3717b drm-intel bc54ff07979f drm/xe: Add xe_gt_tlb_invalidation_fence_init helper d8d6210ed001 drm/xe: Drop xe_gt_tlb_invalidation_wait 3365da42248a drm/xe: Hold a PM ref when GT TLB invalidations are inflight 25e2d94e6396 drm/xe: Build PM into GuC CT layer
Patchwork
CI✗ CI.KUnit: failure for Keep device awake TLB invalidations and G2H in flight (rev2)
In reply to Matthew Brost== Series Details ==
== Series Details ==
Series: Keep device awake TLB invalidations and G2H in flight (rev2)
URL : https://patchwork.freedesktop.org/series/136256/
State : failure
== Summary ==
+ trap cleanup EXIT
+ /kernel/tools/testing/kunit/kunit.py run --kunitconfig /kernel/drivers/gpu/drm/xe/.kunitconfig ERROR:root:../drivers/gpu/drm/xe/xe_device.c: In function ‘xe_file_close’:../drivers/gpu/drm/xe/xe_device.c:149:20: error: ‘xe’ undeclared (first use in this function); did you mean ‘xef’?
149 | xe_pm_runtime_get(xe);
| ^~
| xef../drivers/gpu/drm/xe/xe_device.c:149:20: note: each undeclared identifier is reported only once for each function it appears in make[7]: *** [../scripts/Makefile.build:244: drivers/gpu/drm/xe/xe_device.o] Error 1
make[7]: *** Waiting for unfinished jobs....
make[6]: *** [../scripts/Makefile.build:485: drivers/gpu/drm/xe] Error 2
make[5]: *** [../scripts/Makefile.build:485: drivers/gpu/drm] Error 2
make[4]: *** [../scripts/Makefile.build:485: drivers/gpu] Error 2
make[4]: *** Waiting for unfinished jobs....
make[3]: *** [../scripts/Makefile.build:485: drivers] Error 2
make[3]: *** Waiting for unfinished jobs....../lib/iomap.c:156:5: warning: no previous prototype for ‘ioread64_lo_hi’ [-Wmissing-prototypes]
156 | u64 ioread64_lo_hi(const void __iomem *addr)
| ^~~~~~~~~~~~~~
../lib/iomap.c:163:5: warning: no previous prototype for ‘ioread64_hi_lo’ [-Wmissing-prototypes]
163 | u64 ioread64_hi_lo(const void __iomem *addr)
| ^~~~~~~~~~~~~~
../lib/iomap.c:170:5: warning: no previous prototype for ‘ioread64be_lo_hi’ [-Wmissing-prototypes]
170 | u64 ioread64be_lo_hi(const void __iomem *addr)
| ^~~~~~~~~~~~~~~~
../lib/iomap.c:178:5: warning: no previous prototype for ‘ioread64be_hi_lo’ [-Wmissing-prototypes]
178 | u64 ioread64be_hi_lo(const void __iomem *addr)
| ^~~~~~~~~~~~~~~~
../lib/iomap.c:264:6: warning: no previous prototype for ‘iowrite64_lo_hi’ [-Wmissing-prototypes]
264 | void iowrite64_lo_hi(u64 val, void __iomem *addr)
| ^~~~~~~~~~~~~~~
../lib/iomap.c:272:6: warning: no previous prototype for ‘iowrite64_hi_lo’ [-Wmissing-prototypes]
272 | void iowrite64_hi_lo(u64 val, void __iomem *addr)
| ^~~~~~~~~~~~~~~
../lib/iomap.c:280:6: warning: no previous prototype for ‘iowrite64be_lo_hi’ [-Wmissing-prototypes]
280 | void iowrite64be_lo_hi(u64 val, void __iomem *addr)
| ^~~~~~~~~~~~~~~~~
../lib/iomap.c:288:6: warning: no previous prototype for ‘iowrite64be_hi_lo’ [-Wmissing-prototypes]
288 | void iowrite64be_hi_lo(u64 val, void __iomem *addr)
| ^~~~~~~~~~~~~~~~~make[2]: *** [/kernel/Makefile:1934: .] Error 2 make[1]: *** [/kernel/Makefile:240: __sub-make] Error 2
make: *** [Makefile:240: __sub-make] Error 2
[16:55:05] Configuring KUnit Kernel ...
Generating .config ...
Populating config with:
$ make ARCH=um O=.kunit olddefconfig [16:55:10] Building KUnit Kernel ... Populating config with: $ make ARCH=um O=.kunit olddefconfig Building with: $ make ARCH=um O=.kunit --jobs=48 + cleanup ++ stat -c %u:%g /kernel + chown -R 1003:1003 /kernel