Skip to contentSkip to conversation

drm/i915: Implement range-based TLB

RFC PATCH In review Jonathan Cavitt 3 participants 19 messages Oct 10, 2023 – Oct 21, 2023

Subsystems (2)
lore
Share & tools
lore
#0 lore raw

cover drm/i915: Implement range-based TLB

Implement range-based TLB invalidations on top of GuC-based TLB invalidations. This is the future plan for GuC-based TLB invalidations because it helps improve performance over performing full tlb invalidations all the time.

Jonathan Cavitt (7):
drm/i915: Add GuC TLB Invalidation device info flags drm/i915/guc: Add CT size delay helper
drm/i915: No TLB invalidation on suspended GT drm/i915: No TLB invalidation on wedged GT drm/i915/gt: Increase sleep in gt_tlb selftest sanitycheck drm/i915: Enable GuC TLB invalidations for MTL drm/i915: Use selective tlb invalidations where supported

Prathap Kumar Valsan (3):
drm/i915: Define and use GuC and CTB TLB invalidation routines drm/i915: Define GuC Based TLB invalidation routines drm/i915: Add generic interface for tlb invalidation

 drivers/gpu/drm/i915/gt/intel_ggtt.c          |  34 +-
 drivers/gpu/drm/i915/gt/intel_gt_regs.h       |   8 +
 drivers/gpu/drm/i915/gt/intel_ppgtt.c         |   2 +-
 drivers/gpu/drm/i915/gt/intel_tlb.c           |  68 +++-
 drivers/gpu/drm/i915/gt/intel_tlb.h           |   1 +
 drivers/gpu/drm/i915/gt/selftest_tlb.c        |  99 +++++-
 .../gpu/drm/i915/gt/uc/abi/guc_actions_abi.h  |  35 +++
 drivers/gpu/drm/i915/gt/uc/intel_guc.h        |  34 ++
 drivers/gpu/drm/i915/gt/uc/intel_guc_ct.c     |   4 +
 drivers/gpu/drm/i915/gt/uc/intel_guc_ct.h     |  13 +
 drivers/gpu/drm/i915/gt/uc/intel_guc_fwif.h   |   1 +
 .../gpu/drm/i915/gt/uc/intel_guc_submission.c | 290 +++++++++++++++++-
 drivers/gpu/drm/i915/gt/uc/intel_uc.c         |   7 +
 drivers/gpu/drm/i915/i915_drv.h               |   2 +
 drivers/gpu/drm/i915/i915_pci.c               |   1 +
 drivers/gpu/drm/i915/i915_vma.c               |  14 +-
 drivers/gpu/drm/i915/i915_vma.h               |   3 +-
 drivers/gpu/drm/i915/intel_device_info.h      |   1 +
 .../drm/i915/selftests/i915_mock_selftests.h  |   1 +
 19 files changed, 597 insertions(+), 21 deletions(-)
Signature · 1 line
2.25.1
#1 lore raw

PATCH 1/2 drm/i915: Add generic interface for tlb invalidation

From: Prathap Kumar Valsan <[email protected]>

This supports selective and full tlb invalidations. When GuC is enabled the tlb invalidations use guc ct otherwise use mmio interface.

---
 drivers/gpu/drm/i915/gt/intel_gt_regs.h       |  8 ++
 drivers/gpu/drm/i915/gt/intel_tlb.c           | 58 +++++++++++-
 drivers/gpu/drm/i915/gt/intel_tlb.h           |  1 +
 drivers/gpu/drm/i915/gt/selftest_tlb.c        | 92 +++++++++++++++++++
 .../drm/i915/selftests/i915_mock_selftests.h  |  1 +
 5 files changed, 159 insertions(+), 1 deletion(-)
drivers/gpu/drm/i915/gt/intel_gt_regs.h +8 -0
index eecd0a87a6478..f2ca1c26ecde5 100644
@@ -1124,6 +1124,14 @@
 
 #define GEN12_GAM_DONE				_MMIO(0xcf68)
 
+#define XEHPSDV_TLB_INV_DESC0			_MMIO(0xcf7c)
+#define   XEHPSDV_TLB_INV_DESC0_ADDR_LO		REG_GENMASK(31, 12)
+#define   XEHPSDV_TLB_INV_DESC0_ADDR_MASK	REG_GENMASK(8, 3)
+#define   XEHPSDV_TLB_INV_DESC0_G		REG_GENMASK(2, 1)
+#define   XEHPSDV_TLB_INV_DESC0_VALID		REG_BIT(0)
+#define XEHPSDV_TLB_INV_DESC1			_MMIO(0xcf80)
+#define   XEHPSDV_TLB_INV_DESC0_ADDR_HI		REG_GENMASK(31, 0)
+
 #define GEN7_HALF_SLICE_CHICKEN1		_MMIO(0xe100) /* IVB GT1 + VLV */
 #define GEN8_HALF_SLICE_CHICKEN1		MCR_REG(0xe100)
 #define   GEN7_MAX_PS_THREAD_DEP		(8 << 12)
drivers/gpu/drm/i915/gt/intel_tlb.c +57 -1

80 lines not on this page · raw message

drivers/gpu/drm/i915/gt/intel_tlb.h +1 -0
index 337327af92ac4..9e5fc40c2b08e 100644
@@ -12,6 +12,7 @@
 #include "intel_gt_types.h"
 
 void intel_gt_invalidate_tlb_full(struct intel_gt *gt, u32 seqno);
+bool intel_gt_invalidate_tlb_range(struct intel_gt *gt, u64 start, u64 length);
 
 void intel_gt_init_tlb(struct intel_gt *gt);
 void intel_gt_fini_tlb(struct intel_gt *gt);
drivers/gpu/drm/i915/gt/selftest_tlb.c +92 -0

112 lines not on this page · raw message

drivers/gpu/drm/i915/selftests/i915_mock_selftests.h +1 -0
index 0c22e0fc9059c..3e00cd2b6e53c 100644
@@ -21,6 +21,7 @@ selftest(fence, i915_sw_fence_mock_selftests)
 selftest(scatterlist, scatterlist_mock_selftests)
 selftest(syncmap, i915_syncmap_mock_selftests)
 selftest(uncore, intel_uncore_mock_selftests)
+selftest(tlb, intel_tlb_mock_selftests)
 selftest(ring, intel_ring_mock_selftests)
 selftest(engine, intel_engine_cs_mock_selftests)
 selftest(timelines, intel_timeline_mock_selftests)
Signature · 1 line
2.25.1
#2 lore raw

Re: [Intel-gfx] [PATCH dii-client 1/2] drm/i915: Add generic interface for tlb invalidation

kernel test robot noticed the following build errors:

Hi Jonathan,

kernel test robot noticed the following build errors:

[auto build test ERROR on drm-tip/drm-tip]

url:    https://github.com/intel-lab-lkp/linux/commits/Jonathan-Cavitt/drm-i915-Use-selective-tlb-invalidations-where-supported/20231011-034501
base:   git://anongit.freedesktop.org/drm/drm-tip drm-tip
patch link:    https://lore.kernel.org/r/20231010184423.2118908-3-jonathan.cavitt%40intel.com
patch subject: [Intel-gfx] [PATCH dii-client 1/2] drm/i915: Add generic interface for tlb invalidation
config: i386-buildonly-randconfig-002-20231011 (https://download.01.org/0day-ci/archive/20231011/[email protected]/config)
compiler: gcc-12 (Debian 12.2.0-14) 12.2.0
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20231011/[email protected]/reproduce)

If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <[email protected]>
| Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/

All errors (new ones prefixed by >>):

drivers/gpu/drm/i915/gt/intel_tlb.c: In function 'intel_gt_invalidate_tlb_full':

>> drivers/gpu/drm/i915/gt/intel_tlb.c:141:21: error: implicit declaration of function 'intel_guc_invalidate_tlb_full'; did you mean 'intel_gt_invalidate_tlb_full'? [-Werror=implicit-function-declaration]
     141 |                 if (intel_guc_invalidate_tlb_full(guc, INTEL_GUC_TLB_INVAL_MODE_HEAVY) < 0)
         |                     ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~
         |                     intel_gt_invalidate_tlb_full
>> drivers/gpu/drm/i915/gt/intel_tlb.c:141:56: error: 'INTEL_GUC_TLB_INVAL_MODE_HEAVY' undeclared (first use in this function)
     141 |                 if (intel_guc_invalidate_tlb_full(guc, INTEL_GUC_TLB_INVAL_MODE_HEAVY) < 0)
         |                                                        ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
   drivers/gpu/drm/i915/gt/intel_tlb.c:141:56: note: each undeclared identifier is reported only once for each function it appears in
   In file included from include/linux/bits.h:6,
                    from include/linux/ratelimit_types.h:5,
                    from include/linux/printk.h:9,
                    from include/asm-generic/bug.h:22,
                    from arch/x86/include/asm/bug.h:87,
                    from include/linux/plist.h:80,
                    from include/linux/pm_qos.h:15,
                    from drivers/gpu/drm/i915/i915_drv.h:35,
                    from drivers/gpu/drm/i915/gt/intel_tlb.c:6:
   drivers/gpu/drm/i915/gt/intel_tlb.c: In function 'intel_gt_invalidate_tlb_range':
>> drivers/gpu/drm/i915/gt/intel_tlb.c:190:48: error: 'const struct intel_device_info' has no member named 'ppgtt_size'
     190 |         vm_total = BIT_ULL(INTEL_INFO(gt->i915)->ppgtt_size);
         |                                                ^~
   include/vdso/bits.h:8:45: note: in definition of macro 'BIT_ULL'
       8 | #define BIT_ULL(nr)             (ULL(1) << (nr))
         |                                             ^~
>> drivers/gpu/drm/i915/gt/intel_tlb.c:195:23: error: implicit declaration of function 'intel_guc_invalidate_tlb_page_selective' [-Werror=implicit-function-declaration]
     195 |                 ret = intel_guc_invalidate_tlb_page_selective(guc,
         |                       ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
   drivers/gpu/drm/i915/gt/intel_tlb.c:196:63: error: 'INTEL_GUC_TLB_INVAL_MODE_HEAVY' undeclared (first use in this function)
     196 |                                                               INTEL_GUC_TLB_INVAL_MODE_HEAVY,
         |                                                               ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

cc1: some warnings being treated as errors

vim +141 drivers/gpu/drm/i915/gt/intel_tlb.c

120
121 void intel_gt_invalidate_tlb_full(struct intel_gt *gt, u32 seqno)
122 {
123 intel_wakeref_t wakeref;
124
125 if (I915_SELFTEST_ONLY(gt->awake == -ENODEV))
126 return;
127
128 if (intel_gt_is_wedged(gt))
129 return;
130
131 if (tlb_seqno_passed(gt, seqno))
132 return;
133
134 with_intel_gt_pm_if_awake(gt, wakeref) {
135 struct intel_guc *guc = &gt->uc.guc;
136
137 mutex_lock(&gt->tlb.invalidate_lock);
138 if (tlb_seqno_passed(gt, seqno))
139 goto unlock;
140

 > 141			if (intel_guc_invalidate_tlb_full(guc, INTEL_GUC_TLB_INVAL_MODE_HEAVY) < 0)
   142				mmio_invalidate_full(gt);
   143	
   144			write_seqcount_invalidate(&gt->tlb.seqno);
   145	unlock:
   146			mutex_unlock(&gt->tlb.invalidate_lock);
   147		}
   148	}
   149	
   150	static u64 tlb_page_selective_size(u64 *addr, u64 length)
   151	{
   152		const u64 end = *addr + length;
   153		u64 start;
   154	
   155		/*
   156		 * Minimum invalidation size for a 2MB page that the hardware expects is
   157		 * 16MB
   158		 */
   159		length = max_t(u64, roundup_pow_of_two(length), SZ_4K);
   160		if (length >= SZ_2M)
   161			length = max_t(u64, SZ_16M, length);
   162	
   163		/*
   164		 * We need to invalidate a higher granularity if start address is not
   165		 * aligned to length. When start is not aligned with length we need to
   166		 * find the length large enough to create an address mask covering the
   167		 * required range.
   168		 */
   169		start = round_down(*addr, length);
   170		while (start + length < end) {
   171			length <<= 1;
   172			start = round_down(*addr, length);
   173		}
   174	
   175		*addr = start;
   176		return length;
   177	}
   178	
   179	bool intel_gt_invalidate_tlb_range(struct intel_gt *gt,
   180					   u64 start, u64 length)
   181	{
   182		struct intel_guc *guc = &gt->uc.guc;
   183		intel_wakeref_t wakeref;
   184		u64 size, vm_total;
   185		bool ret = true;
   186	
   187		if (intel_gt_is_wedged(gt))
   188			return true;
   189	
 > 190		vm_total = BIT_ULL(INTEL_INFO(gt->i915)->ppgtt_size);
   191		/* Align start and length */
   192		size =  min_t(u64, vm_total, tlb_page_selective_size(&start, length));
   193	
   194		with_intel_gt_pm_if_awake(gt, wakeref)
 > 195			ret = intel_guc_invalidate_tlb_page_selective(guc,

196 INTEL_GUC_TLB_INVAL_MODE_HEAVY,
197 start, size) == 0;
198
199 return ret;
200 }
201

Signature · 2 lines
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki
#3 lore raw

Re: [Intel-gfx] [PATCH dii-client 1/2] drm/i915: Add generic interface for tlb invalidation

In reply to Jonathan Cavitt
kernel test robot noticed the following build errors:

Hi Jonathan,

kernel test robot noticed the following build errors:

[auto build test ERROR on drm-tip/drm-tip]

url:    https://github.com/intel-lab-lkp/linux/commits/Jonathan-Cavitt/drm-i915-Use-selective-tlb-invalidations-where-supported/20231011-034501
base:   git://anongit.freedesktop.org/drm/drm-tip drm-tip
patch link:    https://lore.kernel.org/r/20231010184423.2118908-3-jonathan.cavitt%40intel.com
patch subject: [Intel-gfx] [PATCH dii-client 1/2] drm/i915: Add generic interface for tlb invalidation
config: x86_64-randconfig-001-20231011 (https://download.01.org/0day-ci/archive/20231011/[email protected]/config)
compiler: gcc-9 (Debian 9.3.0-22) 9.3.0
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20231011/[email protected]/reproduce)

If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <[email protected]>
| Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/

All error/warnings (new ones prefixed by >>):

drivers/gpu/drm/i915/gt/intel_tlb.c: In function 'intel_gt_invalidate_tlb_full':
drivers/gpu/drm/i915/gt/intel_tlb.c:141:7: error: implicit declaration of function 'intel_guc_invalidate_tlb_full'; did you mean 'intel_gt_invalidate_tlb_full'? [-Werror=implicit-function-declaration]

     141 |   if (intel_guc_invalidate_tlb_full(guc, INTEL_GUC_TLB_INVAL_MODE_HEAVY) < 0)
         |       ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~
         |       intel_gt_invalidate_tlb_full
   drivers/gpu/drm/i915/gt/intel_tlb.c:141:42: error: 'INTEL_GUC_TLB_INVAL_MODE_HEAVY' undeclared (first use in this function)
     141 |   if (intel_guc_invalidate_tlb_full(guc, INTEL_GUC_TLB_INVAL_MODE_HEAVY) < 0)
         |                                          ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
   drivers/gpu/drm/i915/gt/intel_tlb.c:141:42: note: each undeclared identifier is reported only once for each function it appears in
   In file included from include/linux/bits.h:6,
                    from include/linux/ratelimit_types.h:5,
                    from include/linux/printk.h:9,
                    from include/asm-generic/bug.h:22,
                    from arch/x86/include/asm/bug.h:87,
                    from include/linux/plist.h:80,
                    from include/linux/pm_qos.h:15,
                    from drivers/gpu/drm/i915/i915_drv.h:35,
                    from drivers/gpu/drm/i915/gt/intel_tlb.c:6:
   drivers/gpu/drm/i915/gt/intel_tlb.c: In function 'intel_gt_invalidate_tlb_range':
   drivers/gpu/drm/i915/gt/intel_tlb.c:190:41: error: 'const struct intel_device_info' has no member named 'ppgtt_size'
     190 |  vm_total = BIT_ULL(INTEL_INFO(gt->i915)->ppgtt_size);
         |                                         ^~
   include/vdso/bits.h:8:34: note: in definition of macro 'BIT_ULL'
       8 | #define BIT_ULL(nr)  (ULL(1) << (nr))
         |                                  ^~
   drivers/gpu/drm/i915/gt/intel_tlb.c:195:9: error: implicit declaration of function 'intel_guc_invalidate_tlb_page_selective' [-Werror=implicit-function-declaration]
     195 |   ret = intel_guc_invalidate_tlb_page_selective(guc,
         |         ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
   drivers/gpu/drm/i915/gt/intel_tlb.c:196:14: error: 'INTEL_GUC_TLB_INVAL_MODE_HEAVY' undeclared (first use in this function)
     196 |              INTEL_GUC_TLB_INVAL_MODE_HEAVY,
         |              ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
   In file included from drivers/gpu/drm/i915/gt/intel_tlb.c:214:
   drivers/gpu/drm/i915/gt/selftest_tlb.c: In function 'pte_tlbinv':
>> drivers/gpu/drm/i915/gt/selftest_tlb.c:161:1: error: version control conflict marker in file
     161 | <<<<<<< HEAD
         | ^~~~~~~
   drivers/gpu/drm/i915/gt/selftest_tlb.c:163:1: error: version control conflict marker in file
     163 | =======
         | ^~~~~~~
   drivers/gpu/drm/i915/gt/selftest_tlb.c:165:1: error: version control conflict marker in file
     165 | >>>>>>> 774058193c61b... INTEL_DII: drm/i915/xehpsdv: Add generic interface for tlb invalidation
         | ^~~~~~~
>> drivers/gpu/drm/i915/gt/selftest_tlb.c:165:9: error: invalid suffix "c61b..." on integer constant
     165 | >>>>>>> 774058193c61b... INTEL_DII: drm/i915/xehpsdv: Add generic interface for tlb invalidation
         |         ^~~~~~~~~~~~~~~~
>> drivers/gpu/drm/i915/gt/selftest_tlb.c:150:28: warning: unused variable 'vb_res' [-Wunused-variable]
     150 |   struct i915_vma_resource vb_res = {
         |                            ^~~~~~
>> drivers/gpu/drm/i915/gt/selftest_tlb.c:40:21: warning: unused variable 'pat_index' [-Wunused-variable]

40 | const unsigned int pat_index =
| ^~~~~~~~~
cc1: some warnings being treated as errors

vim +161 drivers/gpu/drm/i915/gt/selftest_tlb.c

    30	
    31	static int
    32	pte_tlbinv(struct intel_context *ce,
    33		   struct i915_vma *va,
    34		   struct i915_vma *vb,
    35		   u64 align,
    36		   void (*tlbinv)(struct i915_address_space *vm, u64 addr, u64 length),
    37		   u64 length,
    38		   struct rnd_state *prng)
    39	{
  > 40		const unsigned int pat_index =
    41			i915_gem_get_pat_index(ce->vm->i915, I915_CACHE_NONE);
    42		struct drm_i915_gem_object *batch;
    43		struct drm_mm_node vb_node;
    44		struct i915_request *rq;
    45		struct i915_vma *vma;
    46		u64 addr;
    47		int err;
    48		u32 *cs;
    49	
    50		batch = i915_gem_object_create_internal(ce->vm->i915, 4096);
    51		if (IS_ERR(batch))
    52			return PTR_ERR(batch);
    53	
    54		vma = i915_vma_instance(batch, ce->vm, NULL);
    55		if (IS_ERR(vma)) {
    56			err = PTR_ERR(vma);
    57			goto out;
    58		}
    59	
    60		err = i915_vma_pin(vma, 0, 0, PIN_USER);
    61		if (err)
    62			goto out;
    63	
    64		/* Pin va at random but aligned offset after vma */
    65		addr = round_up(vma->node.start + vma->node.size, align);
    66		/* MI_CONDITIONAL_BATCH_BUFFER_END limits address to 48b */
    67		addr = igt_random_offset(prng, addr, min(ce->vm->total, BIT_ULL(48)),
    68					 va->size, align);
    69		err = i915_vma_pin(va,  0, 0, addr | PIN_OFFSET_FIXED | PIN_USER);
    70		if (err) {
    71			pr_err("Cannot pin at %llx+%llx\n", addr, va->size);
    72			goto out;
    73		}
    74		GEM_BUG_ON(i915_vma_offset(va) != addr);
    75		if (vb != va) {
    76			vb_node = vb->node;
    77			vb->node = va->node; /* overwrites the _same_ PTE  */
    78		}
    79	
    80		/*
    81		 * Now choose random dword at the 1st pinned page.
    82		 *
    83		 * SZ_64K pages on dg1 require that the whole PT be marked
    84		 * containing 64KiB entries. So we make sure that vma
    85		 * covers the whole PT, despite being randomly aligned to 64KiB
    86		 * and restrict our sampling to the 2MiB PT within where
    87		 * we know that we will be using 64KiB pages.
    88		 */
    89		if (align == SZ_64K)
    90			addr = round_up(addr, SZ_2M);
    91		addr = igt_random_offset(prng, addr, addr + align, 8, 8);
    92	
    93		if (va != vb)
    94			pr_info("%s(%s): Sampling %llx, with alignment %llx, using PTE size %x (phys %x, sg %x), invalidate:%llx+%llx\n",
    95				ce->engine->name, va->obj->mm.region->name ?: "smem",
    96				addr, align, va->resource->page_sizes_gtt,
    97				va->page_sizes.phys, va->page_sizes.sg,
    98				addr & -length, length);
    99	
   100		cs = i915_gem_object_pin_map_unlocked(batch, I915_MAP_WC);
   101		*cs++ = MI_NOOP; /* for later termination */
   102		/*
   103		 * Sample the target to see if we spot the updated backing store.
   104		 * Gen8 VCS compares immediate value with bitwise-and of two
   105		 * consecutive DWORDS pointed by addr, other gen/engines compare value
   106		 * with DWORD pointed by addr. Moreover we want to exercise DWORD size
   107		 * invalidations. To fulfill all these requirements below values
   108		 * have been chosen.
   109		 */
   110		*cs++ = MI_CONDITIONAL_BATCH_BUFFER_END | MI_DO_COMPARE | 2;
   111		*cs++ = 0; /* break if *addr == 0 */
   112		*cs++ = lower_32_bits(addr);
   113		*cs++ = upper_32_bits(addr);
   114		vma_set_qw(va, addr, -1);
   115		vma_set_qw(vb, addr, 0);
   116	
   117		/* Keep sampling until we get bored */
   118		*cs++ = MI_BATCH_BUFFER_START | BIT(8) | 1;
   119		*cs++ = lower_32_bits(i915_vma_offset(vma));
   120		*cs++ = upper_32_bits(i915_vma_offset(vma));
   121	
   122		i915_gem_object_flush_map(batch);
   123	
   124		rq = i915_request_create(ce);
   125		if (IS_ERR(rq)) {
   126			err = PTR_ERR(rq);
   127			goto out_va;
   128		}
   129	
   130		err = rq->engine->emit_bb_start(rq, i915_vma_offset(vma), 0, 0);
   131		if (err) {
   132			i915_request_add(rq);
   133			goto out_va;
   134		}
   135	
   136		i915_request_get(rq);
   137		i915_request_add(rq);
   138	
   139		/* Short sleep to sanitycheck the batch is spinning before we begin */
   140		msleep(10);
   141		if (va == vb) {
   142			if (!i915_request_completed(rq)) {
   143				pr_err("%s(%s): Semaphore sanitycheck failed %llx, with alignment %llx, using PTE size %x (phys %x, sg %x)\n",
   144				       ce->engine->name, va->obj->mm.region->name ?: "smem",
   145				       addr, align, va->resource->page_sizes_gtt,
   146				       va->page_sizes.phys, va->page_sizes.sg);
   147				err = -EIO;
   148			}
   149		} else if (!i915_request_completed(rq)) {
 > 150			struct i915_vma_resource vb_res = {
   151				.bi.pages = vb->obj->mm.pages,
   152				.bi.page_sizes = vb->obj->mm.page_sizes,
   153				.start = i915_vma_offset(vb),
   154				.vma_size = i915_vma_size(vb)
   155			};
   156			unsigned int pte_flags = 0;
   157	
   158			/* Flip the PTE between A and B */
   159			if (i915_gem_object_is_lmem(vb->obj))
   160				pte_flags |= PTE_LM;
 > 161	<<<<<<< HEAD
   162			ce->vm->insert_entries(ce->vm, &vb_res, pat_index, pte_flags);
   163	=======
   164			ce->vm->insert_entries(ce->vm, &stash, vb, I915_CACHE_NONE, pte_flags);
 > 165	>>>>>>> 774058193c61b... INTEL_DII: drm/i915/xehpsdv: Add generic interface for tlb invalidation

166
167 /* Flush the PTE update to concurrent HW */
168 tlbinv(ce->vm, addr & -length, length);
169
170 if (wait_for(i915_request_completed(rq), HZ / 2)) {
171 pr_err("%s: Request did not complete; the COND_BBE did not read the updated PTE\n",
172 ce->engine->name);
173 err = -EINVAL;
174 }
175 } else {
176 pr_err("Spinner ended unexpectedly\n");
177 err = -EIO;
178 }
179 i915_request_put(rq);
180
181 cs = page_mask_bits(batch->mm.mapping);
182 *cs = MI_BATCH_BUFFER_END;
183 wmb();
184
185 out_va:
186 if (vb != va)
187 vb->node = vb_node;
188 i915_vma_unpin(va);
189 if (i915_vma_unbind_unlocked(va))
190 err = -EIO;
191 out:
192 i915_gem_object_put(batch);
193 return err;
194 }
195

Signature · 2 lines
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki
#4 lore raw

RFC PATCH 1/10 drm/i915: Add GuC TLB Invalidation device info flags

In reply to Jonathan Cavitt

Add device info flags for if GuC TLB Invalidation is enabled.

---
 drivers/gpu/drm/i915/i915_drv.h          | 2 ++
 drivers/gpu/drm/i915/intel_device_info.h | 1 +
 2 files changed, 3 insertions(+)
drivers/gpu/drm/i915/i915_drv.h +2 -0
index cb60fc9cf8737..6a2a78c61f212 100644
@@ -794,6 +794,8 @@ IS_SUBPLATFORM(const struct drm_i915_private *i915,
 #define HAS_GUC_DEPRIVILEGE(i915) \
 	(INTEL_INFO(i915)->has_guc_deprivilege)
 
+#define HAS_GUC_TLB_INVALIDATION(i915)	(INTEL_INFO(i915)->has_guc_tlb_invalidation)
+
 #define HAS_3D_PIPELINE(i915)	(INTEL_INFO(i915)->has_3d_pipeline)
 
 #define HAS_ONE_EU_PER_FUSE_BIT(i915)	(INTEL_INFO(i915)->has_one_eu_per_fuse_bit)
drivers/gpu/drm/i915/intel_device_info.h +1 -0
index 39817490b13fd..eba2f0b919c87 100644
@@ -153,6 +153,7 @@ enum intel_ppgtt_type {
 	func(has_heci_pxp); \
 	func(has_heci_gscfi); \
 	func(has_guc_deprivilege); \
+	func(has_guc_tlb_invalidation); \
 	func(has_l3_ccs_read); \
 	func(has_l3_dpf); \
 	func(has_llc); \
Signature · 1 line
2.25.1
#5 lore raw

PATCH 2/2 drm/i915: Use selective tlb invalidations where supported

In reply to Jonathan Cavitt

For platforms supporting selective tlb invalidations, we don't need to do a full tlb invalidation. Rather do a range based tlb invalidation for every unbind of purged vma belongs to an active vm.

---
 drivers/gpu/drm/i915/gt/intel_ppgtt.c |  2 +-
 drivers/gpu/drm/i915/i915_vma.c       | 14 +++++++++-----
 drivers/gpu/drm/i915/i915_vma.h       |  3 ++-
 3 files changed, 12 insertions(+), 7 deletions(-)
drivers/gpu/drm/i915/gt/intel_ppgtt.c +1 -1
index d07a4f97b9434..b43dae3cbd59f 100644
@@ -211,7 +211,7 @@ void ppgtt_unbind_vma(struct i915_address_space *vm,
 		return;
 
 	vm->clear_range(vm, vma_res->start, vma_res->vma_size);
-	vma_invalidate_tlb(vm, vma_res->tlb);
+	vma_invalidate_tlb(vm, vma_res->tlb, vma_res->start, vma_res->vma_size);
 }
 
 static unsigned long pd_count(u64 size, int shift)
drivers/gpu/drm/i915/i915_vma.c +9 -5
index d09aad34ba37f..cb05d794f0d0f 100644
@@ -1339,7 +1339,8 @@ I915_SELFTEST_EXPORT int i915_vma_get_pages(struct i915_vma *vma)
 	return err;
 }
 
-void vma_invalidate_tlb(struct i915_address_space *vm, u32 *tlb)
+void vma_invalidate_tlb(struct i915_address_space *vm, u32 *tlb
+			u64 start, u64 size)
 {
 	struct intel_gt *gt;
 	int id;
@@ -1355,9 +1356,11 @@ void vma_invalidate_tlb(struct i915_address_space *vm, u32 *tlb)
 	 * the most recent TLB invalidation seqno, and if we have not yet
 	 * flushed the TLBs upon release, perform a full invalidation.
 	 */
-	for_each_gt(gt, vm->i915, id)
-		WRITE_ONCE(tlb[id],
-			   intel_gt_next_invalidate_tlb_full(gt));
+	for_each_gt(gt, vm->i915, id) {
+		if (!intel_gt_invalidate_tlb_range(gt, start, size))
+			WRITE_ONCE(tlb[id],
+				   intel_gt_next_invalidate_tlb_full(gt));
+	}
 }
 
 static void __vma_put_pages(struct i915_vma *vma, unsigned int count)
@@ -2041,7 +2044,8 @@ struct dma_fence *__i915_vma_evict(struct i915_vma *vma, bool async)
 			dma_fence_put(unbind_fence);
 			unbind_fence = NULL;
 		}
-		vma_invalidate_tlb(vma->vm, vma->obj->mm.tlb);
+		vma_invalidate_tlb(vma->vm, vma->obj->mm.tlb,
+				   vma->node.start, vma->size);
 	}
 
 	/*
drivers/gpu/drm/i915/i915_vma.h +2 -1
index e356dfb883d34..5a604aad55dfe 100644
@@ -260,7 +260,8 @@ bool i915_vma_misplaced(const struct i915_vma *vma,
 			u64 size, u64 alignment, u64 flags);
 void __i915_vma_set_map_and_fenceable(struct i915_vma *vma);
 void i915_vma_revoke_mmap(struct i915_vma *vma);
-void vma_invalidate_tlb(struct i915_address_space *vm, u32 *tlb);
+void vma_invalidate_tlb(struct i915_address_space *vm, u32 *tlb,
+			u64 start, u64 size);
 struct dma_fence *__i915_vma_evict(struct i915_vma *vma, bool async);
 int __i915_vma_unbind(struct i915_vma *vma);
 int __must_check i915_vma_unbind(struct i915_vma *vma);
Signature · 1 line
2.25.1
#6 lore raw

Re: [Intel-gfx] [PATCH dii-client 2/2] drm/i915: Use selective tlb invalidations where supported

Ignore this. It's not a security hole: it's just a temporary patch I was using for rebasing purposes that got smuggled into this series on accident. It has a bad tag because of some stale gitconfig params that I've since removed.
-Jonathan Cavitt

-----Original Message-----
From: Cavitt, Jonathan <[email protected]>
Sent: Tuesday, October 10, 2023 11:44 AM
To: [email protected]
Cc: Gupta, saurabhg <[email protected]>; Cavitt, Jonathan <[email protected]>; Das, Nirmoy <[email protected]>; Shyti, Andi <[email protected]>; [email protected]; Harrison, John C <[email protected]>
Subject: [PATCH dii-client 2/2] drm/i915: Use selective tlb invalidations where supported

Quoted 86 lines

For platforms supporting selective tlb invalidations, we don't need to do a full tlb invalidation. Rather do a range based tlb invalidation for every unbind of purged vma belongs to an active vm.

---
 drivers/gpu/drm/i915/gt/intel_ppgtt.c |  2 +-
 drivers/gpu/drm/i915/i915_vma.c       | 14 +++++++++-----
 drivers/gpu/drm/i915/i915_vma.h       |  3 ++-
 3 files changed, 12 insertions(+), 7 deletions(-)
drivers/gpu/drm/i915/gt/intel_ppgtt.c +1 -1
index d07a4f97b9434..b43dae3cbd59f 100644
@@ -211,7 +211,7 @@ void ppgtt_unbind_vma(struct i915_address_space *vm,
 		return;
 
 	vm->clear_range(vm, vma_res->start, vma_res->vma_size);
-	vma_invalidate_tlb(vm, vma_res->tlb);
+	vma_invalidate_tlb(vm, vma_res->tlb, vma_res->start, vma_res->vma_size);
 }
 
 static unsigned long pd_count(u64 size, int shift)
drivers/gpu/drm/i915/i915_vma.c +9 -5
index d09aad34ba37f..cb05d794f0d0f 100644
@@ -1339,7 +1339,8 @@ I915_SELFTEST_EXPORT int i915_vma_get_pages(struct i915_vma *vma)
 	return err;
 }
 
-void vma_invalidate_tlb(struct i915_address_space *vm, u32 *tlb)
+void vma_invalidate_tlb(struct i915_address_space *vm, u32 *tlb
+			u64 start, u64 size)
 {
 	struct intel_gt *gt;
 	int id;
@@ -1355,9 +1356,11 @@ void vma_invalidate_tlb(struct i915_address_space *vm, u32 *tlb)
 	 * the most recent TLB invalidation seqno, and if we have not yet
 	 * flushed the TLBs upon release, perform a full invalidation.
 	 */
-	for_each_gt(gt, vm->i915, id)
-		WRITE_ONCE(tlb[id],
-			   intel_gt_next_invalidate_tlb_full(gt));
+	for_each_gt(gt, vm->i915, id) {
+		if (!intel_gt_invalidate_tlb_range(gt, start, size))
+			WRITE_ONCE(tlb[id],
+				   intel_gt_next_invalidate_tlb_full(gt));
+	}
 }
 
 static void __vma_put_pages(struct i915_vma *vma, unsigned int count)
@@ -2041,7 +2044,8 @@ struct dma_fence *__i915_vma_evict(struct i915_vma *vma, bool async)
 			dma_fence_put(unbind_fence);
 			unbind_fence = NULL;
 		}
-		vma_invalidate_tlb(vma->vm, vma->obj->mm.tlb);
+		vma_invalidate_tlb(vma->vm, vma->obj->mm.tlb,
+				   vma->node.start, vma->size);
 	}
 
 	/*
drivers/gpu/drm/i915/i915_vma.h +2 -1
index e356dfb883d34..5a604aad55dfe 100644
@@ -260,7 +260,8 @@ bool i915_vma_misplaced(const struct i915_vma *vma,
 			u64 size, u64 alignment, u64 flags);
 void __i915_vma_set_map_and_fenceable(struct i915_vma *vma);
 void i915_vma_revoke_mmap(struct i915_vma *vma);
-void vma_invalidate_tlb(struct i915_address_space *vm, u32 *tlb);
+void vma_invalidate_tlb(struct i915_address_space *vm, u32 *tlb,
+			u64 start, u64 size);
 struct dma_fence *__i915_vma_evict(struct i915_vma *vma, bool async);
 int __i915_vma_unbind(struct i915_vma *vma);
 int __must_check i915_vma_unbind(struct i915_vma *vma);
Signature · 1 line
2.25.1
#7 lore raw

Re: [Intel-gfx] [PATCH dii-client 2/2] drm/i915: Use selective tlb invalidations where supported

In reply to Jonathan Cavitt
kernel test robot noticed the following build errors:

Hi Jonathan,

kernel test robot noticed the following build errors:

[auto build test ERROR on drm-tip/drm-tip]

url:    https://github.com/intel-lab-lkp/linux/commits/Jonathan-Cavitt/drm-i915-Use-selective-tlb-invalidations-where-supported/20231011-034501
base:   git://anongit.freedesktop.org/drm/drm-tip drm-tip
patch link:    https://lore.kernel.org/r/20231010184423.2118908-4-jonathan.cavitt%40intel.com
patch subject: [Intel-gfx] [PATCH dii-client 2/2] drm/i915: Use selective tlb invalidations where supported
config: x86_64-rhel-8.3-rust (https://download.01.org/0day-ci/archive/20231012/[email protected]/config)
compiler: clang version 16.0.4 (https://github.com/llvm/llvm-project.git ae42196bc493ffe877a7e3dff8be32035dea4d07)
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20231012/[email protected]/reproduce)

If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <[email protected]>
| Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/

All errors (new ones prefixed by >>):

>> drivers/gpu/drm/i915/i915_vma.c:1343:4: error: expected ')'
                           u64 start, u64 size)
                           ^
   drivers/gpu/drm/i915/i915_vma.c:1342:24: note: to match this '('
   void vma_invalidate_tlb(struct i915_address_space *vm, u32 *tlb
                          ^
>> drivers/gpu/drm/i915/i915_vma.c:1342:6: error: conflicting types for 'vma_invalidate_tlb'
   void vma_invalidate_tlb(struct i915_address_space *vm, u32 *tlb
        ^
   drivers/gpu/drm/i915/i915_vma.h:263:6: note: previous declaration is here
   void vma_invalidate_tlb(struct i915_address_space *vm, u32 *tlb,
        ^
>> drivers/gpu/drm/i915/i915_vma.c:1360:42: error: use of undeclared identifier 'start'; did you mean 'stac'?
                   if (!intel_gt_invalidate_tlb_range(gt, start, size))
                                                          ^~~~~
                                                          stac
   arch/x86/include/asm/smap.h:36:29: note: 'stac' declared here
   static __always_inline void stac(void)
                               ^
>> drivers/gpu/drm/i915/i915_vma.c:1360:49: error: use of undeclared identifier 'size'; did you mean 'ksize'?
                   if (!intel_gt_invalidate_tlb_range(gt, start, size))
                                                                 ^~~~
                                                                 ksize
   include/linux/slab.h:245:8: note: 'ksize' declared here
   size_t ksize(const void *objp);
          ^

4 errors generated.

vim +1343 drivers/gpu/drm/i915/i915_vma.c

1341

> 1342	void vma_invalidate_tlb(struct i915_address_space *vm, u32 *tlb
> 1343				u64 start, u64 size)
  1344	{
  1345		struct intel_gt *gt;
  1346		int id;
  1347	
  1348		if (!tlb)
  1349			return;
  1350	
  1351		/*
  1352		 * Before we release the pages that were bound by this vma, we
  1353		 * must invalidate all the TLBs that may still have a reference
  1354		 * back to our physical address. It only needs to be done once,
  1355		 * so after updating the PTE to point away from the pages, record
  1356		 * the most recent TLB invalidation seqno, and if we have not yet
  1357		 * flushed the TLBs upon release, perform a full invalidation.
  1358		 */
  1359		for_each_gt(gt, vm->i915, id) {
> 1360			if (!intel_gt_invalidate_tlb_range(gt, start, size))

1361 WRITE_ONCE(tlb[id],
1362 intel_gt_next_invalidate_tlb_full(gt));
1363 }
1364 }
1365

Signature · 2 lines
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki
#8 lore raw

Re: [Intel-gfx] [PATCH dii-client 2/2] drm/i915: Use selective tlb invalidations where supported

In reply to Jonathan Cavitt
kernel test robot noticed the following build errors:

Hi Jonathan,

kernel test robot noticed the following build errors:

[auto build test ERROR on drm-tip/drm-tip]

url:    https://github.com/intel-lab-lkp/linux/commits/Jonathan-Cavitt/drm-i915-Use-selective-tlb-invalidations-where-supported/20231011-034501
base:   git://anongit.freedesktop.org/drm/drm-tip drm-tip
patch link:    https://lore.kernel.org/r/20231010184423.2118908-4-jonathan.cavitt%40intel.com
patch subject: [Intel-gfx] [PATCH dii-client 2/2] drm/i915: Use selective tlb invalidations where supported
config: x86_64-allyesconfig (https://download.01.org/0day-ci/archive/20231021/[email protected]/config)
compiler: gcc-12 (Debian 12.2.0-14) 12.2.0
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20231021/[email protected]/reproduce)

If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <[email protected]>
| Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/

All errors (new ones prefixed by >>):

>> drivers/gpu/drm/i915/i915_vma.c:1343:25: error: expected ';', ',' or ')' before 'u64'
    1343 |                         u64 start, u64 size)
         |                         ^~~

vim +1343 drivers/gpu/drm/i915/i915_vma.c

1341
1342 void vma_invalidate_tlb(struct i915_address_space *vm, u32 *tlb

> 1343				u64 start, u64 size)

1344 {
1345 struct intel_gt *gt;
1346 int id;
1347
1348 if (!tlb)
1349 return;
1350
1351 /*
1352 * Before we release the pages that were bound by this vma, we
1353 * must invalidate all the TLBs that may still have a reference
1354 * back to our physical address. It only needs to be done once,
1355 * so after updating the PTE to point away from the pages, record
1356 * the most recent TLB invalidation seqno, and if we have not yet
1357 * flushed the TLBs upon release, perform a full invalidation.
1358 */
1359 for_each_gt(gt, vm->i915, id) {
1360 if (!intel_gt_invalidate_tlb_range(gt, start, size))
1361 WRITE_ONCE(tlb[id],
1362 intel_gt_next_invalidate_tlb_full(gt));
1363 }
1364 }
1365

Signature · 2 lines
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki
#9 lore raw

RFC PATCH 2/10 drm/i915/guc: Add CT size delay helper

In reply to Jonathan Cavitt

Add a helper function to the GuC CT buffer that reports the expected time to process all outstanding requests. As of now, there is no functionality to check number of requests in the buffer, so the helper function just reports 2 seconds, or 1ms per request up to the maximum number of requests the CT buffer can store.

---
 drivers/gpu/drm/i915/gt/uc/intel_guc_ct.h | 13 +++++++++++++
 1 file changed, 13 insertions(+)
drivers/gpu/drm/i915/gt/uc/intel_guc_ct.h +13 -0
index 58e42901ff498..36afc1ce9fabd 100644
@@ -120,6 +120,19 @@ static inline bool intel_guc_ct_enabled(struct intel_guc_ct *ct)
 	return ct->enabled;
 }
 
+/*
+ * GuC has a timeout of 1ms for a TLB invalidation response from GAM.  On a
+ * timeout GuC drops the request and has no mechanism to notify the host about
+ * the timeout.  There is also no mechanism for determining the number of
+ * outstanding requests in the CT buffer.  Ergo, keep a larger timeout that accounts
+ * for this individual timeout and the max number of outstanding requests that
+ * can be queued in CT buffer.
+ */
+static inline long intel_guc_ct_expected_delay(struct intel_guc_ct *ct)
+{
+	return HZ * 2;
+}
+
 #define INTEL_GUC_CT_SEND_NB		BIT(31)
 #define INTEL_GUC_CT_SEND_G2H_DW_SHIFT	0
 #define INTEL_GUC_CT_SEND_G2H_DW_MASK	(0xff << INTEL_GUC_CT_SEND_G2H_DW_SHIFT)
Signature · 1 line
2.25.1
#10 lore raw

RFC PATCH 3/10 drm/i915: Define and use GuC and CTB TLB invalidation routines

In reply to Jonathan Cavitt

From: Prathap Kumar Valsan <[email protected]>

The GuC firmware had defined the interface for Translation Look-Aside Buffer (TLB) invalidation. We should use this interface when invalidating the engine and GuC TLBs.
Add additional functionality to intel_gt_invalidate_tlb, invalidating the GuC TLBs and falling back to GT invalidation when the GuC is disabled.
The invalidation is done by sending a request directly to the GuC tlb_lookup that invalidates the table. The invalidation is submitted as a wait request and is performed in the CT event handler. This means we cannot perform this TLB invalidation path if the CT is not enabled. If the request isn't fulfilled in two seconds, this would constitute an error in the invalidation as that would constitute either a lost request or a severe GuC overload.

With this new invalidation routine, we can perform GuC-based GGTT invalidations. GuC-based GGTT invalidation is incompatible with MMIO invalidation so we should not perform MMIO invalidation when GuC-based GGTT invalidation is expected.

The additional complexity incurred in this patch will be necessary for range-based tlb invalidations, which will be platformed in the future.

---
 drivers/gpu/drm/i915/gt/intel_ggtt.c          |  34 +++-
 drivers/gpu/drm/i915/gt/intel_tlb.c           |  16 +-
 .../gpu/drm/i915/gt/uc/abi/guc_actions_abi.h  |  33 ++++
 drivers/gpu/drm/i915/gt/uc/intel_guc.h        |  22 +++
 drivers/gpu/drm/i915/gt/uc/intel_guc_ct.c     |   4 +
 drivers/gpu/drm/i915/gt/uc/intel_guc_fwif.h   |   1 +
 .../gpu/drm/i915/gt/uc/intel_guc_submission.c | 186 +++++++++++++++++-
 7 files changed, 284 insertions(+), 12 deletions(-)
drivers/gpu/drm/i915/gt/intel_ggtt.c +25 -9
index 4d7d88b92632b..a1f7bdc602996 100644
@@ -206,22 +206,38 @@ static void gen8_ggtt_invalidate(struct i915_ggtt *ggtt)
 	intel_uncore_write_fw(uncore, GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
 }
 
+static void guc_ggtt_ct_invalidate(struct intel_gt *gt)
+{
+	struct intel_uncore *uncore = gt->uncore;
+	intel_wakeref_t wakeref;
+
+	with_intel_runtime_pm_if_active(uncore->rpm, wakeref) {
+		struct intel_guc *guc = &gt->uc.guc;
+
+		intel_guc_invalidate_tlb_guc(guc);
+	}
+}
+
 static void guc_ggtt_invalidate(struct i915_ggtt *ggtt)
 {
 	struct drm_i915_private *i915 = ggtt->vm.i915;
+	struct intel_gt *gt;
 
-	gen8_ggtt_invalidate(ggtt);
-
-	if (GRAPHICS_VER(i915) >= 12) {
-		struct intel_gt *gt;
+	if (!HAS_GUC_TLB_INVALIDATION(i915))
+		gen8_ggtt_invalidate(ggtt);
 
-		list_for_each_entry(gt, &ggtt->gt_list, ggtt_link)
+	list_for_each_entry(gt, &ggtt->gt_list, ggtt_link) {
+		if (HAS_GUC_TLB_INVALIDATION(i915) &&
+		    intel_guc_is_ready(&gt->uc.guc)) {
+			guc_ggtt_ct_invalidate(gt);
+		} else if (GRAPHICS_VER(i915) >= 12) {
 			intel_uncore_write_fw(gt->uncore,
 					      GEN12_GUC_TLB_INV_CR,
 					      GEN12_GUC_TLB_INV_CR_INVALIDATE);
-	} else {
-		intel_uncore_write_fw(ggtt->vm.gt->uncore,
-				      GEN8_GTCR, GEN8_GTCR_INVALIDATE);
+		} else {
+			intel_uncore_write_fw(gt->uncore,
+					      GEN8_GTCR, GEN8_GTCR_INVALIDATE);
+		}
 	}
 }
 
@@ -1243,7 +1259,7 @@ static int gen8_gmch_probe(struct i915_ggtt *ggtt)
 		ggtt->vm.raw_insert_page = gen8_ggtt_insert_page;
 	}
 
-	if (intel_uc_wants_guc(&ggtt->vm.gt->uc))
+	if (intel_uc_wants_guc_submission(&ggtt->vm.gt->uc))
 		ggtt->invalidate = guc_ggtt_invalidate;
 	else
 		ggtt->invalidate = gen8_ggtt_invalidate;
drivers/gpu/drm/i915/gt/intel_tlb.c +15 -1
index 139608c30d978..4bb13d1890e37 100644
@@ -12,6 +12,7 @@
 #include "intel_gt_print.h"
 #include "intel_gt_regs.h"
 #include "intel_tlb.h"
+#include "uc/intel_guc.h"
 
 /*
  * HW architecture suggest typical invalidation time at 40us,
@@ -131,11 +132,24 @@ void intel_gt_invalidate_tlb_full(struct intel_gt *gt, u32 seqno)
 		return;
 
 	with_intel_gt_pm_if_awake(gt, wakeref) {
+		struct intel_guc *guc = &gt->uc.guc;
+
 		mutex_lock(&gt->tlb.invalidate_lock);
 		if (tlb_seqno_passed(gt, seqno))
 			goto unlock;
 
-		mmio_invalidate_full(gt);
+		if (HAS_GUC_TLB_INVALIDATION(gt->i915)) {
+			/*
+			 * Only perform GuC TLB invalidation if GuC is ready.
+			 * The only time GuC could not be ready is on GT reset,
+			 * which would clobber all the TLBs anyways, making
+			 * any TLB invalidation path here unnecessary.
+			 */
+			if (intel_guc_is_ready(guc))
+				intel_guc_invalidate_tlb_engines(guc);
+		} else {
+			mmio_invalidate_full(gt);
+		}
 
 		write_seqcount_invalidate(&gt->tlb.seqno);
 unlock:
drivers/gpu/drm/i915/gt/uc/abi/guc_actions_abi.h +33 -0
index f359bef046e0b..33f253410d0c8 100644
@@ -138,6 +138,8 @@ enum intel_guc_action {
 	INTEL_GUC_ACTION_REGISTER_CONTEXT_MULTI_LRC = 0x4601,
 	INTEL_GUC_ACTION_CLIENT_SOFT_RESET = 0x5507,
 	INTEL_GUC_ACTION_SET_ENG_UTIL_BUFF = 0x550A,
+	INTEL_GUC_ACTION_TLB_INVALIDATION = 0x7000,
+	INTEL_GUC_ACTION_TLB_INVALIDATION_DONE = 0x7001,
 	INTEL_GUC_ACTION_STATE_CAPTURE_NOTIFICATION = 0x8002,
 	INTEL_GUC_ACTION_NOTIFY_FLUSH_LOG_BUFFER_TO_FILE = 0x8003,
 	INTEL_GUC_ACTION_NOTIFY_CRASH_DUMP_POSTED = 0x8004,
@@ -181,4 +183,35 @@ enum intel_guc_state_capture_event_status {
 
 #define INTEL_GUC_STATE_CAPTURE_EVENT_STATUS_MASK      0x000000FF
 
+#define INTEL_GUC_TLB_INVAL_TYPE_MASK	REG_GENMASK(7, 0)
+#define INTEL_GUC_TLB_INVAL_MODE_MASK	REG_GENMASK(11, 8)
+#define INTEL_GUC_TLB_INVAL_FLUSH_CACHE REG_BIT(31)
+
+enum intel_guc_tlb_invalidation_type {
+	INTEL_GUC_TLB_INVAL_ENGINES = 0x0,
+	INTEL_GUC_TLB_INVAL_GUC = 0x3,
+};
+
+/*
+ * 0: Heavy mode of Invalidation:
+ * The pipeline of the engine(s) for which the invalidation is targeted to is
+ * blocked, and all the in-flight transactions are guaranteed to be Globally
+ * Observed before completing the TLB invalidation
+ * 1: Lite mode of Invalidation:
+ * TLBs of the targeted engine(s) are immediately invalidated.
+ * In-flight transactions are NOT guaranteed to be Globally Observed before
+ * completing TLB invalidation.
+ * Light Invalidation Mode is to be used only when
+ * it can be guaranteed (by SW) that the address translations remain invariant
+ * for the in-flight transactions across the TLB invalidation. In other words,
+ * this mode can be used when the TLB invalidation is intended to clear out the
+ * stale cached translations that are no longer in use. Light Invalidation Mode
+ * is much faster than the Heavy Invalidation Mode, as it does not wait for the
+ * in-flight transactions to be GOd.
+ */
+enum intel_guc_tlb_inval_mode {
+	INTEL_GUC_TLB_INVAL_MODE_HEAVY = 0x0,
+	INTEL_GUC_TLB_INVAL_MODE_LITE = 0x1,
+};
+
 #endif /* _ABI_GUC_ACTIONS_ABI_H */
drivers/gpu/drm/i915/gt/uc/intel_guc.h +22 -0
index 818c8c146fd47..f5ede14b18aae 100644
@@ -79,6 +79,18 @@ struct intel_guc {
 	 */
 	atomic_t outstanding_submission_g2h;
 
+	/** @tlb_lookup: xarray to store all pending TLB invalidation requests */
+	struct xarray tlb_lookup;
+
+	/**
+	 * @serial_slot: id to the initial waiter created in tlb_lookup,
+	 * which is used only when failed to allocate new waiter.
+	 */
+	u32 serial_slot;
+
+	/** @next_seqno: the next id (sequence no.) to allocate. */
+	u32 next_seqno;
+
 	/** @interrupts: pointers to GuC interrupt-managing functions. */
 	struct {
 		bool enabled;
@@ -297,6 +309,11 @@ struct intel_guc {
 #define GUC_SUBMIT_VER(guc)		MAKE_GUC_VER_STRUCT((guc)->submission_version)
 #define GUC_FIRMWARE_VER(guc)		MAKE_GUC_VER_STRUCT((guc)->fw.file_selected.ver)
 
+struct intel_guc_tlb_wait {
+	struct wait_queue_head wq;
+	bool busy;
+};
+
 static inline struct intel_guc *log_to_guc(struct intel_guc_log *log)
 {
 	return container_of(log, struct intel_guc, log);
@@ -419,6 +436,11 @@ static inline bool intel_guc_is_supported(struct intel_guc *guc)
 	return intel_uc_fw_is_supported(&guc->fw);
 }
 
+int intel_guc_invalidate_tlb_engines(struct intel_guc *guc);
+int intel_guc_invalidate_tlb_guc(struct intel_guc *guc);
+int intel_guc_tlb_invalidation_done(struct intel_guc *guc,
+				    const u32 *payload, u32 len);
+
 static inline bool intel_guc_is_wanted(struct intel_guc *guc)
 {
 	return intel_uc_fw_is_enabled(&guc->fw);
drivers/gpu/drm/i915/gt/uc/intel_guc_ct.c +4 -0
index c33210ead1ef7..8114b12ac91e3 100644
@@ -1115,6 +1115,9 @@ static int ct_process_request(struct intel_guc_ct *ct, struct ct_incoming_msg *r
 	case INTEL_GUC_ACTION_NOTIFY_EXCEPTION:
 		ret = intel_guc_crash_process_msg(guc, action);
 		break;
+	case INTEL_GUC_ACTION_TLB_INVALIDATION_DONE:
+		ret = intel_guc_tlb_invalidation_done(guc, payload, len);
+		break;
 	default:
 		ret = -EOPNOTSUPP;
 		break;
@@ -1186,6 +1189,7 @@ static int ct_handle_event(struct intel_guc_ct *ct, struct ct_incoming_msg *requ
 	switch (action) {
 	case INTEL_GUC_ACTION_SCHED_CONTEXT_MODE_DONE:
 	case INTEL_GUC_ACTION_DEREGISTER_CONTEXT_DONE:
+	case INTEL_GUC_ACTION_TLB_INVALIDATION_DONE:
 		g2h_release_space(ct, request->size);
 	}
 
drivers/gpu/drm/i915/gt/uc/intel_guc_fwif.h +1 -0
index 123ad75d2eb28..8ae1846431da7 100644
@@ -22,6 +22,7 @@
 /* Payload length only i.e. don't include G2H header length */
 #define G2H_LEN_DW_SCHED_CONTEXT_MODE_SET	2
 #define G2H_LEN_DW_DEREGISTER_CONTEXT		1
+#define G2H_LEN_DW_INVALIDATE_TLB		1
 
 #define GUC_CONTEXT_DISABLE		0
 #define GUC_CONTEXT_ENABLE		1
drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c +184 -2

237 lines not on this page · raw message

Signature · 1 line
2.25.1
#11 lore raw

RFC PATCH 4/10 drm/i915: No TLB invalidation on suspended GT

In reply to Jonathan Cavitt

In case of GT is suspended, don't allow submission of new TLB invalidation request and cancel all pending requests. The TLB entries will be invalidated either during GuC reload or on system resume.

---
 drivers/gpu/drm/i915/gt/uc/intel_guc.h        |  1 +
 .../gpu/drm/i915/gt/uc/intel_guc_submission.c | 21 +++++++++++++------
 drivers/gpu/drm/i915/gt/uc/intel_uc.c         |  7 +++++++
 3 files changed, 23 insertions(+), 6 deletions(-)
drivers/gpu/drm/i915/gt/uc/intel_guc.h +1 -0
index f5ede14b18aae..3fbf4b33ce139 100644
@@ -537,4 +537,5 @@ void intel_guc_dump_time_info(struct intel_guc *guc, struct drm_printer *p);
 
 int intel_guc_sched_disable_gucid_threshold_max(struct intel_guc *guc);
 
+void wake_up_all_tlb_invalidate(struct intel_guc *guc);
 #endif
drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c +15 -6
index e9854652c2b52..b9c168ea57270 100644
@@ -1796,13 +1796,25 @@ static void __guc_reset_context(struct intel_context *ce, intel_engine_mask_t st
 	intel_context_put(parent);
 }
 
-void intel_guc_submission_reset(struct intel_guc *guc, intel_engine_mask_t stalled)
+void wake_up_all_tlb_invalidate(struct intel_guc *guc)
 {
 	struct intel_guc_tlb_wait *wait;
+	unsigned long i;
+
+	if (!HAS_GUC_TLB_INVALIDATION(guc_to_gt(guc)->i915))
+		return;
+
+	xa_lock_irq(&guc->tlb_lookup);
+	xa_for_each(&guc->tlb_lookup, i, wait)
+		wake_up(&wait->wq);
+	xa_unlock_irq(&guc->tlb_lookup);
+}
+
+void intel_guc_submission_reset(struct intel_guc *guc, intel_engine_mask_t stalled)
+{
 	struct intel_context *ce;
 	unsigned long index;
 	unsigned long flags;
-	unsigned long i;
 
 	if (unlikely(!guc_submission_initialized(guc))) {
 		/* Reset called during driver load? GuC not yet initialised! */
@@ -1833,10 +1845,7 @@ void intel_guc_submission_reset(struct intel_guc *guc, intel_engine_mask_t stall
 	 * The full GT reset will have cleared the TLB caches and flushed the
 	 * G2H message queue; we can release all the blocked waiters.
 	 */
-	xa_lock_irq(&guc->tlb_lookup);
-	xa_for_each(&guc->tlb_lookup, i, wait)
-		wake_up(&wait->wq);
-	xa_unlock_irq(&guc->tlb_lookup);
+	wake_up_all_tlb_invalidate(guc);
 }
 
 static void guc_cancel_context_requests(struct intel_context *ce)
drivers/gpu/drm/i915/gt/uc/intel_uc.c +7 -0
index 98b103375b7ab..750cb63503dd7 100644
@@ -688,6 +688,8 @@ void intel_uc_suspend(struct intel_uc *uc)
 	/* flush the GSC worker */
 	intel_gsc_uc_flush_work(&uc->gsc);
 
+	wake_up_all_tlb_invalidate(guc);
+
 	if (!intel_guc_is_ready(guc)) {
 		guc->interrupts.enabled = false;
 		return;
@@ -736,6 +738,11 @@ static int __uc_resume(struct intel_uc *uc, bool enable_communication)
 
 	intel_gsc_uc_resume(&uc->gsc);
 
+	if (HAS_GUC_TLB_INVALIDATION(gt->i915)) {
+		intel_guc_invalidate_tlb_engines(guc);
+		intel_guc_invalidate_tlb_guc(guc);
+	}
+
 	return 0;
 }
 
Signature · 1 line
2.25.1
#12 lore raw

RFC PATCH 5/10 drm/i915: No TLB invalidation on wedged GT

In reply to Jonathan Cavitt

It is not an error for GuC TLB invalidations to fail when the GT is wedged or disabled, so do not process a wait failure as one in guc_send_invalidate_tlb.

---
 .../gpu/drm/i915/gt/uc/intel_guc_submission.c  | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)
drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c +17 -1
index b9c168ea57270..c3c45d3b9e89b 100644
@@ -32,6 +32,7 @@
 
 #include "i915_drv.h"
 #include "i915_reg.h"
+#include "i915_irq.h"
 #include "i915_trace.h"
 
 /**
@@ -1941,6 +1942,12 @@ void intel_guc_submission_cancel_requests(struct intel_guc *guc)
 
 	/* GuC is blown away, drop all references to contexts */
 	xa_destroy(&guc->context_lookup);
+
+	/*
+	 * Wedged GT won't respond to any TLB invalidation request. Simply
+	 * release all the blocked waiters.
+	 */
+	wake_up_all_tlb_invalidate(guc);
 }
 
 void intel_guc_submission_reset_finish(struct intel_guc *guc)
@@ -4738,6 +4745,14 @@ static long must_wait_woken(struct wait_queue_entry *wq_entry, long timeout)
 	return timeout;
 }
 
+static bool intel_gt_is_enabled(const struct intel_gt *gt)
+{
+	/* Check if GT is wedged or suspended */
+	if (intel_gt_is_wedged(gt) || !intel_irqs_enabled(gt->i915))
+		return false;
+	return true;
+}
+
 static int guc_send_invalidate_tlb(struct intel_guc *guc,
 				   enum intel_guc_tlb_invalidation_type type)
 {
@@ -4790,7 +4805,8 @@ static int guc_send_invalidate_tlb(struct intel_guc *guc,
 	if (err)
 		goto out;
 
-	if (!must_wait_woken(&wait, intel_guc_ct_expected_delay(&guc->ct))) {
+	if (intel_gt_is_enabled(guc_to_gt(guc)) &&
+	    !must_wait_woken(&wait, intel_guc_ct_expected_delay(&guc->ct))) {
 		guc_err(guc,
 			"TLB invalidation response timed out for seqno %u\n", seqno);
 		err = -ETIME;
Signature · 1 line
2.25.1
#13 lore raw

RFC PATCH 6/10 drm/i915/gt: Increase sleep in gt_tlb selftest sanitycheck

In reply to Jonathan Cavitt

For the gt_tlb live selftest, when operating on the GSC engine, increase the timeout from 10 ms to 200 ms because the GSC engine is a bit slower than the rest.

Additionally, increase the default timeout from 10 ms to 20 ms because msleep < 20ms can sleep for up to 20ms.

---
 drivers/gpu/drm/i915/gt/selftest_tlb.c | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)
drivers/gpu/drm/i915/gt/selftest_tlb.c +9 -2
index 7e41f69fc818f..24beb94aa7a37 100644
@@ -136,8 +136,15 @@ pte_tlbinv(struct intel_context *ce,
 	i915_request_get(rq);
 	i915_request_add(rq);
 
-	/* Short sleep to sanitycheck the batch is spinning before we begin */
-	msleep(10);
+	/*
+	 * Short sleep to sanitycheck the batch is spinning before we begin.
+	 * FIXME: Why is GSC so slow?
+	 */
+	if (ce->engine->class == OTHER_CLASS)
+		msleep(200);
+	else
+		msleep(20);
+
 	if (va == vb) {
 		if (!i915_request_completed(rq)) {
 			pr_err("%s(%s): Semaphore sanitycheck failed %llx, with alignment %llx, using PTE size %x (phys %x, sg %x)\n",
Signature · 1 line
2.25.1
#14 lore raw

RFC PATCH 7/10 drm/i915: Enable GuC TLB invalidations for MTL

In reply to Jonathan Cavitt

Enable GuC TLB invalidations for MTL. Though more platforms than just MTL support GuC TLB invalidations, MTL is presently the only platform that requires it for any purpose, so only enable it there for now to minimize cross-platform impact.

---
 drivers/gpu/drm/i915/i915_pci.c | 1 +
 1 file changed, 1 insertion(+)
drivers/gpu/drm/i915/i915_pci.c +1 -0
index df7c261410f79..d4b51ececbb12 100644
@@ -829,6 +829,7 @@ static const struct intel_device_info mtl_info = {
 	.has_flat_ccs = 0,
 	.has_gmd_id = 1,
 	.has_guc_deprivilege = 1,
+	.has_guc_tlb_invalidation = 1,
 	.has_llc = 0,
 	.has_mslice_steering = 0,
 	.has_snoop = 1,
Signature · 1 line
2.25.1
#15 lore raw

RFC PATCH 8/10 drm/i915: Define GuC Based TLB invalidation routines

In reply to Jonathan Cavitt

From: Prathap Kumar Valsan <[email protected]>

The GuC firmware has defined the interface for selective TLB invalidation support. This patch adds routines to interface with GuC.

---
 .../gpu/drm/i915/gt/uc/abi/guc_actions_abi.h  |   2 +
 drivers/gpu/drm/i915/gt/uc/intel_guc.h        |  11 ++
 .../gpu/drm/i915/gt/uc/intel_guc_submission.c | 105 +++++++++++++++---
 3 files changed, 105 insertions(+), 13 deletions(-)
drivers/gpu/drm/i915/gt/uc/abi/guc_actions_abi.h +2 -0
index 33f253410d0c8..7bb710fcd9087 100644
@@ -189,6 +189,8 @@ enum intel_guc_state_capture_event_status {
 
 enum intel_guc_tlb_invalidation_type {
 	INTEL_GUC_TLB_INVAL_ENGINES = 0x0,
+	INTEL_GUC_TLB_INVAL_PAGE_SELECTIVE = 0x1,
+	INTEL_GUC_TLB_INVAL_PAGE_SELECTIVE_CTX = 0x2,
 	INTEL_GUC_TLB_INVAL_GUC = 0x3,
 };
 
drivers/gpu/drm/i915/gt/uc/intel_guc.h +11 -0
index 3fbf4b33ce139..369fd2be1c725 100644
@@ -431,6 +431,17 @@ int intel_guc_allocate_and_map_vma(struct intel_guc *guc, u32 size,
 int intel_guc_self_cfg32(struct intel_guc *guc, u16 key, u32 value);
 int intel_guc_self_cfg64(struct intel_guc *guc, u16 key, u64 value);
 
+int intel_guc_g2g_register(struct intel_guc *guc);
+
+int intel_guc_invalidate_tlb_full(struct intel_guc *guc);
+int intel_guc_invalidate_tlb_page_selective(struct intel_guc *guc,
+					    enum intel_guc_tlb_inval_mode mode,
+					    u64 start, u64 length);
+int intel_guc_invalidate_tlb_page_selective_ctx(struct intel_guc *guc,
+						enum intel_guc_tlb_inval_mode mode,
+						u64 start, u64 length, u32 ctxid);
+int intel_guc_invalidate_tlb_guc(struct intel_guc *guc);
+
 static inline bool intel_guc_is_supported(struct intel_guc *guc)
 {
 	return intel_uc_fw_is_supported(&guc->fw);
drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c +92 -13

127 lines not on this page · raw message

Signature · 1 line
2.25.1
#16 lore raw

RFC PATCH 9/10 drm/i915: Add generic interface for tlb invalidation

In reply to Jonathan Cavitt

From: Prathap Kumar Valsan <[email protected]>

This supports selective and full tlb invalidations. When GuC is enabled the tlb invalidations use guc ct otherwise use mmio interface.

---
 drivers/gpu/drm/i915/gt/intel_gt_regs.h       |  8 ++
 drivers/gpu/drm/i915/gt/intel_tlb.c           | 52 +++++++++++
 drivers/gpu/drm/i915/gt/intel_tlb.h           |  1 +
 drivers/gpu/drm/i915/gt/selftest_tlb.c        | 88 +++++++++++++++++++
 .../drm/i915/selftests/i915_mock_selftests.h  |  1 +
 5 files changed, 150 insertions(+)
drivers/gpu/drm/i915/gt/intel_gt_regs.h +8 -0
index eecd0a87a6478..f2ca1c26ecde5 100644
@@ -1124,6 +1124,14 @@
 
 #define GEN12_GAM_DONE				_MMIO(0xcf68)
 
+#define XEHPSDV_TLB_INV_DESC0			_MMIO(0xcf7c)
+#define   XEHPSDV_TLB_INV_DESC0_ADDR_LO		REG_GENMASK(31, 12)
+#define   XEHPSDV_TLB_INV_DESC0_ADDR_MASK	REG_GENMASK(8, 3)
+#define   XEHPSDV_TLB_INV_DESC0_G		REG_GENMASK(2, 1)
+#define   XEHPSDV_TLB_INV_DESC0_VALID		REG_BIT(0)
+#define XEHPSDV_TLB_INV_DESC1			_MMIO(0xcf80)
+#define   XEHPSDV_TLB_INV_DESC0_ADDR_HI		REG_GENMASK(31, 0)
+
 #define GEN7_HALF_SLICE_CHICKEN1		_MMIO(0xe100) /* IVB GT1 + VLV */
 #define GEN8_HALF_SLICE_CHICKEN1		MCR_REG(0xe100)
 #define   GEN7_MAX_PS_THREAD_DEP		(8 << 12)
drivers/gpu/drm/i915/gt/intel_tlb.c +52 -0

58 lines not on this page · raw message

drivers/gpu/drm/i915/gt/intel_tlb.h +1 -0
index 337327af92ac4..9e5fc40c2b08e 100644
@@ -12,6 +12,7 @@
 #include "intel_gt_types.h"
 
 void intel_gt_invalidate_tlb_full(struct intel_gt *gt, u32 seqno);
+bool intel_gt_invalidate_tlb_range(struct intel_gt *gt, u64 start, u64 length);
 
 void intel_gt_init_tlb(struct intel_gt *gt);
 void intel_gt_fini_tlb(struct intel_gt *gt);
drivers/gpu/drm/i915/gt/selftest_tlb.c +88 -0

101 lines not on this page · raw message

drivers/gpu/drm/i915/selftests/i915_mock_selftests.h +1 -0
index 0c22e0fc9059c..3e00cd2b6e53c 100644
@@ -21,6 +21,7 @@ selftest(fence, i915_sw_fence_mock_selftests)
 selftest(scatterlist, scatterlist_mock_selftests)
 selftest(syncmap, i915_syncmap_mock_selftests)
 selftest(uncore, intel_uncore_mock_selftests)
+selftest(tlb, intel_tlb_mock_selftests)
 selftest(ring, intel_ring_mock_selftests)
 selftest(engine, intel_engine_cs_mock_selftests)
 selftest(timelines, intel_timeline_mock_selftests)
Signature · 1 line
2.25.1
#17 lore raw

RFC PATCH 10/10 drm/i915: Use selective tlb invalidations where supported

In reply to Jonathan Cavitt

For platforms supporting selective tlb invalidations, we don't need to do a full tlb invalidation. Rather do a range based tlb invalidation for every unbind of purged vma belongs to an active vm.

---
 drivers/gpu/drm/i915/gt/intel_ppgtt.c |  2 +-
 drivers/gpu/drm/i915/i915_vma.c       | 14 +++++++++-----
 drivers/gpu/drm/i915/i915_vma.h       |  3 ++-
 3 files changed, 12 insertions(+), 7 deletions(-)
drivers/gpu/drm/i915/gt/intel_ppgtt.c +1 -1
index d07a4f97b9434..b43dae3cbd59f 100644
@@ -211,7 +211,7 @@ void ppgtt_unbind_vma(struct i915_address_space *vm,
 		return;
 
 	vm->clear_range(vm, vma_res->start, vma_res->vma_size);
-	vma_invalidate_tlb(vm, vma_res->tlb);
+	vma_invalidate_tlb(vm, vma_res->tlb, vma_res->start, vma_res->vma_size);
 }
 
 static unsigned long pd_count(u64 size, int shift)
drivers/gpu/drm/i915/i915_vma.c +9 -5
index d09aad34ba37f..cb05d794f0d0f 100644
@@ -1339,7 +1339,8 @@ I915_SELFTEST_EXPORT int i915_vma_get_pages(struct i915_vma *vma)
 	return err;
 }
 
-void vma_invalidate_tlb(struct i915_address_space *vm, u32 *tlb)
+void vma_invalidate_tlb(struct i915_address_space *vm, u32 *tlb
+			u64 start, u64 size)
 {
 	struct intel_gt *gt;
 	int id;
@@ -1355,9 +1356,11 @@ void vma_invalidate_tlb(struct i915_address_space *vm, u32 *tlb)
 	 * the most recent TLB invalidation seqno, and if we have not yet
 	 * flushed the TLBs upon release, perform a full invalidation.
 	 */
-	for_each_gt(gt, vm->i915, id)
-		WRITE_ONCE(tlb[id],
-			   intel_gt_next_invalidate_tlb_full(gt));
+	for_each_gt(gt, vm->i915, id) {
+		if (!intel_gt_invalidate_tlb_range(gt, start, size))
+			WRITE_ONCE(tlb[id],
+				   intel_gt_next_invalidate_tlb_full(gt));
+	}
 }
 
 static void __vma_put_pages(struct i915_vma *vma, unsigned int count)
@@ -2041,7 +2044,8 @@ struct dma_fence *__i915_vma_evict(struct i915_vma *vma, bool async)
 			dma_fence_put(unbind_fence);
 			unbind_fence = NULL;
 		}
-		vma_invalidate_tlb(vma->vm, vma->obj->mm.tlb);
+		vma_invalidate_tlb(vma->vm, vma->obj->mm.tlb,
+				   vma->node.start, vma->size);
 	}
 
 	/*
drivers/gpu/drm/i915/i915_vma.h +2 -1
index e356dfb883d34..5a604aad55dfe 100644
@@ -260,7 +260,8 @@ bool i915_vma_misplaced(const struct i915_vma *vma,
 			u64 size, u64 alignment, u64 flags);
 void __i915_vma_set_map_and_fenceable(struct i915_vma *vma);
 void i915_vma_revoke_mmap(struct i915_vma *vma);
-void vma_invalidate_tlb(struct i915_address_space *vm, u32 *tlb);
+void vma_invalidate_tlb(struct i915_address_space *vm, u32 *tlb,
+			u64 start, u64 size);
 struct dma_fence *__i915_vma_evict(struct i915_vma *vma, bool async);
 int __i915_vma_unbind(struct i915_vma *vma);
 int __must_check i915_vma_unbind(struct i915_vma *vma);
Signature · 1 line
2.25.1
#18 lore raw

Re: [Intel-gfx] [RFC PATCH 10/10] drm/i915: Use selective tlb invalidations where supported

Quoted 58 lines · Jonathan Cavitt

For platforms supporting selective tlb invalidations, we don't need to do a full tlb invalidation. Rather do a range based tlb invalidation for every unbind of purged vma belongs to an active vm.

---
  drivers/gpu/drm/i915/gt/intel_ppgtt.c |  2 +-
  drivers/gpu/drm/i915/i915_vma.c       | 14 +++++++++-----
  drivers/gpu/drm/i915/i915_vma.h       |  3 ++-
  3 files changed, 12 insertions(+), 7 deletions(-)
drivers/gpu/drm/i915/gt/intel_ppgtt.c +1 -1
index d07a4f97b9434..b43dae3cbd59f 100644
@@ -211,7 +211,7 @@ void ppgtt_unbind_vma(struct i915_address_space *vm,
  		return;
 
  	vm->clear_range(vm, vma_res->start, vma_res->vma_size);
-	vma_invalidate_tlb(vm, vma_res->tlb);
+	vma_invalidate_tlb(vm, vma_res->tlb, vma_res->start, vma_res->vma_size);
  }
 
  static unsigned long pd_count(u64 size, int shift)
drivers/gpu/drm/i915/i915_vma.c +7 -4
index d09aad34ba37f..cb05d794f0d0f 100644
@@ -1339,7 +1339,8 @@ I915_SELFTEST_EXPORT int i915_vma_get_pages(struct i915_vma *vma)
  	return err;
  }
 
-void vma_invalidate_tlb(struct i915_address_space *vm, u32 *tlb)
+void vma_invalidate_tlb(struct i915_address_space *vm, u32 *tlb
+			u64 start, u64 size)
  {
  	struct intel_gt *gt;
  	int id;
@@ -1355,9 +1356,11 @@ void vma_invalidate_tlb(struct i915_address_space *vm, u32 *tlb)
  	 * the most recent TLB invalidation seqno, and if we have not yet
  	 * flushed the TLBs upon release, perform a full invalidation.
  	 */
-	for_each_gt(gt, vm->i915, id)
-		WRITE_ONCE(tlb[id],
-			   intel_gt_next_invalidate_tlb_full(gt));
+	for_each_gt(gt, vm->i915, id) {
+		if (!intel_gt_invalidate_tlb_range(gt, start, size))
+			WRITE_ONCE(tlb[id],
+				   intel_gt_next_invalidate_tlb_full(gt));
+	}
  }
 
  static void __vma_put_pages(struct i915_vma *vma, unsigned int count)

drivers/gpu/drm/i915/i915_vma.c struct dma_fence *__i915_vma_evict(struct i915_vma *vma, bool async) · line 2048

Quoted 6 lines
drivers/gpu/drm/i915/i915_vma.c +2 -1
@@ -2041,7 +2044,8 @@ struct dma_fence *__i915_vma_evict(struct i915_vma *vma, bool async)
  			dma_fence_put(unbind_fence);
  			unbind_fence = NULL;
  		}
-		vma_invalidate_tlb(vma->vm, vma->obj->mm.tlb);
+		vma_invalidate_tlb(vma->vm, vma->obj->mm.tlb,
+				   vma->node.start, vma->size);

The RFC looks like just what we needed so I'll drop an ack on the other patch series. Thanks for sending it out so quickly.

Regards,

Tvrtko

Quoted 17 lines
drivers/gpu/drm/i915/i915_vma.c +0 -0
  	}
 
  	/*
drivers/gpu/drm/i915/i915_vma.h +2 -1
index e356dfb883d34..5a604aad55dfe 100644
@@ -260,7 +260,8 @@ bool i915_vma_misplaced(const struct i915_vma *vma,
  			u64 size, u64 alignment, u64 flags);
  void __i915_vma_set_map_and_fenceable(struct i915_vma *vma);
  void i915_vma_revoke_mmap(struct i915_vma *vma);
-void vma_invalidate_tlb(struct i915_address_space *vm, u32 *tlb);
+void vma_invalidate_tlb(struct i915_address_space *vm, u32 *tlb,
+			u64 start, u64 size);
  struct dma_fence *__i915_vma_evict(struct i915_vma *vma, bool async);
  int __i915_vma_unbind(struct i915_vma *vma);
  int __must_check i915_vma_unbind(struct i915_vma *vma);