drm/i915: Implement range-based TLB
Subsystems (2)
cover drm/i915: Implement range-based TLB
Implement range-based TLB invalidations on top of GuC-based TLB invalidations. This is the future plan for GuC-based TLB invalidations because it helps improve performance over performing full tlb invalidations all the time.
Jonathan Cavitt (7):
drm/i915: Add GuC TLB Invalidation device info flags drm/i915/guc: Add CT size delay helper
drm/i915: No TLB invalidation on suspended GT drm/i915: No TLB invalidation on wedged GT drm/i915/gt: Increase sleep in gt_tlb selftest sanitycheck drm/i915: Enable GuC TLB invalidations for MTL drm/i915: Use selective tlb invalidations where supported
Prathap Kumar Valsan (3):
drm/i915: Define and use GuC and CTB TLB invalidation routines drm/i915: Define GuC Based TLB invalidation routines drm/i915: Add generic interface for tlb invalidation
drivers/gpu/drm/i915/gt/intel_ggtt.c | 34 +- drivers/gpu/drm/i915/gt/intel_gt_regs.h | 8 + drivers/gpu/drm/i915/gt/intel_ppgtt.c | 2 +- drivers/gpu/drm/i915/gt/intel_tlb.c | 68 +++- drivers/gpu/drm/i915/gt/intel_tlb.h | 1 + drivers/gpu/drm/i915/gt/selftest_tlb.c | 99 +++++- .../gpu/drm/i915/gt/uc/abi/guc_actions_abi.h | 35 +++ drivers/gpu/drm/i915/gt/uc/intel_guc.h | 34 ++ drivers/gpu/drm/i915/gt/uc/intel_guc_ct.c | 4 + drivers/gpu/drm/i915/gt/uc/intel_guc_ct.h | 13 + drivers/gpu/drm/i915/gt/uc/intel_guc_fwif.h | 1 + .../gpu/drm/i915/gt/uc/intel_guc_submission.c | 290 +++++++++++++++++- drivers/gpu/drm/i915/gt/uc/intel_uc.c | 7 + drivers/gpu/drm/i915/i915_drv.h | 2 + drivers/gpu/drm/i915/i915_pci.c | 1 + drivers/gpu/drm/i915/i915_vma.c | 14 +- drivers/gpu/drm/i915/i915_vma.h | 3 +- drivers/gpu/drm/i915/intel_device_info.h | 1 + .../drm/i915/selftests/i915_mock_selftests.h | 1 + 19 files changed, 597 insertions(+), 21 deletions(-)
Signature · 1 line
2.25.1
PATCH 1/2 drm/i915: Add generic interface for tlb invalidation
From: Prathap Kumar Valsan <[email protected]>
This supports selective and full tlb invalidations. When GuC is enabled the tlb invalidations use guc ct otherwise use mmio interface.
--- drivers/gpu/drm/i915/gt/intel_gt_regs.h | 8 ++ drivers/gpu/drm/i915/gt/intel_tlb.c | 58 +++++++++++- drivers/gpu/drm/i915/gt/intel_tlb.h | 1 + drivers/gpu/drm/i915/gt/selftest_tlb.c | 92 +++++++++++++++++++ .../drm/i915/selftests/i915_mock_selftests.h | 1 + 5 files changed, 159 insertions(+), 1 deletion(-)
drivers/gpu/drm/i915/gt/intel_gt_regs.h +8 -0
@@ -1124,6 +1124,14 @@#define GEN12_GAM_DONE _MMIO(0xcf68) +#define XEHPSDV_TLB_INV_DESC0 _MMIO(0xcf7c) +#define XEHPSDV_TLB_INV_DESC0_ADDR_LO REG_GENMASK(31, 12) +#define XEHPSDV_TLB_INV_DESC0_ADDR_MASK REG_GENMASK(8, 3) +#define XEHPSDV_TLB_INV_DESC0_G REG_GENMASK(2, 1) +#define XEHPSDV_TLB_INV_DESC0_VALID REG_BIT(0) +#define XEHPSDV_TLB_INV_DESC1 _MMIO(0xcf80) +#define XEHPSDV_TLB_INV_DESC0_ADDR_HI REG_GENMASK(31, 0) +#define GEN7_HALF_SLICE_CHICKEN1 _MMIO(0xe100)/* IVB GT1 + VLV */ #define GEN8_HALF_SLICE_CHICKEN1 MCR_REG(0xe100)#define GEN7_MAX_PS_THREAD_DEP (8 << 12)
drivers/gpu/drm/i915/gt/intel_tlb.c +57 -1
80 lines not on this page · raw message
drivers/gpu/drm/i915/gt/intel_tlb.h +1 -0
@@ -12,6 +12,7 @@#include "intel_gt_types.h" void intel_gt_invalidate_tlb_full (struct intel_gt *gt,u32 seqno); +bool intel_gt_invalidate_tlb_range (struct intel_gt *gt,u64 start,u64 length);void intel_gt_init_tlb (struct intel_gt *gt);void intel_gt_fini_tlb (struct intel_gt *gt);
drivers/gpu/drm/i915/gt/selftest_tlb.c +92 -0
112 lines not on this page · raw message
drivers/gpu/drm/i915/selftests/i915_mock_selftests.h +1 -0
@@ -21,6 +21,7 @@ selftest(fence, i915_sw_fence_mock_selftests) selftest(scatterlist, scatterlist_mock_selftests) selftest(syncmap, i915_syncmap_mock_selftests) selftest(uncore, intel_uncore_mock_selftests) +selftest(tlb, intel_tlb_mock_selftests) selftest(ring, intel_ring_mock_selftests) selftest(engine, intel_engine_cs_mock_selftests)selftest (timelines, intel_timeline_mock_selftests)
Signature · 1 line
2.25.1
kernel test robot
build botRe: [Intel-gfx] [PATCH dii-client 1/2] drm/i915: Add generic interface for tlb invalidation
kernel test robot noticed the following build errors:
Hi Jonathan,
kernel test robot noticed the following build errors:
[auto build test ERROR on drm-tip/drm-tip]
url: https://github.com/intel-lab-lkp/linux/commits/Jonathan-Cavitt/drm-i915-Use-selective-tlb-invalidations-where-supported/20231011-034501 base: git://anongit.freedesktop.org/drm/drm-tip drm-tip patch link: https://lore.kernel.org/r/20231010184423.2118908-3-jonathan.cavitt%40intel.com patch subject: [Intel-gfx] [PATCH dii-client 1/2] drm/i915: Add generic interface for tlb invalidation config: i386-buildonly-randconfig-002-20231011 (https://download.01.org/0day-ci/archive/20231011/[email protected]/config) compiler: gcc-12 (Debian 12.2.0-14) 12.2.0 reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20231011/[email protected]/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <[email protected]>
| Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/
All errors (new ones prefixed by >>):
drivers/gpu/drm/i915/gt/intel_tlb.c: In function 'intel_gt_invalidate_tlb_full':
>> drivers/gpu/drm/i915/gt/intel_tlb.c:141:21: error: implicit declaration of function 'intel_guc_invalidate_tlb_full'; did you mean 'intel_gt_invalidate_tlb_full'? [-Werror=implicit-function-declaration]
141 | if (intel_guc_invalidate_tlb_full(guc, INTEL_GUC_TLB_INVAL_MODE_HEAVY) < 0)
| ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~
| intel_gt_invalidate_tlb_full
>> drivers/gpu/drm/i915/gt/intel_tlb.c:141:56: error: 'INTEL_GUC_TLB_INVAL_MODE_HEAVY' undeclared (first use in this function)
141 | if (intel_guc_invalidate_tlb_full(guc, INTEL_GUC_TLB_INVAL_MODE_HEAVY) < 0)
| ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
drivers/gpu/drm/i915/gt/intel_tlb.c:141:56: note: each undeclared identifier is reported only once for each function it appears in
In file included from include/linux/bits.h:6,
from include/linux/ratelimit_types.h:5,
from include/linux/printk.h:9,
from include/asm-generic/bug.h:22,
from arch/x86/include/asm/bug.h:87,
from include/linux/plist.h:80,
from include/linux/pm_qos.h:15,
from drivers/gpu/drm/i915/i915_drv.h:35,
from drivers/gpu/drm/i915/gt/intel_tlb.c:6:
drivers/gpu/drm/i915/gt/intel_tlb.c: In function 'intel_gt_invalidate_tlb_range':
>> drivers/gpu/drm/i915/gt/intel_tlb.c:190:48: error: 'const struct intel_device_info' has no member named 'ppgtt_size'
190 | vm_total = BIT_ULL(INTEL_INFO(gt->i915)->ppgtt_size);
| ^~
include/vdso/bits.h:8:45: note: in definition of macro 'BIT_ULL'
8 | #define BIT_ULL(nr) (ULL(1) << (nr))
| ^~
>> drivers/gpu/drm/i915/gt/intel_tlb.c:195:23: error: implicit declaration of function 'intel_guc_invalidate_tlb_page_selective' [-Werror=implicit-function-declaration]
195 | ret = intel_guc_invalidate_tlb_page_selective(guc,
| ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
drivers/gpu/drm/i915/gt/intel_tlb.c:196:63: error: 'INTEL_GUC_TLB_INVAL_MODE_HEAVY' undeclared (first use in this function)
196 | INTEL_GUC_TLB_INVAL_MODE_HEAVY,
| ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~cc1: some warnings being treated as errors
vim +141 drivers/gpu/drm/i915/gt/intel_tlb.c
120
121 void intel_gt_invalidate_tlb_full(struct intel_gt *gt, u32 seqno)
122 {
123 intel_wakeref_t wakeref;
124
125 if (I915_SELFTEST_ONLY(gt->awake == -ENODEV))
126 return;
127
128 if (intel_gt_is_wedged(gt))
129 return;
130
131 if (tlb_seqno_passed(gt, seqno))
132 return;
133
134 with_intel_gt_pm_if_awake(gt, wakeref) {
135 struct intel_guc *guc = >->uc.guc;
136
137 mutex_lock(>->tlb.invalidate_lock);
138 if (tlb_seqno_passed(gt, seqno))
139 goto unlock;
140
> 141 if (intel_guc_invalidate_tlb_full(guc, INTEL_GUC_TLB_INVAL_MODE_HEAVY) < 0)
142 mmio_invalidate_full(gt);
143
144 write_seqcount_invalidate(>->tlb.seqno);
145 unlock:
146 mutex_unlock(>->tlb.invalidate_lock);
147 }
148 }
149
150 static u64 tlb_page_selective_size(u64 *addr, u64 length)
151 {
152 const u64 end = *addr + length;
153 u64 start;
154
155 /*
156 * Minimum invalidation size for a 2MB page that the hardware expects is
157 * 16MB
158 */
159 length = max_t(u64, roundup_pow_of_two(length), SZ_4K);
160 if (length >= SZ_2M)
161 length = max_t(u64, SZ_16M, length);
162
163 /*
164 * We need to invalidate a higher granularity if start address is not
165 * aligned to length. When start is not aligned with length we need to
166 * find the length large enough to create an address mask covering the
167 * required range.
168 */
169 start = round_down(*addr, length);
170 while (start + length < end) {
171 length <<= 1;
172 start = round_down(*addr, length);
173 }
174
175 *addr = start;
176 return length;
177 }
178
179 bool intel_gt_invalidate_tlb_range(struct intel_gt *gt,
180 u64 start, u64 length)
181 {
182 struct intel_guc *guc = >->uc.guc;
183 intel_wakeref_t wakeref;
184 u64 size, vm_total;
185 bool ret = true;
186
187 if (intel_gt_is_wedged(gt))
188 return true;
189
> 190 vm_total = BIT_ULL(INTEL_INFO(gt->i915)->ppgtt_size);
191 /* Align start and length */
192 size = min_t(u64, vm_total, tlb_page_selective_size(&start, length));
193
194 with_intel_gt_pm_if_awake(gt, wakeref)
> 195 ret = intel_guc_invalidate_tlb_page_selective(guc, 196 INTEL_GUC_TLB_INVAL_MODE_HEAVY,
197 start, size) == 0;
198
199 return ret;
200 }
201
Signature · 2 lines
0-DAY CI Kernel Test Service https://github.com/intel/lkp-tests/wiki
kernel test robot
build botRe: [Intel-gfx] [PATCH dii-client 1/2] drm/i915: Add generic interface for tlb invalidation
In reply to Jonathan Cavittkernel test robot noticed the following build errors:
Hi Jonathan,
kernel test robot noticed the following build errors:
[auto build test ERROR on drm-tip/drm-tip]
url: https://github.com/intel-lab-lkp/linux/commits/Jonathan-Cavitt/drm-i915-Use-selective-tlb-invalidations-where-supported/20231011-034501 base: git://anongit.freedesktop.org/drm/drm-tip drm-tip patch link: https://lore.kernel.org/r/20231010184423.2118908-3-jonathan.cavitt%40intel.com patch subject: [Intel-gfx] [PATCH dii-client 1/2] drm/i915: Add generic interface for tlb invalidation config: x86_64-randconfig-001-20231011 (https://download.01.org/0day-ci/archive/20231011/[email protected]/config) compiler: gcc-9 (Debian 9.3.0-22) 9.3.0 reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20231011/[email protected]/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <[email protected]>
| Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/
All error/warnings (new ones prefixed by >>):
drivers/gpu/drm/i915/gt/intel_tlb.c: In function 'intel_gt_invalidate_tlb_full':
drivers/gpu/drm/i915/gt/intel_tlb.c:141:7: error: implicit declaration of function 'intel_guc_invalidate_tlb_full'; did you mean 'intel_gt_invalidate_tlb_full'? [-Werror=implicit-function-declaration]
141 | if (intel_guc_invalidate_tlb_full(guc, INTEL_GUC_TLB_INVAL_MODE_HEAVY) < 0)
| ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~
| intel_gt_invalidate_tlb_full
drivers/gpu/drm/i915/gt/intel_tlb.c:141:42: error: 'INTEL_GUC_TLB_INVAL_MODE_HEAVY' undeclared (first use in this function)
141 | if (intel_guc_invalidate_tlb_full(guc, INTEL_GUC_TLB_INVAL_MODE_HEAVY) < 0)
| ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
drivers/gpu/drm/i915/gt/intel_tlb.c:141:42: note: each undeclared identifier is reported only once for each function it appears in
In file included from include/linux/bits.h:6,
from include/linux/ratelimit_types.h:5,
from include/linux/printk.h:9,
from include/asm-generic/bug.h:22,
from arch/x86/include/asm/bug.h:87,
from include/linux/plist.h:80,
from include/linux/pm_qos.h:15,
from drivers/gpu/drm/i915/i915_drv.h:35,
from drivers/gpu/drm/i915/gt/intel_tlb.c:6:
drivers/gpu/drm/i915/gt/intel_tlb.c: In function 'intel_gt_invalidate_tlb_range':
drivers/gpu/drm/i915/gt/intel_tlb.c:190:41: error: 'const struct intel_device_info' has no member named 'ppgtt_size'
190 | vm_total = BIT_ULL(INTEL_INFO(gt->i915)->ppgtt_size);
| ^~
include/vdso/bits.h:8:34: note: in definition of macro 'BIT_ULL'
8 | #define BIT_ULL(nr) (ULL(1) << (nr))
| ^~
drivers/gpu/drm/i915/gt/intel_tlb.c:195:9: error: implicit declaration of function 'intel_guc_invalidate_tlb_page_selective' [-Werror=implicit-function-declaration]
195 | ret = intel_guc_invalidate_tlb_page_selective(guc,
| ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
drivers/gpu/drm/i915/gt/intel_tlb.c:196:14: error: 'INTEL_GUC_TLB_INVAL_MODE_HEAVY' undeclared (first use in this function)
196 | INTEL_GUC_TLB_INVAL_MODE_HEAVY,
| ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
In file included from drivers/gpu/drm/i915/gt/intel_tlb.c:214:
drivers/gpu/drm/i915/gt/selftest_tlb.c: In function 'pte_tlbinv':
>> drivers/gpu/drm/i915/gt/selftest_tlb.c:161:1: error: version control conflict marker in file
161 | <<<<<<< HEAD
| ^~~~~~~
drivers/gpu/drm/i915/gt/selftest_tlb.c:163:1: error: version control conflict marker in file
163 | =======
| ^~~~~~~
drivers/gpu/drm/i915/gt/selftest_tlb.c:165:1: error: version control conflict marker in file
165 | >>>>>>> 774058193c61b... INTEL_DII: drm/i915/xehpsdv: Add generic interface for tlb invalidation
| ^~~~~~~
>> drivers/gpu/drm/i915/gt/selftest_tlb.c:165:9: error: invalid suffix "c61b..." on integer constant
165 | >>>>>>> 774058193c61b... INTEL_DII: drm/i915/xehpsdv: Add generic interface for tlb invalidation
| ^~~~~~~~~~~~~~~~
>> drivers/gpu/drm/i915/gt/selftest_tlb.c:150:28: warning: unused variable 'vb_res' [-Wunused-variable]
150 | struct i915_vma_resource vb_res = {
| ^~~~~~
>> drivers/gpu/drm/i915/gt/selftest_tlb.c:40:21: warning: unused variable 'pat_index' [-Wunused-variable] 40 | const unsigned int pat_index =
| ^~~~~~~~~
cc1: some warnings being treated as errors
vim +161 drivers/gpu/drm/i915/gt/selftest_tlb.c
30
31 static int
32 pte_tlbinv(struct intel_context *ce,
33 struct i915_vma *va,
34 struct i915_vma *vb,
35 u64 align,
36 void (*tlbinv)(struct i915_address_space *vm, u64 addr, u64 length),
37 u64 length,
38 struct rnd_state *prng)
39 {
> 40 const unsigned int pat_index =
41 i915_gem_get_pat_index(ce->vm->i915, I915_CACHE_NONE);
42 struct drm_i915_gem_object *batch;
43 struct drm_mm_node vb_node;
44 struct i915_request *rq;
45 struct i915_vma *vma;
46 u64 addr;
47 int err;
48 u32 *cs;
49
50 batch = i915_gem_object_create_internal(ce->vm->i915, 4096);
51 if (IS_ERR(batch))
52 return PTR_ERR(batch);
53
54 vma = i915_vma_instance(batch, ce->vm, NULL);
55 if (IS_ERR(vma)) {
56 err = PTR_ERR(vma);
57 goto out;
58 }
59
60 err = i915_vma_pin(vma, 0, 0, PIN_USER);
61 if (err)
62 goto out;
63
64 /* Pin va at random but aligned offset after vma */
65 addr = round_up(vma->node.start + vma->node.size, align);
66 /* MI_CONDITIONAL_BATCH_BUFFER_END limits address to 48b */
67 addr = igt_random_offset(prng, addr, min(ce->vm->total, BIT_ULL(48)),
68 va->size, align);
69 err = i915_vma_pin(va, 0, 0, addr | PIN_OFFSET_FIXED | PIN_USER);
70 if (err) {
71 pr_err("Cannot pin at %llx+%llx\n", addr, va->size);
72 goto out;
73 }
74 GEM_BUG_ON(i915_vma_offset(va) != addr);
75 if (vb != va) {
76 vb_node = vb->node;
77 vb->node = va->node; /* overwrites the _same_ PTE */
78 }
79
80 /*
81 * Now choose random dword at the 1st pinned page.
82 *
83 * SZ_64K pages on dg1 require that the whole PT be marked
84 * containing 64KiB entries. So we make sure that vma
85 * covers the whole PT, despite being randomly aligned to 64KiB
86 * and restrict our sampling to the 2MiB PT within where
87 * we know that we will be using 64KiB pages.
88 */
89 if (align == SZ_64K)
90 addr = round_up(addr, SZ_2M);
91 addr = igt_random_offset(prng, addr, addr + align, 8, 8);
92
93 if (va != vb)
94 pr_info("%s(%s): Sampling %llx, with alignment %llx, using PTE size %x (phys %x, sg %x), invalidate:%llx+%llx\n",
95 ce->engine->name, va->obj->mm.region->name ?: "smem",
96 addr, align, va->resource->page_sizes_gtt,
97 va->page_sizes.phys, va->page_sizes.sg,
98 addr & -length, length);
99
100 cs = i915_gem_object_pin_map_unlocked(batch, I915_MAP_WC);
101 *cs++ = MI_NOOP; /* for later termination */
102 /*
103 * Sample the target to see if we spot the updated backing store.
104 * Gen8 VCS compares immediate value with bitwise-and of two
105 * consecutive DWORDS pointed by addr, other gen/engines compare value
106 * with DWORD pointed by addr. Moreover we want to exercise DWORD size
107 * invalidations. To fulfill all these requirements below values
108 * have been chosen.
109 */
110 *cs++ = MI_CONDITIONAL_BATCH_BUFFER_END | MI_DO_COMPARE | 2;
111 *cs++ = 0; /* break if *addr == 0 */
112 *cs++ = lower_32_bits(addr);
113 *cs++ = upper_32_bits(addr);
114 vma_set_qw(va, addr, -1);
115 vma_set_qw(vb, addr, 0);
116
117 /* Keep sampling until we get bored */
118 *cs++ = MI_BATCH_BUFFER_START | BIT(8) | 1;
119 *cs++ = lower_32_bits(i915_vma_offset(vma));
120 *cs++ = upper_32_bits(i915_vma_offset(vma));
121
122 i915_gem_object_flush_map(batch);
123
124 rq = i915_request_create(ce);
125 if (IS_ERR(rq)) {
126 err = PTR_ERR(rq);
127 goto out_va;
128 }
129
130 err = rq->engine->emit_bb_start(rq, i915_vma_offset(vma), 0, 0);
131 if (err) {
132 i915_request_add(rq);
133 goto out_va;
134 }
135
136 i915_request_get(rq);
137 i915_request_add(rq);
138
139 /* Short sleep to sanitycheck the batch is spinning before we begin */
140 msleep(10);
141 if (va == vb) {
142 if (!i915_request_completed(rq)) {
143 pr_err("%s(%s): Semaphore sanitycheck failed %llx, with alignment %llx, using PTE size %x (phys %x, sg %x)\n",
144 ce->engine->name, va->obj->mm.region->name ?: "smem",
145 addr, align, va->resource->page_sizes_gtt,
146 va->page_sizes.phys, va->page_sizes.sg);
147 err = -EIO;
148 }
149 } else if (!i915_request_completed(rq)) {
> 150 struct i915_vma_resource vb_res = {
151 .bi.pages = vb->obj->mm.pages,
152 .bi.page_sizes = vb->obj->mm.page_sizes,
153 .start = i915_vma_offset(vb),
154 .vma_size = i915_vma_size(vb)
155 };
156 unsigned int pte_flags = 0;
157
158 /* Flip the PTE between A and B */
159 if (i915_gem_object_is_lmem(vb->obj))
160 pte_flags |= PTE_LM;
> 161 <<<<<<< HEAD
162 ce->vm->insert_entries(ce->vm, &vb_res, pat_index, pte_flags);
163 =======
164 ce->vm->insert_entries(ce->vm, &stash, vb, I915_CACHE_NONE, pte_flags);
> 165 >>>>>>> 774058193c61b... INTEL_DII: drm/i915/xehpsdv: Add generic interface for tlb invalidation 166
167 /* Flush the PTE update to concurrent HW */
168 tlbinv(ce->vm, addr & -length, length);
169
170 if (wait_for(i915_request_completed(rq), HZ / 2)) {
171 pr_err("%s: Request did not complete; the COND_BBE did not read the updated PTE\n",
172 ce->engine->name);
173 err = -EINVAL;
174 }
175 } else {
176 pr_err("Spinner ended unexpectedly\n");
177 err = -EIO;
178 }
179 i915_request_put(rq);
180
181 cs = page_mask_bits(batch->mm.mapping);
182 *cs = MI_BATCH_BUFFER_END;
183 wmb();
184
185 out_va:
186 if (vb != va)
187 vb->node = vb_node;
188 i915_vma_unpin(va);
189 if (i915_vma_unbind_unlocked(va))
190 err = -EIO;
191 out:
192 i915_gem_object_put(batch);
193 return err;
194 }
195
Signature · 2 lines
0-DAY CI Kernel Test Service https://github.com/intel/lkp-tests/wiki
RFC PATCH 1/10 drm/i915: Add GuC TLB Invalidation device info flags
In reply to Jonathan CavittAdd device info flags for if GuC TLB Invalidation is enabled.
--- drivers/gpu/drm/i915/i915_drv.h | 2 ++ drivers/gpu/drm/i915/intel_device_info.h | 1 + 2 files changed, 3 insertions(+)
drivers/gpu/drm/i915/i915_drv.h +2 -0
@@ -794,6 +794,8 @@ IS_SUBPLATFORM(const struct drm_i915_private *i915,#define HAS_GUC_DEPRIVILEGE (i915) \ (INTEL_INFO(i915)->has_guc_deprivilege) +#define HAS_GUC_TLB_INVALIDATION (i915) (INTEL_INFO(i915)->has_guc_tlb_invalidation) +#define HAS_3D_PIPELINE (i915) (INTEL_INFO(i915)->has_3d_pipeline)#define HAS_ONE_EU_PER_FUSE_BIT (i915) (INTEL_INFO(i915)->has_one_eu_per_fuse_bit)
drivers/gpu/drm/i915/intel_device_info.h +1 -0
@@ -153,6 +153,7 @@ enum intel_ppgtt_type {func (has_heci_pxp); \func (has_heci_gscfi); \func (has_guc_deprivilege); \ +func (has_guc_tlb_invalidation); \func (has_l3_ccs_read); \func (has_l3_dpf); \func (has_llc); \
Signature · 1 line
2.25.1
PATCH 2/2 drm/i915: Use selective tlb invalidations where supported
In reply to Jonathan CavittFor platforms supporting selective tlb invalidations, we don't need to do a full tlb invalidation. Rather do a range based tlb invalidation for every unbind of purged vma belongs to an active vm.
--- drivers/gpu/drm/i915/gt/intel_ppgtt.c | 2 +- drivers/gpu/drm/i915/i915_vma.c | 14 +++++++++----- drivers/gpu/drm/i915/i915_vma.h | 3 ++- 3 files changed, 12 insertions(+), 7 deletions(-)
drivers/gpu/drm/i915/gt/intel_ppgtt.c +1 -1
@@ -211,7 +211,7 @@ void ppgtt_unbind_vma(struct i915_address_space *vm,return ; vm->clear_range (vm, vma_res->start, vma_res->vma_size); -vma_invalidate_tlb (vm, vma_res->tlb); +vma_invalidate_tlb (vm, vma_res->tlb, vma_res->start, vma_res->vma_size); }static unsigned long pd_count (u64 size,int shift)
drivers/gpu/drm/i915/i915_vma.c +9 -5
@@ -1339,7 +1339,8 @@ I915_SELFTEST_EXPORT int i915_vma_get_pages(struct i915_vma *vma)return err; } -void vma_invalidate_tlb (struct i915_address_space *vm,u32 *tlb) +void vma_invalidate_tlb(struct i915_address_space *vm, u32 *tlb + u64 start, u64 size) {struct intel_gt *gt;int id;
@@ -1355,9 +1356,11 @@ void vma_invalidate_tlb(struct i915_address_space *vm, u32 *tlb)* the most recent TLB invalidation seqno, and if we have not yet * flushed the TLBs upon release, perform a full invalidation. */ -for_each_gt (gt, vm->i915, id) -WRITE_ONCE (tlb[id], -intel_gt_next_invalidate_tlb_full (gt)); +for_each_gt (gt, vm->i915, id) { +if (!intel_gt_invalidate_tlb_range (gt, start, size)) +WRITE_ONCE (tlb[id], +intel_gt_next_invalidate_tlb_full (gt)); + } }static void __vma_put_pages (struct i915_vma *vma,unsigned int count)
@@ -2041,7 +2044,8 @@ struct dma_fence *__i915_vma_evict(struct i915_vma *vma, bool async)dma_fence_put (unbind_fence); unbind_fence =NULL ; } -vma_invalidate_tlb (vma->vm, vma->obj->mm.tlb); +vma_invalidate_tlb (vma->vm, vma->obj->mm.tlb, + vma->node.start, vma->size); }/*
drivers/gpu/drm/i915/i915_vma.h +2 -1
@@ -260,7 +260,8 @@ bool i915_vma_misplaced(const struct i915_vma *vma, u64 size, u64 alignment, u64 flags);void __i915_vma_set_map_and_fenceable (struct i915_vma *vma);void i915_vma_revoke_mmap (struct i915_vma *vma); -void vma_invalidate_tlb (struct i915_address_space *vm,u32 *tlb); +void vma_invalidate_tlb (struct i915_address_space *vm,u32 *tlb, +u64 start,u64 size);struct dma_fence *__i915_vma_evict (struct i915_vma *vma,bool async);int __i915_vma_unbind (struct i915_vma *vma);int __must_check i915_vma_unbind (struct i915_vma *vma);
Signature · 1 line
2.25.1
Re: [Intel-gfx] [PATCH dii-client 2/2] drm/i915: Use selective tlb invalidations where supported
Ignore this. It's not a security hole: it's just a temporary patch I was using for rebasing purposes that got smuggled into this series on accident. It has a bad tag because of some stale gitconfig params that I've since removed.
-Jonathan Cavitt
-----Original Message-----
From: Cavitt, Jonathan <[email protected]>
Sent: Tuesday, October 10, 2023 11:44 AM
To: [email protected]
Cc: Gupta, saurabhg <[email protected]>; Cavitt, Jonathan <[email protected]>; Das, Nirmoy <[email protected]>; Shyti, Andi <[email protected]>; [email protected]; Harrison, John C <[email protected]>
Subject: [PATCH dii-client 2/2] drm/i915: Use selective tlb invalidations where supported
Quoted 86 lines
For platforms supporting selective tlb invalidations, we don't need to do a full tlb invalidation. Rather do a range based tlb invalidation for every unbind of purged vma belongs to an active vm.
Signed-off-by Prathap Kumar Valsan <[email protected]>Cc Niranjana Vishwanathapura <[email protected]>Cc Fei Yang <[email protected]>Signed-off-by Mauro Carvalho Chehab <[email protected]>Signed-off-by Jonathan Cavitt <[email protected]>--- drivers/gpu/drm/i915/gt/intel_ppgtt.c | 2 +- drivers/gpu/drm/i915/i915_vma.c | 14 +++++++++----- drivers/gpu/drm/i915/i915_vma.h | 3 ++- 3 files changed, 12 insertions(+), 7 deletions(-)drivers/gpu/drm/i915/gt/intel_ppgtt.c +1
-1@@ -211,7 +211,7 @@ void ppgtt_unbind_vma(struct i915_address_space *vm,return ; vm->clear_range (vm, vma_res->start, vma_res->vma_size); -vma_invalidate_tlb (vm, vma_res->tlb); +vma_invalidate_tlb (vm, vma_res->tlb, vma_res->start, vma_res->vma_size); }static unsigned long pd_count (u64 size,int shift)drivers/gpu/drm/i915/i915_vma.c +9
-5@@ -1339,7 +1339,8 @@ I915_SELFTEST_EXPORT int i915_vma_get_pages(struct i915_vma *vma)return err; } -void vma_invalidate_tlb (struct i915_address_space *vm,u32 *tlb) +void vma_invalidate_tlb(struct i915_address_space *vm, u32 *tlb + u64 start, u64 size) {struct intel_gt *gt;int id;@@ -1355,9 +1356,11 @@ void vma_invalidate_tlb(struct i915_address_space *vm, u32 *tlb)* the most recent TLB invalidation seqno, and if we have not yet * flushed the TLBs upon release, perform a full invalidation. */ -for_each_gt (gt, vm->i915, id) -WRITE_ONCE (tlb[id], -intel_gt_next_invalidate_tlb_full (gt)); +for_each_gt (gt, vm->i915, id) { +if (!intel_gt_invalidate_tlb_range (gt, start, size)) +WRITE_ONCE (tlb[id], +intel_gt_next_invalidate_tlb_full (gt)); + } }static void __vma_put_pages (struct i915_vma *vma,unsigned int count)@@ -2041,7 +2044,8 @@ struct dma_fence *__i915_vma_evict(struct i915_vma *vma, bool async)dma_fence_put (unbind_fence); unbind_fence =NULL ; } -vma_invalidate_tlb (vma->vm, vma->obj->mm.tlb); +vma_invalidate_tlb (vma->vm, vma->obj->mm.tlb, + vma->node.start, vma->size); }/* drivers/gpu/drm/i915/i915_vma.h +2
-1@@ -260,7 +260,8 @@ bool i915_vma_misplaced(const struct i915_vma *vma, u64 size, u64 alignment, u64 flags);void __i915_vma_set_map_and_fenceable (struct i915_vma *vma);void i915_vma_revoke_mmap (struct i915_vma *vma); -void vma_invalidate_tlb (struct i915_address_space *vm,u32 *tlb); +void vma_invalidate_tlb (struct i915_address_space *vm,u32 *tlb, +u64 start,u64 size);struct dma_fence *__i915_vma_evict (struct i915_vma *vma,bool async);int __i915_vma_unbind (struct i915_vma *vma);int __must_check i915_vma_unbind (struct i915_vma *vma);Signature · 1 line
2.25.1
kernel test robot
build botRe: [Intel-gfx] [PATCH dii-client 2/2] drm/i915: Use selective tlb invalidations where supported
In reply to Jonathan Cavittkernel test robot noticed the following build errors:
Hi Jonathan,
kernel test robot noticed the following build errors:
[auto build test ERROR on drm-tip/drm-tip]
url: https://github.com/intel-lab-lkp/linux/commits/Jonathan-Cavitt/drm-i915-Use-selective-tlb-invalidations-where-supported/20231011-034501 base: git://anongit.freedesktop.org/drm/drm-tip drm-tip patch link: https://lore.kernel.org/r/20231010184423.2118908-4-jonathan.cavitt%40intel.com patch subject: [Intel-gfx] [PATCH dii-client 2/2] drm/i915: Use selective tlb invalidations where supported config: x86_64-rhel-8.3-rust (https://download.01.org/0day-ci/archive/20231012/[email protected]/config) compiler: clang version 16.0.4 (https://github.com/llvm/llvm-project.git ae42196bc493ffe877a7e3dff8be32035dea4d07) reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20231012/[email protected]/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <[email protected]>
| Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/
All errors (new ones prefixed by >>):
>> drivers/gpu/drm/i915/i915_vma.c:1343:4: error: expected ')'
u64 start, u64 size)
^
drivers/gpu/drm/i915/i915_vma.c:1342:24: note: to match this '('
void vma_invalidate_tlb(struct i915_address_space *vm, u32 *tlb
^
>> drivers/gpu/drm/i915/i915_vma.c:1342:6: error: conflicting types for 'vma_invalidate_tlb'
void vma_invalidate_tlb(struct i915_address_space *vm, u32 *tlb
^
drivers/gpu/drm/i915/i915_vma.h:263:6: note: previous declaration is here
void vma_invalidate_tlb(struct i915_address_space *vm, u32 *tlb,
^
>> drivers/gpu/drm/i915/i915_vma.c:1360:42: error: use of undeclared identifier 'start'; did you mean 'stac'?
if (!intel_gt_invalidate_tlb_range(gt, start, size))
^~~~~
stac
arch/x86/include/asm/smap.h:36:29: note: 'stac' declared here
static __always_inline void stac(void)
^
>> drivers/gpu/drm/i915/i915_vma.c:1360:49: error: use of undeclared identifier 'size'; did you mean 'ksize'?
if (!intel_gt_invalidate_tlb_range(gt, start, size))
^~~~
ksize
include/linux/slab.h:245:8: note: 'ksize' declared here
size_t ksize(const void *objp);
^4 errors generated.
vim +1343 drivers/gpu/drm/i915/i915_vma.c
1341
> 1342 void vma_invalidate_tlb(struct i915_address_space *vm, u32 *tlb
> 1343 u64 start, u64 size)
1344 {
1345 struct intel_gt *gt;
1346 int id;
1347
1348 if (!tlb)
1349 return;
1350
1351 /*
1352 * Before we release the pages that were bound by this vma, we
1353 * must invalidate all the TLBs that may still have a reference
1354 * back to our physical address. It only needs to be done once,
1355 * so after updating the PTE to point away from the pages, record
1356 * the most recent TLB invalidation seqno, and if we have not yet
1357 * flushed the TLBs upon release, perform a full invalidation.
1358 */
1359 for_each_gt(gt, vm->i915, id) {
> 1360 if (!intel_gt_invalidate_tlb_range(gt, start, size)) 1361 WRITE_ONCE(tlb[id],
1362 intel_gt_next_invalidate_tlb_full(gt));
1363 }
1364 }
1365
Signature · 2 lines
0-DAY CI Kernel Test Service https://github.com/intel/lkp-tests/wiki
kernel test robot
build botRe: [Intel-gfx] [PATCH dii-client 2/2] drm/i915: Use selective tlb invalidations where supported
In reply to Jonathan Cavittkernel test robot noticed the following build errors:
Hi Jonathan,
kernel test robot noticed the following build errors:
[auto build test ERROR on drm-tip/drm-tip]
url: https://github.com/intel-lab-lkp/linux/commits/Jonathan-Cavitt/drm-i915-Use-selective-tlb-invalidations-where-supported/20231011-034501 base: git://anongit.freedesktop.org/drm/drm-tip drm-tip patch link: https://lore.kernel.org/r/20231010184423.2118908-4-jonathan.cavitt%40intel.com patch subject: [Intel-gfx] [PATCH dii-client 2/2] drm/i915: Use selective tlb invalidations where supported config: x86_64-allyesconfig (https://download.01.org/0day-ci/archive/20231021/[email protected]/config) compiler: gcc-12 (Debian 12.2.0-14) 12.2.0 reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20231021/[email protected]/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <[email protected]>
| Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/
All errors (new ones prefixed by >>):
>> drivers/gpu/drm/i915/i915_vma.c:1343:25: error: expected ';', ',' or ')' before 'u64'
1343 | u64 start, u64 size)
| ^~~vim +1343 drivers/gpu/drm/i915/i915_vma.c
1341
1342 void vma_invalidate_tlb(struct i915_address_space *vm, u32 *tlb
> 1343 u64 start, u64 size)
1344 {
1345 struct intel_gt *gt;
1346 int id;
1347
1348 if (!tlb)
1349 return;
1350
1351 /*
1352 * Before we release the pages that were bound by this vma, we
1353 * must invalidate all the TLBs that may still have a reference
1354 * back to our physical address. It only needs to be done once,
1355 * so after updating the PTE to point away from the pages, record
1356 * the most recent TLB invalidation seqno, and if we have not yet
1357 * flushed the TLBs upon release, perform a full invalidation.
1358 */
1359 for_each_gt(gt, vm->i915, id) {
1360 if (!intel_gt_invalidate_tlb_range(gt, start, size))
1361 WRITE_ONCE(tlb[id],
1362 intel_gt_next_invalidate_tlb_full(gt));
1363 }
1364 }
1365
Signature · 2 lines
0-DAY CI Kernel Test Service https://github.com/intel/lkp-tests/wiki
RFC PATCH 2/10 drm/i915/guc: Add CT size delay helper
In reply to Jonathan CavittAdd a helper function to the GuC CT buffer that reports the expected time to process all outstanding requests. As of now, there is no functionality to check number of requests in the buffer, so the helper function just reports 2 seconds, or 1ms per request up to the maximum number of requests the CT buffer can store.
--- drivers/gpu/drm/i915/gt/uc/intel_guc_ct.h | 13 +++++++++++++ 1 file changed, 13 insertions(+)
drivers/gpu/drm/i915/gt/uc/intel_guc_ct.h +13 -0
@@ -120,6 +120,19 @@ static inline bool intel_guc_ct_enabled(struct intel_guc_ct *ct)return ct->enabled; } +/* +* GuC has a timeout of 1ms for a TLB invalidation response from GAM. On a +* timeout GuC drops the request and has no mechanism to notify the host about +* the timeout. There is also no mechanism for determining the number of +* outstanding requests in the CT buffer. Ergo, keep a larger timeout that accounts +* for this individual timeout and the max number of outstanding requests that +* can be queued in CT buffer. +*/ +static inline long intel_guc_ct_expected_delay (struct intel_guc_ct *ct) +{ +return HZ *2 ; +} +#define INTEL_GUC_CT_SEND_NB BIT(31)#define INTEL_GUC_CT_SEND_G2H_DW_SHIFT 0#define INTEL_GUC_CT_SEND_G2H_DW_MASK (0xff << INTEL_GUC_CT_SEND_G2H_DW_SHIFT)
Signature · 1 line
2.25.1
RFC PATCH 3/10 drm/i915: Define and use GuC and CTB TLB invalidation routines
In reply to Jonathan CavittFrom: Prathap Kumar Valsan <[email protected]>
The GuC firmware had defined the interface for Translation Look-Aside Buffer (TLB) invalidation. We should use this interface when invalidating the engine and GuC TLBs.
Add additional functionality to intel_gt_invalidate_tlb, invalidating the GuC TLBs and falling back to GT invalidation when the GuC is disabled.
The invalidation is done by sending a request directly to the GuC tlb_lookup that invalidates the table. The invalidation is submitted as a wait request and is performed in the CT event handler. This means we cannot perform this TLB invalidation path if the CT is not enabled. If the request isn't fulfilled in two seconds, this would constitute an error in the invalidation as that would constitute either a lost request or a severe GuC overload.
With this new invalidation routine, we can perform GuC-based GGTT invalidations. GuC-based GGTT invalidation is incompatible with MMIO invalidation so we should not perform MMIO invalidation when GuC-based GGTT invalidation is expected.
The additional complexity incurred in this patch will be necessary for range-based tlb invalidations, which will be platformed in the future.
--- drivers/gpu/drm/i915/gt/intel_ggtt.c | 34 +++- drivers/gpu/drm/i915/gt/intel_tlb.c | 16 +- .../gpu/drm/i915/gt/uc/abi/guc_actions_abi.h | 33 ++++ drivers/gpu/drm/i915/gt/uc/intel_guc.h | 22 +++ drivers/gpu/drm/i915/gt/uc/intel_guc_ct.c | 4 + drivers/gpu/drm/i915/gt/uc/intel_guc_fwif.h | 1 + .../gpu/drm/i915/gt/uc/intel_guc_submission.c | 186 +++++++++++++++++- 7 files changed, 284 insertions(+), 12 deletions(-)
drivers/gpu/drm/i915/gt/intel_ggtt.c +25 -9
@@ -206,22 +206,38 @@ static void gen8_ggtt_invalidate(struct i915_ggtt *ggtt)intel_uncore_write_fw (uncore,GFX_FLSH_CNTL_GEN6 ,GFX_FLSH_CNTL_EN ); } +static void guc_ggtt_ct_invalidate (struct intel_gt *gt) +{ +struct intel_uncore *uncore = gt->uncore; +intel_wakeref_t wakeref; + + with_intel_runtime_pm_if_active(uncore->rpm, wakeref) { +struct intel_guc *guc = >->uc.guc; + +intel_guc_invalidate_tlb_guc (guc); + } +} +static void guc_ggtt_invalidate (struct i915_ggtt *ggtt) {struct drm_i915_private *i915 = ggtt->vm.i915; +struct intel_gt *gt; -gen8_ggtt_invalidate (ggtt); - -if (GRAPHICS_VER (i915) >=12 ) { -struct intel_gt *gt; +if (!HAS_GUC_TLB_INVALIDATION (i915)) +gen8_ggtt_invalidate (ggtt); -list_for_each_entry (gt, &ggtt->gt_list, ggtt_link) +list_for_each_entry (gt, &ggtt->gt_list, ggtt_link) { +if (HAS_GUC_TLB_INVALIDATION (i915) && +intel_guc_is_ready (>->uc.guc)) { +guc_ggtt_ct_invalidate (gt); + }else if (GRAPHICS_VER (i915) >=12 ) {intel_uncore_write_fw (gt->uncore,GEN12_GUC_TLB_INV_CR ,GEN12_GUC_TLB_INV_CR_INVALIDATE ); - }else { -intel_uncore_write_fw (ggtt->vm.gt->uncore, -GEN8_GTCR ,GEN8_GTCR_INVALIDATE ); + }else { +intel_uncore_write_fw (gt->uncore, +GEN8_GTCR ,GEN8_GTCR_INVALIDATE ); + } } }
@@ -1243,7 +1259,7 @@ static int gen8_gmch_probe(struct i915_ggtt *ggtt) ggtt->vm.raw_insert_page = gen8_ggtt_insert_page; } -if (intel_uc_wants_guc (&ggtt->vm.gt->uc)) +if (intel_uc_wants_guc_submission (&ggtt->vm.gt->uc)) ggtt->invalidate = guc_ggtt_invalidate;else ggtt->invalidate = gen8_ggtt_invalidate;
drivers/gpu/drm/i915/gt/intel_tlb.c +15 -1
@@ -12,6 +12,7 @@#include "intel_gt_print.h" #include "intel_gt_regs.h" #include "intel_tlb.h" +#include "uc/intel_guc.h" /* * HW architecture suggest typical invalidation time at 40us,
@@ -131,11 +132,24 @@ void intel_gt_invalidate_tlb_full(struct intel_gt *gt, u32 seqno)return ; with_intel_gt_pm_if_awake(gt, wakeref) { +struct intel_guc *guc = >->uc.guc; +mutex_lock (>->tlb.invalidate_lock);if (tlb_seqno_passed (gt, seqno)) goto unlock; -mmio_invalidate_full (gt); +if (HAS_GUC_TLB_INVALIDATION (gt->i915)) { +/* +* Only perform GuC TLB invalidation if GuC is ready. +* The only time GuC could not be ready is on GT reset, +* which would clobber all the TLBs anyways, making +* any TLB invalidation path here unnecessary. +*/ +if (intel_guc_is_ready (guc)) +intel_guc_invalidate_tlb_engines (guc); + }else { +mmio_invalidate_full (gt); + }write_seqcount_invalidate (>->tlb.seqno); unlock:
drivers/gpu/drm/i915/gt/uc/abi/guc_actions_abi.h +33 -0
@@ -138,6 +138,8 @@ enum intel_guc_action {INTEL_GUC_ACTION_REGISTER_CONTEXT_MULTI_LRC =0x4601 ,INTEL_GUC_ACTION_CLIENT_SOFT_RESET =0x5507 ,INTEL_GUC_ACTION_SET_ENG_UTIL_BUFF =0x550A , +INTEL_GUC_ACTION_TLB_INVALIDATION =0x7000 , +INTEL_GUC_ACTION_TLB_INVALIDATION_DONE =0x7001 ,INTEL_GUC_ACTION_STATE_CAPTURE_NOTIFICATION =0x8002 ,INTEL_GUC_ACTION_NOTIFY_FLUSH_LOG_BUFFER_TO_FILE =0x8003 , INTEL_GUC_ACTION_NOTIFY_CRASH_DUMP_POSTED = 0x8004,
@@ -181,4 +183,35 @@ enum intel_guc_state_capture_event_status {#define INTEL_GUC_STATE_CAPTURE_EVENT_STATUS_MASK 0x000000FF +#define INTEL_GUC_TLB_INVAL_TYPE_MASK REG_GENMASK(7, 0) +#define INTEL_GUC_TLB_INVAL_MODE_MASK REG_GENMASK(11, 8) +#define INTEL_GUC_TLB_INVAL_FLUSH_CACHE REG_BIT(31) + +enum intel_guc_tlb_invalidation_type { +INTEL_GUC_TLB_INVAL_ENGINES =0x0 , +INTEL_GUC_TLB_INVAL_GUC =0x3 , +}; + +/* +* 0: Heavy mode of Invalidation: +* The pipeline of the engine(s) for which the invalidation is targeted to is +* blocked, and all the in-flight transactions are guaranteed to be Globally +* Observed before completing the TLB invalidation +* 1: Lite mode of Invalidation: +* TLBs of the targeted engine(s) are immediately invalidated. +* In-flight transactions are NOT guaranteed to be Globally Observed before +* completing TLB invalidation. +* Light Invalidation Mode is to be used only when +* it can be guaranteed (by SW) that the address translations remain invariant +* for the in-flight transactions across the TLB invalidation. In other words, +* this mode can be used when the TLB invalidation is intended to clear out the +* stale cached translations that are no longer in use. Light Invalidation Mode +* is much faster than the Heavy Invalidation Mode, as it does not wait for the +* in-flight transactions to be GOd. +*/ +enum intel_guc_tlb_inval_mode { +INTEL_GUC_TLB_INVAL_MODE_HEAVY =0x0 , +INTEL_GUC_TLB_INVAL_MODE_LITE =0x1 , +}; +#endif /* _ABI_GUC_ACTIONS_ABI_H */
drivers/gpu/drm/i915/gt/uc/intel_guc.h +22 -0
@@ -79,6 +79,18 @@ struct intel_guc {*/ atomic_t outstanding_submission_g2h; +/** @tlb_lookup: xarray to store all pending TLB invalidation requests */ +struct xarray tlb_lookup; + +/** +* @serial_slot: id to the initial waiter created in tlb_lookup, +* which is used only when failed to allocate new waiter. +*/ +u32 serial_slot; + +/** @next_seqno: the next id (sequence no.) to allocate. */ +u32 next_seqno; +/** @interrupts: pointers to GuC interrupt-managing functions. */ struct {bool enabled;
@@ -297,6 +309,11 @@ struct intel_guc {#define GUC_SUBMIT_VER (guc) MAKE_GUC_VER_STRUCT((guc)->submission_version)#define GUC_FIRMWARE_VER (guc) MAKE_GUC_VER_STRUCT((guc)->fw.file_selected.ver) +struct intel_guc_tlb_wait { +struct wait_queue_head wq; +bool busy; +}; +static inline struct intel_guc *log_to_guc (struct intel_guc_log *log) { return container_of(log, struct intel_guc, log);
@@ -419,6 +436,11 @@ static inline bool intel_guc_is_supported(struct intel_guc *guc)return intel_uc_fw_is_supported (&guc->fw); } +int intel_guc_invalidate_tlb_engines (struct intel_guc *guc); +int intel_guc_invalidate_tlb_guc (struct intel_guc *guc); +int intel_guc_tlb_invalidation_done (struct intel_guc *guc, +const u32 *payload,u32 len); +static inline bool intel_guc_is_wanted (struct intel_guc *guc) {return intel_uc_fw_is_enabled (&guc->fw);
drivers/gpu/drm/i915/gt/uc/intel_guc_ct.c +4 -0
@@ -1115,6 +1115,9 @@ static int ct_process_request(struct intel_guc_ct *ct, struct ct_incoming_msg *rcase INTEL_GUC_ACTION_NOTIFY_EXCEPTION : ret =intel_guc_crash_process_msg (guc, action);break ; +case INTEL_GUC_ACTION_TLB_INVALIDATION_DONE : + ret =intel_guc_tlb_invalidation_done (guc, payload, len); +break ;default : ret = -EOPNOTSUPP ;break ;
@@ -1186,6 +1189,7 @@ static int ct_handle_event(struct intel_guc_ct *ct, struct ct_incoming_msg *requswitch (action) {case INTEL_GUC_ACTION_SCHED_CONTEXT_MODE_DONE :case INTEL_GUC_ACTION_DEREGISTER_CONTEXT_DONE : +case INTEL_GUC_ACTION_TLB_INVALIDATION_DONE :g2h_release_space (ct, request->size); }
drivers/gpu/drm/i915/gt/uc/intel_guc_fwif.h +1 -0
@@ -22,6 +22,7 @@/* Payload length only i.e. don't include G2H header length */ #define G2H_LEN_DW_SCHED_CONTEXT_MODE_SET 2#define G2H_LEN_DW_DEREGISTER_CONTEXT 1 +#define G2H_LEN_DW_INVALIDATE_TLB 1#define GUC_CONTEXT_DISABLE 0#define GUC_CONTEXT_ENABLE 1
drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c +184 -2
237 lines not on this page · raw message
Signature · 1 line
2.25.1
RFC PATCH 4/10 drm/i915: No TLB invalidation on suspended GT
In reply to Jonathan CavittIn case of GT is suspended, don't allow submission of new TLB invalidation request and cancel all pending requests. The TLB entries will be invalidated either during GuC reload or on system resume.
--- drivers/gpu/drm/i915/gt/uc/intel_guc.h | 1 + .../gpu/drm/i915/gt/uc/intel_guc_submission.c | 21 +++++++++++++------ drivers/gpu/drm/i915/gt/uc/intel_uc.c | 7 +++++++ 3 files changed, 23 insertions(+), 6 deletions(-)
drivers/gpu/drm/i915/gt/uc/intel_guc.h +1 -0
@@ -537,4 +537,5 @@ void intel_guc_dump_time_info(struct intel_guc *guc, struct drm_printer *p);int intel_guc_sched_disable_gucid_threshold_max (struct intel_guc *guc); +void wake_up_all_tlb_invalidate (struct intel_guc *guc);#endif
drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c +15 -6
@@ -1796,13 +1796,25 @@ static void __guc_reset_context(struct intel_context *ce, intel_engine_mask_t stintel_context_put (parent); } -void intel_guc_submission_reset (struct intel_guc *guc,intel_engine_mask_t stalled) +void wake_up_all_tlb_invalidate (struct intel_guc *guc) {struct intel_guc_tlb_wait *wait; +unsigned long i; + +if (!HAS_GUC_TLB_INVALIDATION (guc_to_gt (guc)->i915)) +return ; + +xa_lock_irq (&guc->tlb_lookup); +xa_for_each (&guc->tlb_lookup, i, wait) +wake_up (&wait->wq); +xa_unlock_irq (&guc->tlb_lookup); +} + +void intel_guc_submission_reset (struct intel_guc *guc,intel_engine_mask_t stalled) +{struct intel_context *ce;unsigned long index;unsigned long flags; -unsigned long i;if (unlikely (!guc_submission_initialized (guc))) {/* Reset called during driver load? GuC not yet initialised! */
@@ -1833,10 +1845,7 @@ void intel_guc_submission_reset(struct intel_guc *guc, intel_engine_mask_t stall* The full GT reset will have cleared the TLB caches and flushed the * G2H message queue; we can release all the blocked waiters. */ -xa_lock_irq (&guc->tlb_lookup); -xa_for_each (&guc->tlb_lookup, i, wait) -wake_up (&wait->wq); -xa_unlock_irq (&guc->tlb_lookup); +wake_up_all_tlb_invalidate (guc); }static void guc_cancel_context_requests (struct intel_context *ce)
drivers/gpu/drm/i915/gt/uc/intel_uc.c +7 -0
@@ -688,6 +688,8 @@ void intel_uc_suspend(struct intel_uc *uc)/* flush the GSC worker */ intel_gsc_uc_flush_work (&uc->gsc); +wake_up_all_tlb_invalidate (guc); +if (!intel_guc_is_ready (guc)) { guc->interrupts.enabled = false;return ;
@@ -736,6 +738,11 @@ static int __uc_resume(struct intel_uc *uc, bool enable_communication)intel_gsc_uc_resume (&uc->gsc); +if (HAS_GUC_TLB_INVALIDATION (gt->i915)) { +intel_guc_invalidate_tlb_engines (guc); +intel_guc_invalidate_tlb_guc (guc); + } +return 0 ; }
Signature · 1 line
2.25.1
RFC PATCH 5/10 drm/i915: No TLB invalidation on wedged GT
In reply to Jonathan CavittIt is not an error for GuC TLB invalidations to fail when the GT is wedged or disabled, so do not process a wait failure as one in guc_send_invalidate_tlb.
--- .../gpu/drm/i915/gt/uc/intel_guc_submission.c | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-)
drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c +17 -1
@@ -32,6 +32,7 @@#include "i915_drv.h" #include "i915_reg.h" +#include "i915_irq.h" #include "i915_trace.h" /**
@@ -1941,6 +1942,12 @@ void intel_guc_submission_cancel_requests(struct intel_guc *guc)/* GuC is blown away, drop all references to contexts */ xa_destroy (&guc->context_lookup); + +/* +* Wedged GT won't respond to any TLB invalidation request. Simply +* release all the blocked waiters. +*/ +wake_up_all_tlb_invalidate (guc); }void intel_guc_submission_reset_finish (struct intel_guc *guc)
@@ -4738,6 +4745,14 @@ static long must_wait_woken(struct wait_queue_entry *wq_entry, long timeout)return timeout; } +static bool intel_gt_is_enabled (const struct intel_gt *gt) +{ +/* Check if GT is wedged or suspended */ +if (intel_gt_is_wedged (gt) || !intel_irqs_enabled (gt->i915)) +return false; +return true; +} +static int guc_send_invalidate_tlb (struct intel_guc *guc,enum intel_guc_tlb_invalidation_type type) {
@@ -4790,7 +4805,8 @@ static int guc_send_invalidate_tlb(struct intel_guc *guc,if (err) goto out; -if (!must_wait_woken (&wait,intel_guc_ct_expected_delay (&guc->ct))) { +if (intel_gt_is_enabled (guc_to_gt (guc)) && + !must_wait_woken (&wait,intel_guc_ct_expected_delay (&guc->ct))) {guc_err (guc,"TLB invalidation response timed out for seqno %u\n" , seqno); err = -ETIME ;
Signature · 1 line
2.25.1
RFC PATCH 6/10 drm/i915/gt: Increase sleep in gt_tlb selftest sanitycheck
In reply to Jonathan CavittFor the gt_tlb live selftest, when operating on the GSC engine, increase the timeout from 10 ms to 200 ms because the GSC engine is a bit slower than the rest.
Additionally, increase the default timeout from 10 ms to 20 ms because msleep < 20ms can sleep for up to 20ms.
--- drivers/gpu/drm/i915/gt/selftest_tlb.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-)
drivers/gpu/drm/i915/gt/selftest_tlb.c +9 -2
@@ -136,8 +136,15 @@ pte_tlbinv(struct intel_context *ce,i915_request_get (rq);i915_request_add (rq); -/* Short sleep to sanitycheck the batch is spinning before we begin */ -msleep (10 ); +/* +* Short sleep to sanitycheck the batch is spinning before we begin. +* FIXME: Why is GSC so slow? +*/ +if (ce->engine->class ==OTHER_CLASS ) +msleep (200 ); +else +msleep (20 ); +if (va == vb) {if (!i915_request_completed (rq)) { pr_err("%s(%s): Semaphore sanitycheck failed %llx, with alignment %llx, using PTE size %x (phys %x, sg %x)\n",
Signature · 1 line
2.25.1
RFC PATCH 7/10 drm/i915: Enable GuC TLB invalidations for MTL
In reply to Jonathan CavittEnable GuC TLB invalidations for MTL. Though more platforms than just MTL support GuC TLB invalidations, MTL is presently the only platform that requires it for any purpose, so only enable it there for now to minimize cross-platform impact.
--- drivers/gpu/drm/i915/i915_pci.c | 1 + 1 file changed, 1 insertion(+)
drivers/gpu/drm/i915/i915_pci.c +1 -0
@@ -829,6 +829,7 @@ static const struct intel_device_info mtl_info = { .has_flat_ccs = 0, .has_gmd_id = 1, .has_guc_deprivilege = 1, + .has_guc_tlb_invalidation = 1, .has_llc = 0, .has_mslice_steering = 0, .has_snoop = 1,
Signature · 1 line
2.25.1
RFC PATCH 8/10 drm/i915: Define GuC Based TLB invalidation routines
In reply to Jonathan CavittFrom: Prathap Kumar Valsan <[email protected]>
The GuC firmware has defined the interface for selective TLB invalidation support. This patch adds routines to interface with GuC.
--- .../gpu/drm/i915/gt/uc/abi/guc_actions_abi.h | 2 + drivers/gpu/drm/i915/gt/uc/intel_guc.h | 11 ++ .../gpu/drm/i915/gt/uc/intel_guc_submission.c | 105 +++++++++++++++--- 3 files changed, 105 insertions(+), 13 deletions(-)
drivers/gpu/drm/i915/gt/uc/abi/guc_actions_abi.h +2 -0
@@ -189,6 +189,8 @@ enum intel_guc_state_capture_event_status {enum intel_guc_tlb_invalidation_type {INTEL_GUC_TLB_INVAL_ENGINES =0x0 , +INTEL_GUC_TLB_INVAL_PAGE_SELECTIVE =0x1 , +INTEL_GUC_TLB_INVAL_PAGE_SELECTIVE_CTX =0x2 ,INTEL_GUC_TLB_INVAL_GUC =0x3 , };
drivers/gpu/drm/i915/gt/uc/intel_guc.h +11 -0
@@ -431,6 +431,17 @@ int intel_guc_allocate_and_map_vma(struct intel_guc *guc, u32 size,int intel_guc_self_cfg32 (struct intel_guc *guc,u16 key,u32 value);int intel_guc_self_cfg64 (struct intel_guc *guc,u16 key,u64 value); +int intel_guc_g2g_register (struct intel_guc *guc); + +int intel_guc_invalidate_tlb_full (struct intel_guc *guc); +int intel_guc_invalidate_tlb_page_selective (struct intel_guc *guc, +enum intel_guc_tlb_inval_mode mode, +u64 start,u64 length); +int intel_guc_invalidate_tlb_page_selective_ctx (struct intel_guc *guc, +enum intel_guc_tlb_inval_mode mode, +u64 start,u64 length,u32 ctxid); +int intel_guc_invalidate_tlb_guc (struct intel_guc *guc); +static inline bool intel_guc_is_supported (struct intel_guc *guc) {return intel_uc_fw_is_supported (&guc->fw);
drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c +92 -13
127 lines not on this page · raw message
Signature · 1 line
2.25.1
RFC PATCH 9/10 drm/i915: Add generic interface for tlb invalidation
In reply to Jonathan CavittFrom: Prathap Kumar Valsan <[email protected]>
This supports selective and full tlb invalidations. When GuC is enabled the tlb invalidations use guc ct otherwise use mmio interface.
--- drivers/gpu/drm/i915/gt/intel_gt_regs.h | 8 ++ drivers/gpu/drm/i915/gt/intel_tlb.c | 52 +++++++++++ drivers/gpu/drm/i915/gt/intel_tlb.h | 1 + drivers/gpu/drm/i915/gt/selftest_tlb.c | 88 +++++++++++++++++++ .../drm/i915/selftests/i915_mock_selftests.h | 1 + 5 files changed, 150 insertions(+)
drivers/gpu/drm/i915/gt/intel_gt_regs.h +8 -0
@@ -1124,6 +1124,14 @@#define GEN12_GAM_DONE _MMIO(0xcf68) +#define XEHPSDV_TLB_INV_DESC0 _MMIO(0xcf7c) +#define XEHPSDV_TLB_INV_DESC0_ADDR_LO REG_GENMASK(31, 12) +#define XEHPSDV_TLB_INV_DESC0_ADDR_MASK REG_GENMASK(8, 3) +#define XEHPSDV_TLB_INV_DESC0_G REG_GENMASK(2, 1) +#define XEHPSDV_TLB_INV_DESC0_VALID REG_BIT(0) +#define XEHPSDV_TLB_INV_DESC1 _MMIO(0xcf80) +#define XEHPSDV_TLB_INV_DESC0_ADDR_HI REG_GENMASK(31, 0) +#define GEN7_HALF_SLICE_CHICKEN1 _MMIO(0xe100)/* IVB GT1 + VLV */ #define GEN8_HALF_SLICE_CHICKEN1 MCR_REG(0xe100)#define GEN7_MAX_PS_THREAD_DEP (8 << 12)
drivers/gpu/drm/i915/gt/intel_tlb.c +52 -0
58 lines not on this page · raw message
drivers/gpu/drm/i915/gt/intel_tlb.h +1 -0
@@ -12,6 +12,7 @@#include "intel_gt_types.h" void intel_gt_invalidate_tlb_full (struct intel_gt *gt,u32 seqno); +bool intel_gt_invalidate_tlb_range (struct intel_gt *gt,u64 start,u64 length);void intel_gt_init_tlb (struct intel_gt *gt);void intel_gt_fini_tlb (struct intel_gt *gt);
drivers/gpu/drm/i915/gt/selftest_tlb.c +88 -0
101 lines not on this page · raw message
drivers/gpu/drm/i915/selftests/i915_mock_selftests.h +1 -0
@@ -21,6 +21,7 @@ selftest(fence, i915_sw_fence_mock_selftests) selftest(scatterlist, scatterlist_mock_selftests) selftest(syncmap, i915_syncmap_mock_selftests) selftest(uncore, intel_uncore_mock_selftests) +selftest(tlb, intel_tlb_mock_selftests) selftest(ring, intel_ring_mock_selftests) selftest(engine, intel_engine_cs_mock_selftests)selftest (timelines, intel_timeline_mock_selftests)
Signature · 1 line
2.25.1
RFC PATCH 10/10 drm/i915: Use selective tlb invalidations where supported
In reply to Jonathan CavittFor platforms supporting selective tlb invalidations, we don't need to do a full tlb invalidation. Rather do a range based tlb invalidation for every unbind of purged vma belongs to an active vm.
--- drivers/gpu/drm/i915/gt/intel_ppgtt.c | 2 +- drivers/gpu/drm/i915/i915_vma.c | 14 +++++++++----- drivers/gpu/drm/i915/i915_vma.h | 3 ++- 3 files changed, 12 insertions(+), 7 deletions(-)
drivers/gpu/drm/i915/gt/intel_ppgtt.c +1 -1
@@ -211,7 +211,7 @@ void ppgtt_unbind_vma(struct i915_address_space *vm,return ; vm->clear_range (vm, vma_res->start, vma_res->vma_size); -vma_invalidate_tlb (vm, vma_res->tlb); +vma_invalidate_tlb (vm, vma_res->tlb, vma_res->start, vma_res->vma_size); }static unsigned long pd_count (u64 size,int shift)
drivers/gpu/drm/i915/i915_vma.c +9 -5
@@ -1339,7 +1339,8 @@ I915_SELFTEST_EXPORT int i915_vma_get_pages(struct i915_vma *vma)return err; } -void vma_invalidate_tlb (struct i915_address_space *vm,u32 *tlb) +void vma_invalidate_tlb(struct i915_address_space *vm, u32 *tlb + u64 start, u64 size) {struct intel_gt *gt;int id;
@@ -1355,9 +1356,11 @@ void vma_invalidate_tlb(struct i915_address_space *vm, u32 *tlb)* the most recent TLB invalidation seqno, and if we have not yet * flushed the TLBs upon release, perform a full invalidation. */ -for_each_gt (gt, vm->i915, id) -WRITE_ONCE (tlb[id], -intel_gt_next_invalidate_tlb_full (gt)); +for_each_gt (gt, vm->i915, id) { +if (!intel_gt_invalidate_tlb_range (gt, start, size)) +WRITE_ONCE (tlb[id], +intel_gt_next_invalidate_tlb_full (gt)); + } }static void __vma_put_pages (struct i915_vma *vma,unsigned int count)
@@ -2041,7 +2044,8 @@ struct dma_fence *__i915_vma_evict(struct i915_vma *vma, bool async)dma_fence_put (unbind_fence); unbind_fence =NULL ; } -vma_invalidate_tlb (vma->vm, vma->obj->mm.tlb); +vma_invalidate_tlb (vma->vm, vma->obj->mm.tlb, + vma->node.start, vma->size); }/*
drivers/gpu/drm/i915/i915_vma.h +2 -1
@@ -260,7 +260,8 @@ bool i915_vma_misplaced(const struct i915_vma *vma, u64 size, u64 alignment, u64 flags);void __i915_vma_set_map_and_fenceable (struct i915_vma *vma);void i915_vma_revoke_mmap (struct i915_vma *vma); -void vma_invalidate_tlb (struct i915_address_space *vm,u32 *tlb); +void vma_invalidate_tlb (struct i915_address_space *vm,u32 *tlb, +u64 start,u64 size);struct dma_fence *__i915_vma_evict (struct i915_vma *vma,bool async);int __i915_vma_unbind (struct i915_vma *vma);int __must_check i915_vma_unbind (struct i915_vma *vma);
Signature · 1 line
2.25.1
Re: [Intel-gfx] [RFC PATCH 10/10] drm/i915: Use selective tlb invalidations where supported
Quoted 58 lines · Jonathan Cavitt
For platforms supporting selective tlb invalidations, we don't need to do a full tlb invalidation. Rather do a range based tlb invalidation for every unbind of purged vma belongs to an active vm.
Signed-off-by Prathap Kumar Valsan <[email protected]>Cc Niranjana Vishwanathapura <[email protected]>Cc Fei Yang <[email protected]>Signed-off-by Mauro Carvalho Chehab <[email protected]>Signed-off-by Jonathan Cavitt <[email protected]>--- drivers/gpu/drm/i915/gt/intel_ppgtt.c | 2 +- drivers/gpu/drm/i915/i915_vma.c | 14 +++++++++----- drivers/gpu/drm/i915/i915_vma.h | 3 ++- 3 files changed, 12 insertions(+), 7 deletions(-)drivers/gpu/drm/i915/gt/intel_ppgtt.c +1
-1@@ -211,7 +211,7 @@ void ppgtt_unbind_vma(struct i915_address_space *vm,return ; vm->clear_range (vm, vma_res->start, vma_res->vma_size); -vma_invalidate_tlb (vm, vma_res->tlb); +vma_invalidate_tlb (vm, vma_res->tlb, vma_res->start, vma_res->vma_size); }static unsigned long pd_count (u64 size,int shift)drivers/gpu/drm/i915/i915_vma.c +7
-4@@ -1339,7 +1339,8 @@ I915_SELFTEST_EXPORT int i915_vma_get_pages(struct i915_vma *vma)return err; } -void vma_invalidate_tlb (struct i915_address_space *vm,u32 *tlb) +void vma_invalidate_tlb(struct i915_address_space *vm, u32 *tlb + u64 start, u64 size) {struct intel_gt *gt;int id;@@ -1355,9 +1356,11 @@ void vma_invalidate_tlb(struct i915_address_space *vm, u32 *tlb)* the most recent TLB invalidation seqno, and if we have not yet * flushed the TLBs upon release, perform a full invalidation. */ -for_each_gt (gt, vm->i915, id) -WRITE_ONCE (tlb[id], -intel_gt_next_invalidate_tlb_full (gt)); +for_each_gt (gt, vm->i915, id) { +if (!intel_gt_invalidate_tlb_range (gt, start, size)) +WRITE_ONCE (tlb[id], +intel_gt_next_invalidate_tlb_full (gt)); + } }static void __vma_put_pages (struct i915_vma *vma,unsigned int count)
drivers/gpu/drm/i915/i915_vma.c struct dma_fence *__i915_vma_evict(struct i915_vma *vma, bool async) · line 2048
Quoted 6 lines
drivers/gpu/drm/i915/i915_vma.c +2
-1@@ -2041,7 +2044,8 @@ struct dma_fence *__i915_vma_evict(struct i915_vma *vma, bool async)dma_fence_put (unbind_fence); unbind_fence =NULL ; } -vma_invalidate_tlb (vma->vm, vma->obj->mm.tlb); +vma_invalidate_tlb (vma->vm, vma->obj->mm.tlb, + vma->node.start, vma->size);
The RFC looks like just what we needed so I'll drop an ack on the other patch series. Thanks for sending it out so quickly.
Regards,
Tvrtko
Quoted 17 lines
drivers/gpu/drm/i915/i915_vma.c +0
-0}/* drivers/gpu/drm/i915/i915_vma.h +2
-1@@ -260,7 +260,8 @@ bool i915_vma_misplaced(const struct i915_vma *vma, u64 size, u64 alignment, u64 flags);void __i915_vma_set_map_and_fenceable (struct i915_vma *vma);void i915_vma_revoke_mmap (struct i915_vma *vma); -void vma_invalidate_tlb (struct i915_address_space *vm,u32 *tlb); +void vma_invalidate_tlb (struct i915_address_space *vm,u32 *tlb, +u64 start,u64 size);struct dma_fence *__i915_vma_evict (struct i915_vma *vma,bool async);int __i915_vma_unbind (struct i915_vma *vma);int __must_check i915_vma_unbind (struct i915_vma *vma);