PPGTT dynamic page allocations
Subsystems (2)
cover PPGTT dynamic page allocations
This new version tries to remove as many unnecessary changes as possible from the previous RFC.
For GEN8, it has also been extended to work in logical ring submission (lrc) mode, as it will be the preferred mode of operation.
I also tried to update the lrc code at the same time the ppgtt refactoring occurred, leaving only one patch that is exclusively for lrc.
This list can be seen in 3 parts:
[01-10] Include code rework for PPGTT (all GENs).
[11-14] Adds page table allocation for GEN6/GEN7
[15-24] Enables dynamic allocation in GEN8. It is enabled for both legacy and execlist submission modes.
Ben Widawsky (23):
drm/i915: Add some extra guards in evict_vm drm/i915/trace: Fix offsets for 64b
drm/i915: Rename to GEN8_LEGACY_PDPES
drm/i915: Setup less PPGTT on failed pagedir drm/i915/gen8: Un-hardcode number of page directories drm/i915: Range clearing is PPGTT agnostic
drm/i915: page table abstractions
drm/i915: Complete page table structures
drm/i915: Create page table allocators
drm/i915: Track GEN6 page table usage
drm/i915: Extract context switch skip logic
drm/i915: Track page table reload need
drm/i915: Initialize all contexts
drm/i915: Finish gen6/7 dynamic page table allocation drm/i915/bdw: Use dynamic allocation idioms on free drm/i915/bdw: pagedirs rework allocation
drm/i915/bdw: pagetable allocation rework
drm/i915/bdw: Update pdp switch and point unused PDPs to scratch page drm/i915: num_pd_pages/num_pd_entries isn't useful drm/i915: Extract PPGTT param from pagedir alloc drm/i915/bdw: Split out mappings
drm/i915/bdw: begin bitmap tracking
drm/i915/bdw: Dynamic page table allocations
Michel Thierry (1):
drm/i915/bdw: Dynamic page table allocations in lrc mode
drivers/gpu/drm/i915/i915_debugfs.c | 7 +- drivers/gpu/drm/i915/i915_drv.h | 7 + drivers/gpu/drm/i915/i915_gem_context.c | 62 +- drivers/gpu/drm/i915/i915_gem_evict.c | 3 + drivers/gpu/drm/i915/i915_gem_execbuffer.c | 11 + drivers/gpu/drm/i915/i915_gem_gtt.c | 1224 ++++++++++++++++++++-------- drivers/gpu/drm/i915/i915_gem_gtt.h | 252 +++++- drivers/gpu/drm/i915/i915_trace.h | 124 ++- drivers/gpu/drm/i915/intel_lrc.c | 80 +- 9 files changed, 1378 insertions(+), 392 deletions(-)
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH 1/24 drm/i915: Add some extra guards in evict_vm
From: Ben Widawsky <[email protected]>
v2: Use WARN_ONs (Daniel)
--- drivers/gpu/drm/i915/i915_gem_evict.c | 3 +++ 1 file changed, 3 insertions(+)
drivers/gpu/drm/i915/i915_gem_evict.c +3 -0
@@ -214,6 +214,7 @@ int i915_gem_evict_vm(struct i915_address_space *vm, bool do_idle)struct i915_vma *vma, *next;int ret; +WARN_ON (!mutex_is_locked (&vm->dev->struct_mutex));trace_i915_gem_evict_vm (vm);if (do_idle) {
@@ -222,6 +223,8 @@ int i915_gem_evict_vm(struct i915_address_space *vm, bool do_idle)return ret;i915_gem_retire_requests (vm->dev); + +WARN_ON (!list_empty (&vm->active_list)); }list_for_each_entry_safe (vma, next, &vm->inactive_list, mm_list)
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH 2/24 drm/i915/trace: Fix offsets for 64b
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
--- drivers/gpu/drm/i915/i915_trace.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-)
drivers/gpu/drm/i915/i915_trace.h +4 -4
@@ -115,7 +115,7 @@ TRACE_EVENT(i915_vma_bind,TP_STRUCT__entry (__field (struct drm_i915_gem_object *,obj ) __field(struct i915_address_space *, vm) -__field (u32, offset) +__field (u64, offset)__field (u32, size)__field (unsigned, flags) ),
@@ -128,7 +128,7 @@ TRACE_EVENT(i915_vma_bind, __entry->flags = flags; ), -TP_printk ("obj=%p, offset=%08x size=%x%s vm=%p" , +TP_printk ("obj=%p, offset=%016llx size=%x%s vm=%p" , __entry->obj, __entry->offset, __entry->size, __entry->flags &PIN_MAPPABLE ?", mappable" :"" , __entry->vm)
@@ -141,7 +141,7 @@ TRACE_EVENT(i915_vma_unbind,TP_STRUCT__entry (__field (struct drm_i915_gem_object *,obj ) __field(struct i915_address_space *, vm) -__field (u32, offset) +__field (u64, offset)__field (u32, size) ),
@@ -152,7 +152,7 @@ TRACE_EVENT(i915_vma_unbind, __entry->size = vma->node.size; ), -TP_printk ("obj=%p, offset=%08x size=%x vm=%p" , +TP_printk ("obj=%p, offset=%016llx size=%x vm=%p" , __entry->obj, __entry->offset, __entry->size, __entry->vm) );
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH 3/24 drm/i915: Rename to GEN8_LEGACY_PDPES
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
In gen8, 32b PPGTT has always had one "pdp" (it doesn't actually have one, but it resembles having one). The #define was confusing as is, and using "PDPE" is a much better description.
sed -i 's/GEN8_LEGACY_PDPS/GEN8_LEGACY_PDPES/' drivers/gpu/drm/i915/*.[ch]
--- drivers/gpu/drm/i915/i915_gem_gtt.c | 6 +++--- drivers/gpu/drm/i915/i915_gem_gtt.h | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-)
drivers/gpu/drm/i915/i915_gem_gtt.c +3 -3
@@ -375,7 +375,7 @@ static void gen8_ppgtt_insert_entries(struct i915_address_space *vm, pt_vaddr =NULL ;for_each_sg_page (pages->sgl, &sg_iter, pages->nents,0 ) { -if (WARN_ON (pdpe >=GEN8_LEGACY_PDPS )) +if (WARN_ON (pdpe >=GEN8_LEGACY_PDPES ))break ;if (pt_vaddr ==NULL )
@@ -486,7 +486,7 @@ bail:static int gen8_ppgtt_allocate_page_tables (struct i915_hw_ppgtt *ppgtt,const int max_pdp) { -struct page **pt_pages[GEN8_LEGACY_PDPS ]; +struct page **pt_pages[GEN8_LEGACY_PDPES ];int i, ret;for (i =0 ; i < max_pdp; i++) {
@@ -537,7 +537,7 @@ static int gen8_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt,return -ENOMEM ; ppgtt->num_pd_pages =1 <<get_order (max_pdp <<PAGE_SHIFT ); -BUG_ON (ppgtt->num_pd_pages >GEN8_LEGACY_PDPS ); +BUG_ON (ppgtt->num_pd_pages >GEN8_LEGACY_PDPES );return 0 ; }
drivers/gpu/drm/i915/i915_gem_gtt.h +3 -3
@@ -88,7 +88,7 @@ typedef gen8_gtt_pte_t gen8_ppgtt_pde_t;#define GEN8_PDE_MASK 0x1ff#define GEN8_PTE_SHIFT 12#define GEN8_PTE_MASK 0x1ff -#define GEN8_LEGACY_PDPS 4 +#define GEN8_LEGACY_PDPES 4#define GEN8_PTES_PER_PAGE (PAGE_SIZE / sizeof(gen8_gtt_pte_t))#define GEN8_PDES_PER_PAGE (PAGE_SIZE / sizeof(gen8_ppgtt_pde_t))
@@ -273,12 +273,12 @@ struct i915_hw_ppgtt {unsigned num_pd_pages;/* gen8+ */ union {struct page **pt_pages; -struct page **gen8_pt_pages[GEN8_LEGACY_PDPS ]; +struct page **gen8_pt_pages[GEN8_LEGACY_PDPES ]; };struct page *pd_pages;union {uint32_t pd_offset; -dma_addr_t pd_dma_addr[GEN8_LEGACY_PDPS ]; +dma_addr_t pd_dma_addr[GEN8_LEGACY_PDPES ]; };union {dma_addr_t *pt_dma_addr;
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
Re: [PATCH 03/24] drm/i915: Rename to GEN8_LEGACY_PDPES
Quoted 7 lines · Michel Thierry
From: Ben Widawsky <[email protected]>
In gen8, 32b PPGTT has always had one "pdp" (it doesn't actually have one, but it resembles having one). The #define was confusing as is, and using "PDPE" is a much better description.
sed -i 's/GEN8_LEGACY_PDPS/GEN8_LEGACY_PDPES/' drivers/gpu/drm/i915/*.[ch]
Hm generally I've thought the abbreviations are pdp (for the page itself) and pde (for the entries within). I still have no idea what pdpe means ...
So either please explain that or pick one of the others. -Daniel
Quoted 74 lines
Signed-off-by Ben Widawsky <[email protected]>Signed-off-by Michel Thierry <[email protected]>--- drivers/gpu/drm/i915/i915_gem_gtt.c | 6 +++--- drivers/gpu/drm/i915/i915_gem_gtt.h | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-)drivers/gpu/drm/i915/i915_gem_gtt.c +3
-3@@ -375,7 +375,7 @@ static void gen8_ppgtt_insert_entries(struct i915_address_space *vm, pt_vaddr =NULL ;for_each_sg_page (pages->sgl, &sg_iter, pages->nents,0 ) { -if (WARN_ON (pdpe >=GEN8_LEGACY_PDPS )) +if (WARN_ON (pdpe >=GEN8_LEGACY_PDPES ))break ;if (pt_vaddr ==NULL )@@ -486,7 +486,7 @@ bail:static int gen8_ppgtt_allocate_page_tables (struct i915_hw_ppgtt *ppgtt,const int max_pdp) { -struct page **pt_pages[GEN8_LEGACY_PDPS ]; +struct page **pt_pages[GEN8_LEGACY_PDPES ];int i, ret;for (i =0 ; i < max_pdp; i++) {@@ -537,7 +537,7 @@ static int gen8_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt,return -ENOMEM ; ppgtt->num_pd_pages =1 <<get_order (max_pdp <<PAGE_SHIFT ); -BUG_ON (ppgtt->num_pd_pages >GEN8_LEGACY_PDPS ); +BUG_ON (ppgtt->num_pd_pages >GEN8_LEGACY_PDPES );return 0 ; }drivers/gpu/drm/i915/i915_gem_gtt.h +3
-3@@ -88,7 +88,7 @@ typedef gen8_gtt_pte_t gen8_ppgtt_pde_t;#define GEN8_PDE_MASK 0x1ff#define GEN8_PTE_SHIFT 12#define GEN8_PTE_MASK 0x1ff -#define GEN8_LEGACY_PDPS 4 +#define GEN8_LEGACY_PDPES 4#define GEN8_PTES_PER_PAGE (PAGE_SIZE / sizeof(gen8_gtt_pte_t))#define GEN8_PDES_PER_PAGE (PAGE_SIZE / sizeof(gen8_ppgtt_pde_t))@@ -273,12 +273,12 @@ struct i915_hw_ppgtt {unsigned num_pd_pages;/* gen8+ */ union {struct page **pt_pages; -struct page **gen8_pt_pages[GEN8_LEGACY_PDPS ]; +struct page **gen8_pt_pages[GEN8_LEGACY_PDPES ]; };struct page *pd_pages;union {uint32_t pd_offset; -dma_addr_t pd_dma_addr[GEN8_LEGACY_PDPS ]; +dma_addr_t pd_dma_addr[GEN8_LEGACY_PDPES ]; };union {dma_addr_t *pt_dma_addr;Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
Signature · 7 lines
Daniel Vetter Software Engineer, Intel Corporation +41 (0) 79 365 57 48 - http://blog.ffwll.ch _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
Re: [PATCH 03/24] drm/i915: Rename to GEN8_LEGACY_PDPES
Quoted 7 lines · Daniel Vetter
Quoted 6 lines · Michel Thierry
From: Ben Widawsky <[email protected]>
In gen8, 32b PPGTT has always had one "pdp" (it doesn't actually have one, but it resembles having one). The #define was confusing as is, and using "PDPE" is a much better description.
Quoted 6 lines
Quoted 1 line
sed -i 's/GEN8_LEGACY_PDPS/GEN8_LEGACY_PDPES/' drivers/gpu/drm/i915/*.[ch]
Hm generally I've thought the abbreviations are pdp (for the page itself) and pde (for the entries within). I still have no idea what pdpe means ...
So either please explain that or pick one of the others.
In case you fear the rebase pain of renaming this:
1. Export entire series as patches with git format-patch.
2. sed -e 's/PDPE/PDE/g' on all the patch files
3. Import the changed patches into a new fresh branch.'
That's all. Feels really crazy the first time you do it, but after having done this a lot with the internal branch when something random (function name or so) changed in upstream it's a fairly simple trick to pull off ;-)
Cheers, Daniel
Signature · 7 lines
Daniel Vetter Software Engineer, Intel Corporation +41 (0) 79 365 57 48 - http://blog.ffwll.ch _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
Re: [PATCH 03/24] drm/i915: Rename to GEN8_LEGACY_PDPES
Quoted 21 lines · Daniel Vetter
Quoted 7 lines · Daniel Vetter
Quoted 6 lines · Michel Thierry
From: Ben Widawsky <[email protected]>
In gen8, 32b PPGTT has always had one "pdp" (it doesn't actually have one, but it resembles having one). The #define was confusing as is, and using "PDPE" is a much better description.
Quoted 6 lines
Quoted 1 line
sed -i 's/GEN8_LEGACY_PDPS/GEN8_LEGACY_PDPES/' drivers/gpu/drm/i915/*.[ch]
Hm generally I've thought the abbreviations are pdp (for the page itself) and pde (for the entries within). I still have no idea what pdpe means ...
So either please explain that or pick one of the others.
In case you fear the rebase pain of renaming this:
1. Export entire series as patches with git format-patch.
2. sed -e 's/PDPE/PDE/g' on all the patch files
3. Import the changed patches into a new fresh branch.'
Quoted 5 lines
That's all. Feels really crazy the first time you do it, but after having done this a lot with the internal branch when something random (function name or so) changed in upstream it's a fairly simple trick to pull off ;-)
Cheers, Daniel
The specific #define is inconsistent with the naming used for other #defines and in the associated comments. Here's the relevant chunk of i915_gem_gtt.h:
/* GEN8 legacy style address is defined as a 3 level page table:
* 31:30 | 29:21 | 20:12 | 11:0 * PDPE | PDE | PTE | offset
* The difference as compared to normal x86 3 level page table is the
PDPEs are
* programmed via register.
*/
#define GEN8_PDPE_SHIFT 30 #define GEN8_PDPE_MASK 0x3 #define GEN8_PDE_SHIFT 21 #define GEN8_PDE_MASK 0x1ff #define GEN8_PTE_SHIFT 12 #define GEN8_PTE_MASK 0x1ff #define GEN8_LEGACY_PDPS 4 #define GEN8_PTES_PER_PAGE (PAGE_SIZE / sizeof(gen8_gtt_pte_t)) #define GEN8_PDES_PER_PAGE (PAGE_SIZE /
sizeof(gen8_ppgtt_pde_t))
So 'LEGACY_PDPS' is inconsistent with 'PDPE_SHIFT/MASK'.
PTE = Page Table Entry
PDE = Page Directory Entry
PDPE = Page Directory Pointer Entry
See http://www.pagetable.com/?p=308
.Dave.
_______________________________________________
Intel-gfx mailing list
[email protected]
http://lists.freedesktop.org/mailman/listinfo/intel-gfx
Re: [PATCH 03/24] drm/i915: Rename to GEN8_LEGACY_PDPES
Quoted 48 lines · Dave Gordon
Quoted 21 lines · Daniel Vetter
Quoted 7 lines · Daniel Vetter
Quoted 6 lines · Michel Thierry
From: Ben Widawsky <[email protected]>
In gen8, 32b PPGTT has always had one "pdp" (it doesn't actually have one, but it resembles having one). The #define was confusing as is, and using "PDPE" is a much better description.
Quoted 6 lines
Quoted 1 line
sed -i 's/GEN8_LEGACY_PDPS/GEN8_LEGACY_PDPES/' drivers/gpu/drm/i915/*.[ch]
Hm generally I've thought the abbreviations are pdp (for the page itself) and pde (for the entries within). I still have no idea what pdpe means ...
So either please explain that or pick one of the others.
In case you fear the rebase pain of renaming this:
1. Export entire series as patches with git format-patch.
2. sed -e 's/PDPE/PDE/g' on all the patch files
3. Import the changed patches into a new fresh branch.'Quoted 5 lines
That's all. Feels really crazy the first time you do it, but after having done this a lot with the internal branch when something random (function name or so) changed in upstream it's a fairly simple trick to pull off ;-)
Cheers, Daniel
The specific #define is inconsistent with the naming used for other #defines and in the associated comments. Here's the relevant chunk of
i915_gem_gtt.h:/* GEN8 legacy style address is defined as a 3 level page table:
* 31:30 | 29:21 | 20:12 | 11:0 * PDPE | PDE | PTE | offset* The difference as compared to normal x86 3 level page table is the
PDPEs are
* programmed via register.
*/#define GEN8_PDPE_SHIFT 30 #define GEN8_PDPE_MASK 0x3 #define GEN8_PDE_SHIFT 21 #define GEN8_PDE_MASK 0x1ff #define GEN8_PTE_SHIFT 12 #define GEN8_PTE_MASK 0x1ff #define GEN8_LEGACY_PDPS 4 #define GEN8_PTES_PER_PAGE (PAGE_SIZE / sizeof(gen8_gtt_pte_t)) #define GEN8_PDES_PER_PAGE (PAGE_SIZE /
Quoted 7 lines
sizeof(gen8_ppgtt_pde_t))So '
LEGACY_PDPS' is inconsistent with 'PDPE_SHIFT/MASK'.PTE = Page Table Entry
PDE = Page Directory Entry
PDPE = Page Directory Pointer Entry
Ok, I just wasn't aware of the intel nomeclatura for page directories. Imo if we want to rename we should consider the names established by the linux vm, mostly because svm and also because they're actually sane:
pgd -> pud -> pmd -> pt if you have 4 levels
pgd -> pmd -> pt if you have 3 levels and just
pgd -> pt for just 2
g = global
u = upper
m = middle
But I guess we can bikeshed this later on. Just please add the above expansion to the commit message and mention that this is what intel has picked in the IA PRM, too.
-Daniel
Signature · 7 lines
Daniel Vetter Software Engineer, Intel Corporation +41 (0) 79 365 57 48 - http://blog.ffwll.ch _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH 4/24 drm/i915: Setup less PPGTT on failed pagedir
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
The current code will both potentially print a WARN, and setup part of the PPGTT structure. Neither of these harm the current code, it is simply for clarity, and to perhaps prevent later bugs, or weird debug messages.
--- drivers/gpu/drm/i915/i915_gem_gtt.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-)
drivers/gpu/drm/i915/i915_gem_gtt.c +4 -1
@@ -1032,11 +1032,14 @@ alloc: goto alloc; } +if (ret) +return ret; +if (ppgtt->node.start < dev_priv->gtt.mappable_end)DRM_DEBUG ("Forced to use aperture for PDEs\n" ); ppgtt->num_pd_entries =GEN6_PPGTT_PD_ENTRIES ; -return ret; +return 0 ; }static int gen6_ppgtt_allocate_page_tables (struct i915_hw_ppgtt *ppgtt)
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH 5/24 drm/i915/gen8: Un-hardcode number of page directories
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
--- drivers/gpu/drm/i915/i915_gem_gtt.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
drivers/gpu/drm/i915/i915_gem_gtt.h +1 -1
@@ -282,7 +282,7 @@ struct i915_hw_ppgtt { };union {dma_addr_t *pt_dma_addr; -dma_addr_t *gen8_pt_dma_addr[4 ]; +dma_addr_t *gen8_pt_dma_addr[GEN8_LEGACY_PDPES ]; };struct drm_i915_file_private *file_priv;
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH 6/24 drm/i915: Range clearing is PPGTT agnostic
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
Therefore we can do it from our general init function. Eventually, I hope to have a lot more commonality like this. It won't arrive yet, but this was a nice easy one.
--- drivers/gpu/drm/i915/i915_gem_gtt.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-)
drivers/gpu/drm/i915/i915_gem_gtt.c +2 -4
@@ -672,8 +672,6 @@ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size) ppgtt->base.start =0 ; ppgtt->base.total = ppgtt->num_pd_entries *GEN8_PTES_PER_PAGE *PAGE_SIZE ; - ppgtt->base.clear_range (&ppgtt->base,0 , ppgtt->base.total, true); -DRM_DEBUG_DRIVER ("Allocated %d pages for page directories (%d wasted)\n" , ppgtt->num_pd_pages, ppgtt->num_pd_pages - max_pdp); DRM_DEBUG_DRIVER("Allocated %d pages for page tables (%lld wasted)\n",
@@ -1146,8 +1144,6 @@ static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt) ppgtt->pd_offset = ppgtt->node.start /PAGE_SIZE *sizeof (gen6_gtt_pte_t); - ppgtt->base.clear_range (&ppgtt->base,0 , ppgtt->base.total, true); -DRM_DEBUG_DRIVER ("Allocated pde space (%ldM) at GTT entry: %lx\n" , ppgtt->node.size >>20 , ppgtt->node.start /PAGE_SIZE );
@@ -1183,6 +1179,8 @@ int i915_ppgtt_init(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt)kref_init (&ppgtt->ref);drm_mm_init (&ppgtt->base.mm, ppgtt->base.start, ppgtt->base.total); + ppgtt->base.clear_range (&ppgtt->base,0 , + ppgtt->base.total, true);i915_init_vm (dev_priv, &ppgtt->base); }
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH 7/24 drm/i915: page table abstractions
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
When we move to dynamic page allocation, keeping pagedir and pagetabs as separate structures will help to break actions into simpler tasks.
To help transition the code nicely there is some wasted space in gen6/7. This will be ameliorated shortly.
v2: fixed mismatches after clean-up/rebase.
--- drivers/gpu/drm/i915/i915_gem_gtt.c | 177 ++++++++++++++++++------------------ drivers/gpu/drm/i915/i915_gem_gtt.h | 23 ++++- 2 files changed, 107 insertions(+), 93 deletions(-)
drivers/gpu/drm/i915/i915_gem_gtt.c +89 -88
@@ -334,7 +334,8 @@ static void gen8_ppgtt_clear_range(struct i915_address_space *vm, I915_CACHE_LLC, use_scratch);while (num_entries) { -struct page *page_table = ppgtt->gen8_pt_pages[pdpe][pde]; +struct i915_pagedir *pd = &ppgtt->pdp.pagedir[pdpe]; +struct page *page_table = pd->page_tables[pde].page; last_pte = pte + num_entries;if (last_pte >GEN8_PTES_PER_PAGE )
@@ -378,8 +379,12 @@ static void gen8_ppgtt_insert_entries(struct i915_address_space *vm,if (WARN_ON (pdpe >=GEN8_LEGACY_PDPES ))break ; -if (pt_vaddr ==NULL ) - pt_vaddr =kmap_atomic (ppgtt->gen8_pt_pages[pdpe][pde]); +if (pt_vaddr ==NULL ) { +struct i915_pagedir *pd = &ppgtt->pdp.pagedir[pdpe]; +struct page *page_table = pd->page_tables[pde].page; + + pt_vaddr =kmap_atomic (page_table); + } pt_vaddr[pte] = gen8_pte_encode(sg_page_iter_dma_address(&sg_iter),
@@ -403,29 +408,33 @@ static void gen8_ppgtt_insert_entries(struct i915_address_space *vm, } } -static void gen8_free_page_tables (struct page **pt_pages) +static void gen8_free_page_tables (struct i915_pagedir *pd) {int i; -if (pt_pages ==NULL ) +if (pd->page_tables ==NULL )return ;for (i =0 ; i <GEN8_PDES_PER_PAGE ; i++) -if (pt_pages[i]) -__free_pages (pt_pages[i],0 ); +if (pd->page_tables[i].page) +__free_page (pd->page_tables[i].page); } -static void gen8_ppgtt_free (const struct i915_hw_ppgtt *ppgtt) +static void gen8_free_page_directories (struct i915_pagedir *pd) +{ +kfree (pd->page_tables); +__free_page (pd->page); +} + +static void gen8_ppgtt_free (struct i915_hw_ppgtt *ppgtt) {int i;for (i =0 ; i < ppgtt->num_pd_pages; i++) { -gen8_free_page_tables (ppgtt->gen8_pt_pages[i]); -kfree (ppgtt->gen8_pt_pages[i]); +gen8_free_page_tables (&ppgtt->pdp.pagedir[i]); +gen8_free_page_directories (&ppgtt->pdp.pagedir[i]);kfree (ppgtt->gen8_pt_dma_addr[i]); } - -__free_pages (ppgtt->pd_pages,get_order (ppgtt->num_pd_pages <<PAGE_SHIFT )); }static void gen8_ppgtt_unmap_pages (struct i915_hw_ppgtt *ppgtt)
@@ -460,86 +469,75 @@ static void gen8_ppgtt_cleanup(struct i915_address_space *vm)gen8_ppgtt_free (ppgtt); } -static struct page **__gen8_alloc_page_tables (void ) +static int gen8_ppgtt_allocate_dma (struct i915_hw_ppgtt *ppgtt) { -struct page **pt_pages;int i; - pt_pages =kcalloc (GEN8_PDES_PER_PAGE ,sizeof (struct page *),GFP_KERNEL ); -if (!pt_pages) -return ERR_PTR (-ENOMEM ); - -for (i =0 ; i <GEN8_PDES_PER_PAGE ; i++) { - pt_pages[i] =alloc_page (GFP_KERNEL ); -if (!pt_pages[i]) - goto bail; +for (i =0 ; i < ppgtt->num_pd_pages; i++) { + ppgtt->gen8_pt_dma_addr[i] =kcalloc (GEN8_PDES_PER_PAGE , +sizeof (dma_addr_t), +GFP_KERNEL ); +if (!ppgtt->gen8_pt_dma_addr[i]) +return -ENOMEM ; } -return pt_pages; - -bail: -gen8_free_page_tables (pt_pages); -kfree (pt_pages); -return ERR_PTR (-ENOMEM ); +return 0 ; } -static int gen8_ppgtt_allocate_page_tables (struct i915_hw_ppgtt *ppgtt, -const int max_pdp) +static int gen8_ppgtt_allocate_page_tables (struct i915_hw_ppgtt *ppgtt) { -struct page **pt_pages[GEN8_LEGACY_PDPES ]; -int i, ret; +int i, j; -for (i =0 ; i < max_pdp; i++) { - pt_pages[i] =__gen8_alloc_page_tables (); -if (IS_ERR (pt_pages[i])) { - ret =PTR_ERR (pt_pages[i]); - goto unwind_out; +for (i =0 ; i < ppgtt->num_pd_pages; i++) { +for (j =0 ; j <GEN8_PDES_PER_PAGE ; j++) { +struct i915_pagetab *pt = &ppgtt->pdp.pagedir[i].page_tables[j]; + + pt->page =alloc_page (GFP_KERNEL | __GFP_ZERO); +if (!pt->page) + goto unwind_out; } } -/* NB: Avoid touching gen8_pt_pages until last to keep the allocation, -* "atomic" - for cleanup purposes. -*/ -for (i =0 ; i < max_pdp; i++) - ppgtt->gen8_pt_pages[i] = pt_pages[i]; -return 0 ; unwind_out: -while (i--) { -gen8_free_page_tables (pt_pages[i]); -kfree (pt_pages[i]); - } +while (i--) +gen8_free_page_tables (&ppgtt->pdp.pagedir[i]); -return ret; +return -ENOMEM ; } -static int gen8_ppgtt_allocate_dma (struct i915_hw_ppgtt *ppgtt) +static int gen8_ppgtt_allocate_page_directories (struct i915_hw_ppgtt *ppgtt, +const int max_pdp) {int i; -for (i =0 ; i < ppgtt->num_pd_pages; i++) { - ppgtt->gen8_pt_dma_addr[i] =kcalloc (GEN8_PDES_PER_PAGE , -sizeof (dma_addr_t), -GFP_KERNEL ); -if (!ppgtt->gen8_pt_dma_addr[i]) -return -ENOMEM ; - } +for (i =0 ; i < max_pdp; i++) { +struct i915_pagetab *pt; -return 0 ; -} + pt =kcalloc (GEN8_PDES_PER_PAGE ,sizeof (*pt),GFP_KERNEL ); +if (!pt) + goto unwind_out; -static int gen8_ppgtt_allocate_page_directories (struct i915_hw_ppgtt *ppgtt, -const int max_pdp) -{ - ppgtt->pd_pages =alloc_pages (GFP_KERNEL ,get_order (max_pdp <<PAGE_SHIFT )); -if (!ppgtt->pd_pages) -return -ENOMEM ; + ppgtt->pdp.pagedir[i].page =alloc_page (GFP_KERNEL ); +if (!ppgtt->pdp.pagedir[i].page) + goto unwind_out; + + ppgtt->pdp.pagedir[i].page_tables = pt; + } - ppgtt->num_pd_pages =1 <<get_order (max_pdp <<PAGE_SHIFT ); + ppgtt->num_pd_pages = max_pdp;BUG_ON (ppgtt->num_pd_pages >GEN8_LEGACY_PDPES );return 0 ; + +unwind_out: +while (i--) { +kfree (ppgtt->pdp.pagedir[i].page_tables); +__free_page (ppgtt->pdp.pagedir[i].page); + } + +return -ENOMEM ; } static int gen8_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt,
@@ -551,18 +549,19 @@ static int gen8_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt,if (ret)return ret; - ret =gen8_ppgtt_allocate_page_tables (ppgtt, max_pdp); -if (ret) { -__free_pages (ppgtt->pd_pages,get_order (max_pdp <<PAGE_SHIFT )); -return ret; - } + ret =gen8_ppgtt_allocate_page_tables (ppgtt); +if (ret) + goto err_out; ppgtt->num_pd_entries = max_pdp *GEN8_PDES_PER_PAGE ; ret =gen8_ppgtt_allocate_dma (ppgtt); -if (ret) -gen8_ppgtt_free (ppgtt); +if (!ret) +return ret; +/* TODO: Check this for all cases */ +err_out: +gen8_ppgtt_free (ppgtt);return ret; }
@@ -573,7 +572,7 @@ static int gen8_ppgtt_setup_page_directories(struct i915_hw_ppgtt *ppgtt,int ret; pd_addr =pci_map_page (ppgtt->base.dev->pdev, - &ppgtt->pd_pages[pd],0 , + ppgtt->pdp.pagedir[pd].page,0 ,PAGE_SIZE ,PCI_DMA_BIDIRECTIONAL ); ret =pci_dma_mapping_error (ppgtt->base.dev->pdev, pd_addr);
@@ -593,7 +592,7 @@ static int gen8_ppgtt_setup_page_tables(struct i915_hw_ppgtt *ppgtt,struct page *p;int ret; - p = ppgtt->gen8_pt_pages[pd][pt]; + p = ppgtt->pdp.pagedir[pd].page_tables[pt].page; pt_addr =pci_map_page (ppgtt->base.dev->pdev, p,0 ,PAGE_SIZE ,PCI_DMA_BIDIRECTIONAL ); ret =pci_dma_mapping_error (ppgtt->base.dev->pdev, pt_addr);
@@ -654,7 +653,7 @@ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size)*/ for (i =0 ; i < max_pdp; i++) {gen8_ppgtt_pde_t *pd_vaddr; - pd_vaddr =kmap_atomic (&ppgtt->pd_pages[i]); + pd_vaddr =kmap_atomic (ppgtt->pdp.pagedir[i].page);for (j =0 ; j <GEN8_PDES_PER_PAGE ; j++) {dma_addr_t addr = ppgtt->gen8_pt_dma_addr[i][j]; pd_vaddr[j] = gen8_pde_encode(ppgtt->base.dev, addr,
@@ -715,7 +714,7 @@ static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m) expected);seq_printf (m,"\tPDE: %x\n" , pd_entry); - pt_vaddr =kmap_atomic (ppgtt->pt_pages[pde]); + pt_vaddr =kmap_atomic (ppgtt->pd.page_tables[pde].page);for (pte =0 ; pte <I915_PPGTT_PT_ENTRIES ; pte+=4 ) {unsigned long va = (pde * PAGE_SIZE * I915_PPGTT_PT_ENTRIES) +
@@ -920,7 +919,7 @@ static void gen6_ppgtt_clear_range(struct i915_address_space *vm,if (last_pte >I915_PPGTT_PT_ENTRIES ) last_pte =I915_PPGTT_PT_ENTRIES ; - pt_vaddr =kmap_atomic (ppgtt->pt_pages[act_pt]); + pt_vaddr =kmap_atomic (ppgtt->pd.page_tables[act_pt].page);for (i = first_pte; i < last_pte; i++) pt_vaddr[i] = scratch_pte;
@@ -949,7 +948,7 @@ static void gen6_ppgtt_insert_entries(struct i915_address_space *vm, pt_vaddr =NULL ;for_each_sg_page (pages->sgl, &sg_iter, pages->nents,0 ) {if (pt_vaddr ==NULL ) - pt_vaddr =kmap_atomic (ppgtt->pt_pages[act_pt]); + pt_vaddr =kmap_atomic (ppgtt->pd.page_tables[act_pt].page); pt_vaddr[act_pte] = vm->pte_encode(sg_page_iter_dma_address(&sg_iter),
@@ -984,8 +983,8 @@ static void gen6_ppgtt_free(struct i915_hw_ppgtt *ppgtt)kfree (ppgtt->pt_dma_addr);for (i =0 ; i < ppgtt->num_pd_entries; i++) -__free_page (ppgtt->pt_pages[i]); -kfree (ppgtt->pt_pages); +__free_page (ppgtt->pd.page_tables[i].page); +kfree (ppgtt->pd.page_tables); }static void gen6_ppgtt_cleanup (struct i915_address_space *vm)
@@ -1042,22 +1041,22 @@ alloc:static int gen6_ppgtt_allocate_page_tables (struct i915_hw_ppgtt *ppgtt) { +struct i915_pagetab *pt;int i; - ppgtt->pt_pages =kcalloc (ppgtt->num_pd_entries,sizeof (struct page *), -GFP_KERNEL ); - -if (!ppgtt->pt_pages) + pt =kcalloc (ppgtt->num_pd_entries,sizeof (*pt),GFP_KERNEL ); +if (!pt)return -ENOMEM ;for (i =0 ; i < ppgtt->num_pd_entries; i++) { - ppgtt->pt_pages[i] =alloc_page (GFP_KERNEL ); -if (!ppgtt->pt_pages[i]) { + pt[i].page =alloc_page (GFP_KERNEL ); +if (!pt->page) {gen6_ppgtt_free (ppgtt);return -ENOMEM ; } } + ppgtt->pd.page_tables = pt;return 0 ; }
@@ -1092,9 +1091,11 @@ static int gen6_ppgtt_setup_page_tables(struct i915_hw_ppgtt *ppgtt)int i;for (i =0 ; i < ppgtt->num_pd_entries; i++) { +struct page *page;dma_addr_t pt_addr; - pt_addr =pci_map_page (dev->pdev, ppgtt->pt_pages[i],0 ,4096 , + page = ppgtt->pd.page_tables[i].page; + pt_addr =pci_map_page (dev->pdev, page,0 ,4096 ,PCI_DMA_BIDIRECTIONAL );if (pci_dma_mapping_error (dev->pdev, pt_addr)) {
@@ -1138,7 +1139,7 @@ static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt) ppgtt->base.insert_entries = gen6_ppgtt_insert_entries; ppgtt->base.cleanup = gen6_ppgtt_cleanup; ppgtt->base.start =0 ; - ppgtt->base.total = ppgtt->num_pd_entries *I915_PPGTT_PT_ENTRIES *PAGE_SIZE ; + ppgtt->base.total = ppgtt->num_pd_entries *I915_PPGTT_PT_ENTRIES *PAGE_SIZE ; ppgtt->debug_dump = gen6_dump_ppgtt; ppgtt->pd_offset =
drivers/gpu/drm/i915/i915_gem_gtt.h +18 -5
@@ -265,6 +265,20 @@ struct i915_gtt { unsigned long *mappable_end); }; +struct i915_pagetab { +struct page *page; +}; + +struct i915_pagedir { +struct page *page;/* NULL for GEN6-GEN7 */ +struct i915_pagetab *page_tables; +}; + +struct i915_pagedirpo { +/* struct page *page; */ +struct i915_pagedir pagedir[GEN8_LEGACY_PDPES ]; +}; +struct i915_hw_ppgtt {struct i915_address_space base;struct kref ref;
@@ -272,11 +286,6 @@ struct i915_hw_ppgtt {unsigned num_pd_entries;unsigned num_pd_pages;/* gen8+ */ union { -struct page **pt_pages; -struct page **gen8_pt_pages[GEN8_LEGACY_PDPES ]; - }; -struct page *pd_pages; -union {uint32_t pd_offset;dma_addr_t pd_dma_addr[GEN8_LEGACY_PDPES ]; };
@@ -284,6 +293,10 @@ struct i915_hw_ppgtt {dma_addr_t *pt_dma_addr;dma_addr_t *gen8_pt_dma_addr[GEN8_LEGACY_PDPES ]; }; +union { +struct i915_pagedirpo pdp; +struct i915_pagedir pd; + };struct drm_i915_file_private *file_priv;
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH 8/24 drm/i915: Complete page table structures
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
Move the remaining members over to the new page table structures.
This can be squashed with the previous commit if desire. The reasoning is the same as that patch. I simply felt it is easier to review if split.
v2: In lrc: s/ppgtt->pd_dma_addr[i]/ppgtt->pdp.pagedir[i].daddr/
--- drivers/gpu/drm/i915/i915_debugfs.c | 2 +- drivers/gpu/drm/i915/i915_gem_gtt.c | 85 +++++++++++++------------------------ drivers/gpu/drm/i915/i915_gem_gtt.h | 14 +++--- drivers/gpu/drm/i915/intel_lrc.c | 16 +++---- 4 files changed, 45 insertions(+), 72 deletions(-)
drivers/gpu/drm/i915/i915_debugfs.c +1 -1
@@ -2153,7 +2153,7 @@ static void gen6_ppgtt_info(struct seq_file *m, struct drm_device *dev)struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;seq_puts (m,"aliasing PPGTT:\n" ); -seq_printf (m,"pd gtt offset: 0x%08x\n" , ppgtt->pd_offset); +seq_printf (m,"pd gtt offset: 0x%08x\n" , ppgtt->pd.pd_offset); ppgtt->debug_dump (ppgtt, m); }
drivers/gpu/drm/i915/i915_gem_gtt.c +30 -55
220 lines not on this page · raw message
drivers/gpu/drm/i915/i915_gem_gtt.h +6 -8
@@ -267,10 +267,16 @@ struct i915_gtt {struct i915_pagetab {struct page *page; +dma_addr_t daddr; };struct i915_pagedir {struct page *page;/* NULL for GEN6-GEN7 */ +union { +uint32_t pd_offset; +dma_addr_t daddr; + }; +struct i915_pagetab *page_tables; };
@@ -286,14 +292,6 @@ struct i915_hw_ppgtt {unsigned num_pd_entries;unsigned num_pd_pages;/* gen8+ */ union { -uint32_t pd_offset; -dma_addr_t pd_dma_addr[GEN8_LEGACY_PDPES ]; - }; -union { -dma_addr_t *pt_dma_addr; -dma_addr_t *gen8_pt_dma_addr[GEN8_LEGACY_PDPES ]; - }; -union {struct i915_pagedirpo pdp;struct i915_pagedir pd; };
drivers/gpu/drm/i915/intel_lrc.c +8 -8
@@ -1731,14 +1731,14 @@ populate_lr_context(struct intel_context *ctx, struct drm_i915_gem_object *ctx_o reg_state[CTX_PDP1_LDW ] =GEN8_RING_PDP_LDW (ring,1 ); reg_state[CTX_PDP0_UDW ] =GEN8_RING_PDP_UDW (ring,0 ); reg_state[CTX_PDP0_LDW ] =GEN8_RING_PDP_LDW (ring,0 ); - reg_state[CTX_PDP3_UDW +1 ] =upper_32_bits (ppgtt->pd_dma_addr[3 ]); - reg_state[CTX_PDP3_LDW +1 ] =lower_32_bits (ppgtt->pd_dma_addr[3 ]); - reg_state[CTX_PDP2_UDW +1 ] =upper_32_bits (ppgtt->pd_dma_addr[2 ]); - reg_state[CTX_PDP2_LDW +1 ] =lower_32_bits (ppgtt->pd_dma_addr[2 ]); - reg_state[CTX_PDP1_UDW +1 ] =upper_32_bits (ppgtt->pd_dma_addr[1 ]); - reg_state[CTX_PDP1_LDW +1 ] =lower_32_bits (ppgtt->pd_dma_addr[1 ]); - reg_state[CTX_PDP0_UDW +1 ] =upper_32_bits (ppgtt->pd_dma_addr[0 ]); - reg_state[CTX_PDP0_LDW +1 ] =lower_32_bits (ppgtt->pd_dma_addr[0 ]); + reg_state[CTX_PDP3_UDW +1 ] =upper_32_bits (ppgtt->pdp.pagedir[3 ].daddr); + reg_state[CTX_PDP3_LDW +1 ] =lower_32_bits (ppgtt->pdp.pagedir[3 ].daddr); + reg_state[CTX_PDP2_UDW +1 ] =upper_32_bits (ppgtt->pdp.pagedir[2 ].daddr); + reg_state[CTX_PDP2_LDW +1 ] =lower_32_bits (ppgtt->pdp.pagedir[2 ].daddr); + reg_state[CTX_PDP1_UDW +1 ] =upper_32_bits (ppgtt->pdp.pagedir[1 ].daddr); + reg_state[CTX_PDP1_LDW +1 ] =lower_32_bits (ppgtt->pdp.pagedir[1 ].daddr); + reg_state[CTX_PDP0_UDW +1 ] =upper_32_bits (ppgtt->pdp.pagedir[0 ].daddr); + reg_state[CTX_PDP0_LDW +1 ] =lower_32_bits (ppgtt->pdp.pagedir[0 ].daddr);if (ring->id ==RCS ) { reg_state[CTX_LRI_HEADER_2 ] =MI_LOAD_REGISTER_IMM (1 ); reg_state[CTX_R_PWR_CLK_STATE ] =0x20c8 ;
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH 9/24 drm/i915: Create page table allocators
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
As we move toward dynamic page table allocation, it becomes much easier to manage our data structures if break do things less coarsely by breaking up all of our actions into individual tasks. This makes the code easier to write, read, and verify.
Aside from the dissection of the allocation functions, the patch statically allocates the page table structures without a page directory. This remains the same for all platforms,
The patch itself should not have much functional difference. The primary noticeable difference is the fact that page tables are no longer allocated, but rather statically declared as part of the page directory. This has non-zero overhead, but things gain non-trivial complexity as a result.
This patch exists for a few reasons:
1. Splitting out the functions allows easily combining GEN6 and GEN8 code. Page tables have no difference based on GEN8. As we'll see in a future patch when we add the DMA mappings to the allocations, it requires only one small change to make work, and error handling should just fall into place.
2. Unless we always want to allocate all page tables under a given PDE, we'll have to eventually break this up into an array of pointers (or pointer to pointer).
3. Having the discrete functions is easier to review, and understand. All allocations and frees now take place in just a couple of locations. Reviewing, and catching leaks should be easy.
4. Less important: the GFP flags are confined to one location, which makes playing around with such things trivial.
v2: Updated commit message to explain why this patch exists
v3: For lrc, s/pdp.pagedir[i].daddr/pdp.pagedir[i]->daddr/
--- drivers/gpu/drm/i915/i915_gem_gtt.c | 228 +++++++++++++++++++++++------------- drivers/gpu/drm/i915/i915_gem_gtt.h | 4 +- drivers/gpu/drm/i915/intel_lrc.c | 16 +-- 3 files changed, 155 insertions(+), 93 deletions(-)
drivers/gpu/drm/i915/i915_gem_gtt.c +145 -83
@@ -275,6 +275,102 @@ static gen6_gtt_pte_t iris_pte_encode(dma_addr_t addr,return pte; } +static void free_pt_single (struct i915_pagetab *pt) +{ +if (WARN_ON (!pt->page)) +return ; +__free_page (pt->page); +kfree (pt); +} + +static struct i915_pagetab *alloc_pt_single (void ) +{ +struct i915_pagetab *pt; + + pt =kzalloc (sizeof (*pt),GFP_KERNEL ); +if (!pt) +return ERR_PTR (-ENOMEM ); + + pt->page =alloc_page (GFP_KERNEL | __GFP_ZERO); +if (!pt->page) { +kfree (pt); +return ERR_PTR (-ENOMEM ); + } + +return pt; +} + +/** +* alloc_pt_range() - Allocate a multiple page tables +* @pd: The page directory which will have at least @count entries +* available to point to the allocated page tables. +* @pde: First page directory entry for which we are allocating. +* @count: Number of pages to allocate. +* +* Allocates multiple page table pages and sets the appropriate entries in the +* page table structure within the page directory. Function cleans up after +* itself on any failures. +* +* Return: 0 if allocation succeeded. +*/ +static int alloc_pt_range (struct i915_pagedir *pd,uint16_t pde,size_t count) +{ +int i, ret; + +/* 512 is the max page tables per pagedir on any platform. +* TODO: make WARN after patch series is done +*/ +BUG_ON (pde + count >GEN6_PPGTT_PD_ENTRIES ); + +for (i = pde; i < pde + count; i++) { +struct i915_pagetab *pt =alloc_pt_single (); +if (IS_ERR (pt)) { + ret =PTR_ERR (pt); + goto err_out; + } +WARN (pd->page_tables[i], +"Leaking page directory entry %d (%pa)\n" , + i, pd->page_tables[i]); + pd->page_tables[i] = pt; + } + +return 0 ; + +err_out: +while (i--) +free_pt_single (pd->page_tables[i]); +return ret; +} + +static void __free_pd_single (struct i915_pagedir *pd) +{ +__free_page (pd->page); +kfree (pd); +} + +#define free_pd_single (pd) do { \ + if ((pd)->page) { \ + __free_pd_single(pd); \ + } \ +} while (0) + +static struct i915_pagedir *alloc_pd_single (void ) +{ +struct i915_pagedir *pd; + + pd =kzalloc (sizeof (*pd),GFP_KERNEL ); +if (!pd) +return ERR_PTR (-ENOMEM ); + + pd->page =alloc_page (GFP_KERNEL | __GFP_ZERO); +if (!pd->page) { +kfree (pd); +return ERR_PTR (-ENOMEM ); + } + +return pd; +} +/* Broadwell Page Directory Pointer Descriptors */ static int gen8_write_pdp (struct intel_engine_cs *ring,unsigned entry,uint64_t val)
@@ -307,7 +403,7 @@ static int gen8_mm_switch(struct i915_hw_ppgtt *ppgtt,int used_pd = ppgtt->num_pd_entries /GEN8_PDES_PER_PAGE ;for (i = used_pd -1 ; i >=0 ; i--) { -dma_addr_t addr = ppgtt->pdp.pagedir[i].daddr; +dma_addr_t addr = ppgtt->pdp.pagedir[i]->daddr; ret =gen8_write_pdp (ring, i, addr);if (ret)return ret;
@@ -334,8 +430,9 @@ static void gen8_ppgtt_clear_range(struct i915_address_space *vm, I915_CACHE_LLC, use_scratch);while (num_entries) { -struct i915_pagedir *pd = &ppgtt->pdp.pagedir[pdpe]; -struct page *page_table = pd->page_tables[pde].page; +struct i915_pagedir *pd = ppgtt->pdp.pagedir[pdpe]; +struct i915_pagetab *pt = pd->page_tables[pde]; +struct page *page_table = pt->page; last_pte = pte + num_entries;if (last_pte >GEN8_PTES_PER_PAGE )
@@ -380,8 +477,9 @@ static void gen8_ppgtt_insert_entries(struct i915_address_space *vm,break ;if (pt_vaddr ==NULL ) { -struct i915_pagedir *pd = &ppgtt->pdp.pagedir[pdpe]; -struct page *page_table = pd->page_tables[pde].page; +struct i915_pagedir *pd = ppgtt->pdp.pagedir[pdpe]; +struct i915_pagetab *pt = pd->page_tables[pde]; +struct page *page_table = pt->page; pt_vaddr =kmap_atomic (page_table); }
@@ -412,18 +510,13 @@ static void gen8_free_page_tables(struct i915_pagedir *pd) {int i; -if (pd->page_tables ==NULL ) +if (!pd->page)return ; -for (i =0 ; i <GEN8_PDES_PER_PAGE ; i++) -if (pd->page_tables[i].page) -__free_page (pd->page_tables[i].page); -} - -static void gen8_free_page_directories (struct i915_pagedir *pd) -{ -kfree (pd->page_tables); -__free_page (pd->page); +for (i =0 ; i <GEN8_PDES_PER_PAGE ; i++) { +free_pt_single (pd->page_tables[i]); + pd->page_tables[i] =NULL ; + } }static void gen8_ppgtt_free (struct i915_hw_ppgtt *ppgtt)
@@ -431,8 +524,8 @@ static void gen8_ppgtt_free(struct i915_hw_ppgtt *ppgtt)int i;for (i =0 ; i < ppgtt->num_pd_pages; i++) { -gen8_free_page_tables (&ppgtt->pdp.pagedir[i]); -gen8_free_page_directories (&ppgtt->pdp.pagedir[i]); +gen8_free_page_tables (ppgtt->pdp.pagedir[i]); +free_pd_single (ppgtt->pdp.pagedir[i]); } }
@@ -444,14 +537,16 @@ static void gen8_ppgtt_unmap_pages(struct i915_hw_ppgtt *ppgtt)for (i =0 ; i < ppgtt->num_pd_pages; i++) {/* TODO: In the future we'll support sparse mappings, so this * will have to change. */ -if (!ppgtt->pdp.pagedir[i].daddr) +if (!ppgtt->pdp.pagedir[i]->daddr)continue ; -pci_unmap_page (hwdev, ppgtt->pdp.pagedir[i].daddr,PAGE_SIZE , +pci_unmap_page (hwdev, ppgtt->pdp.pagedir[i]->daddr,PAGE_SIZE ,PCI_DMA_BIDIRECTIONAL );for (j =0 ; j <GEN8_PDES_PER_PAGE ; j++) { -dma_addr_t addr = ppgtt->pdp.pagedir[i].page_tables[j].daddr; +struct i915_pagedir *pd = ppgtt->pdp.pagedir[i]; +struct i915_pagetab *pt = pd->page_tables[j]; +dma_addr_t addr = pt->daddr;if (addr)pci_unmap_page (hwdev, addr,PAGE_SIZE ,PCI_DMA_BIDIRECTIONAL );
@@ -470,25 +565,20 @@ static void gen8_ppgtt_cleanup(struct i915_address_space *vm)static int gen8_ppgtt_allocate_page_tables (struct i915_hw_ppgtt *ppgtt) { -int i, j; +int i, ret;for (i =0 ; i < ppgtt->num_pd_pages; i++) { -struct i915_pagedir *pd = &ppgtt->pdp.pagedir[i]; -for (j =0 ; j <GEN8_PDES_PER_PAGE ; j++) { -struct i915_pagetab *pt = &pd->page_tables[j]; - - pt->page =alloc_page (GFP_KERNEL | __GFP_ZERO); -if (!pt->page) - goto unwind_out; - - } + ret =alloc_pt_range (ppgtt->pdp.pagedir[i], +0 ,GEN8_PDES_PER_PAGE ); +if (ret) + goto unwind_out; }return 0 ; unwind_out:while (i--) -gen8_free_page_tables (&ppgtt->pdp.pagedir[i]); +gen8_free_page_tables (ppgtt->pdp.pagedir[i]);return -ENOMEM ; }
@@ -499,17 +589,9 @@ static int gen8_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt,int i;for (i =0 ; i < max_pdp; i++) { -struct i915_pagetab *pt; - - pt =kcalloc (GEN8_PDES_PER_PAGE ,sizeof (*pt),GFP_KERNEL ); -if (!pt) + ppgtt->pdp.pagedir[i] =alloc_pd_single (); +if (IS_ERR (ppgtt->pdp.pagedir[i])) goto unwind_out; - - ppgtt->pdp.pagedir[i].page =alloc_page (GFP_KERNEL ); -if (!ppgtt->pdp.pagedir[i].page) - goto unwind_out; - - ppgtt->pdp.pagedir[i].page_tables = pt; } ppgtt->num_pd_pages = max_pdp;
@@ -518,10 +600,8 @@ static int gen8_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt,return 0 ; unwind_out: -while (i--) { -kfree (ppgtt->pdp.pagedir[i].page_tables); -__free_page (ppgtt->pdp.pagedir[i].page); - } +while (i--) +free_pd_single (ppgtt->pdp.pagedir[i]);return -ENOMEM ; }
@@ -556,14 +636,14 @@ static int gen8_ppgtt_setup_page_directories(struct i915_hw_ppgtt *ppgtt,int ret; pd_addr =pci_map_page (ppgtt->base.dev->pdev, - ppgtt->pdp.pagedir[pd].page,0 , + ppgtt->pdp.pagedir[pd]->page,0 ,PAGE_SIZE ,PCI_DMA_BIDIRECTIONAL ); ret =pci_dma_mapping_error (ppgtt->base.dev->pdev, pd_addr);if (ret)return ret; - ppgtt->pdp.pagedir[pd].daddr = pd_addr; + ppgtt->pdp.pagedir[pd]->daddr = pd_addr;return 0 ; }
@@ -573,8 +653,8 @@ static int gen8_ppgtt_setup_page_tables(struct i915_hw_ppgtt *ppgtt, const int pt) {dma_addr_t pt_addr; -struct i915_pagedir *pdir = &ppgtt->pdp.pagedir[pd]; -struct i915_pagetab *ptab = &pdir->page_tables[pt]; +struct i915_pagedir *pdir = ppgtt->pdp.pagedir[pd]; +struct i915_pagetab *ptab = pdir->page_tables[pt];struct page *p = ptab->page;int ret;
@@ -637,10 +717,12 @@ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size)* will never need to touch the PDEs again. */ for (i =0 ; i < max_pdp; i++) { +struct i915_pagedir *pd = ppgtt->pdp.pagedir[i];gen8_ppgtt_pde_t *pd_vaddr; - pd_vaddr =kmap_atomic (ppgtt->pdp.pagedir[i].page); + pd_vaddr =kmap_atomic (ppgtt->pdp.pagedir[i]->page);for (j =0 ; j <GEN8_PDES_PER_PAGE ; j++) { -dma_addr_t addr = ppgtt->pdp.pagedir[i].page_tables[j].daddr; +struct i915_pagetab *pt = pd->page_tables[j]; +dma_addr_t addr = pt->daddr; pd_vaddr[j] =gen8_pde_encode (ppgtt->base.dev, addr,I915_CACHE_LLC ); }
@@ -689,7 +771,7 @@ static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m)for (pde =0 ; pde < ppgtt->num_pd_entries; pde++) {u32 expected;gen6_gtt_pte_t *pt_vaddr; -dma_addr_t pt_addr = ppgtt->pd.page_tables[pde].daddr; +dma_addr_t pt_addr = ppgtt->pd.page_tables[pde]->daddr; pd_entry =readl (pd_addr + pde); expected = (GEN6_PDE_ADDR_ENCODE (pt_addr) |GEN6_PDE_VALID );
@@ -700,7 +782,7 @@ static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m) expected);seq_printf (m,"\tPDE: %x\n" , pd_entry); - pt_vaddr =kmap_atomic (ppgtt->pd.page_tables[pde].page); + pt_vaddr =kmap_atomic (ppgtt->pd.page_tables[pde]->page);for (pte =0 ; pte <I915_PPGTT_PT_ENTRIES ; pte+=4 ) {unsigned long va = (pde * PAGE_SIZE * I915_PPGTT_PT_ENTRIES) +
@@ -739,7 +821,7 @@ static void gen6_write_pdes(struct i915_hw_ppgtt *ppgtt)for (i =0 ; i < ppgtt->num_pd_entries; i++) {dma_addr_t pt_addr; - pt_addr = ppgtt->pd.page_tables[i].daddr; + pt_addr = ppgtt->pd.page_tables[i]->daddr; pd_entry =GEN6_PDE_ADDR_ENCODE (pt_addr); pd_entry |=GEN6_PDE_VALID ;
@@ -905,7 +987,7 @@ static void gen6_ppgtt_clear_range(struct i915_address_space *vm,if (last_pte >I915_PPGTT_PT_ENTRIES ) last_pte =I915_PPGTT_PT_ENTRIES ; - pt_vaddr =kmap_atomic (ppgtt->pd.page_tables[act_pt].page); + pt_vaddr =kmap_atomic (ppgtt->pd.page_tables[act_pt]->page);for (i = first_pte; i < last_pte; i++) pt_vaddr[i] = scratch_pte;
@@ -934,7 +1016,7 @@ static void gen6_ppgtt_insert_entries(struct i915_address_space *vm, pt_vaddr =NULL ;for_each_sg_page (pages->sgl, &sg_iter, pages->nents,0 ) {if (pt_vaddr ==NULL ) - pt_vaddr =kmap_atomic (ppgtt->pd.page_tables[act_pt].page); + pt_vaddr =kmap_atomic (ppgtt->pd.page_tables[act_pt]->page); pt_vaddr[act_pte] = vm->pte_encode(sg_page_iter_dma_address(&sg_iter),
@@ -957,7 +1039,7 @@ static void gen6_ppgtt_unmap_pages(struct i915_hw_ppgtt *ppgtt)for (i =0 ; i < ppgtt->num_pd_entries; i++)pci_unmap_page (ppgtt->base.dev->pdev, - ppgtt->pd.page_tables[i].daddr, + ppgtt->pd.page_tables[i]->daddr,4096 ,PCI_DMA_BIDIRECTIONAL ); }
@@ -966,8 +1048,9 @@ static void gen6_ppgtt_free(struct i915_hw_ppgtt *ppgtt)int i;for (i =0 ; i < ppgtt->num_pd_entries; i++) -__free_page (ppgtt->pd.page_tables[i].page); -kfree (ppgtt->pd.page_tables); +free_pt_single (ppgtt->pd.page_tables[i]); + +free_pd_single (&ppgtt->pd); }static void gen6_ppgtt_cleanup (struct i915_address_space *vm)
@@ -1022,27 +1105,6 @@ alloc:return 0 ; } -static int gen6_ppgtt_allocate_page_tables (struct i915_hw_ppgtt *ppgtt) -{ -struct i915_pagetab *pt; -int i; - - pt =kcalloc (ppgtt->num_pd_entries,sizeof (*pt),GFP_KERNEL ); -if (!pt) -return -ENOMEM ; - -for (i =0 ; i < ppgtt->num_pd_entries; i++) { - pt[i].page =alloc_page (GFP_KERNEL ); -if (!pt->page) { -gen6_ppgtt_free (ppgtt); -return -ENOMEM ; - } - } - - ppgtt->pd.page_tables = pt; -return 0 ; -} -static int gen6_ppgtt_alloc (struct i915_hw_ppgtt *ppgtt) {int ret;
@@ -1051,7 +1113,7 @@ static int gen6_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt)if (ret)return ret; - ret =gen6_ppgtt_allocate_page_tables (ppgtt); + ret =alloc_pt_range (&ppgtt->pd,0 , ppgtt->num_pd_entries);if (ret) {drm_mm_remove_node (&ppgtt->node);return ret;
@@ -1069,7 +1131,7 @@ static int gen6_ppgtt_setup_page_tables(struct i915_hw_ppgtt *ppgtt)struct page *page;dma_addr_t pt_addr; - page = ppgtt->pd.page_tables[i].page; + page = ppgtt->pd.page_tables[i]->page; pt_addr =pci_map_page (dev->pdev, page,0 ,4096 ,PCI_DMA_BIDIRECTIONAL );
@@ -1078,7 +1140,7 @@ static int gen6_ppgtt_setup_page_tables(struct i915_hw_ppgtt *ppgtt)return -EIO ; } - ppgtt->pd.page_tables[i].daddr = pt_addr; + ppgtt->pd.page_tables[i]->daddr = pt_addr; }return 0 ;
drivers/gpu/drm/i915/i915_gem_gtt.h +2 -2
@@ -277,12 +277,12 @@ struct i915_pagedir {dma_addr_t daddr; }; -struct i915_pagetab *page_tables; +struct i915_pagetab *page_tables[GEN6_PPGTT_PD_ENTRIES ];/* PDEs */ };struct i915_pagedirpo {/* struct page *page; */ -struct i915_pagedir pagedir[GEN8_LEGACY_PDPES ]; +struct i915_pagedir *pagedir[GEN8_LEGACY_PDPES ]; };struct i915_hw_ppgtt {
drivers/gpu/drm/i915/intel_lrc.c +8 -8
@@ -1731,14 +1731,14 @@ populate_lr_context(struct intel_context *ctx, struct drm_i915_gem_object *ctx_o reg_state[CTX_PDP1_LDW ] =GEN8_RING_PDP_LDW (ring,1 ); reg_state[CTX_PDP0_UDW ] =GEN8_RING_PDP_UDW (ring,0 ); reg_state[CTX_PDP0_LDW ] =GEN8_RING_PDP_LDW (ring,0 ); - reg_state[CTX_PDP3_UDW +1 ] =upper_32_bits (ppgtt->pdp.pagedir[3 ].daddr); - reg_state[CTX_PDP3_LDW +1 ] =lower_32_bits (ppgtt->pdp.pagedir[3 ].daddr); - reg_state[CTX_PDP2_UDW +1 ] =upper_32_bits (ppgtt->pdp.pagedir[2 ].daddr); - reg_state[CTX_PDP2_LDW +1 ] =lower_32_bits (ppgtt->pdp.pagedir[2 ].daddr); - reg_state[CTX_PDP1_UDW +1 ] =upper_32_bits (ppgtt->pdp.pagedir[1 ].daddr); - reg_state[CTX_PDP1_LDW +1 ] =lower_32_bits (ppgtt->pdp.pagedir[1 ].daddr); - reg_state[CTX_PDP0_UDW +1 ] =upper_32_bits (ppgtt->pdp.pagedir[0 ].daddr); - reg_state[CTX_PDP0_LDW +1 ] =lower_32_bits (ppgtt->pdp.pagedir[0 ].daddr); + reg_state[CTX_PDP3_UDW +1 ] =upper_32_bits (ppgtt->pdp.pagedir[3 ]->daddr); + reg_state[CTX_PDP3_LDW +1 ] =lower_32_bits (ppgtt->pdp.pagedir[3 ]->daddr); + reg_state[CTX_PDP2_UDW +1 ] =upper_32_bits (ppgtt->pdp.pagedir[2 ]->daddr); + reg_state[CTX_PDP2_LDW +1 ] =lower_32_bits (ppgtt->pdp.pagedir[2 ]->daddr); + reg_state[CTX_PDP1_UDW +1 ] =upper_32_bits (ppgtt->pdp.pagedir[1 ]->daddr); + reg_state[CTX_PDP1_LDW +1 ] =lower_32_bits (ppgtt->pdp.pagedir[1 ]->daddr); + reg_state[CTX_PDP0_UDW +1 ] =upper_32_bits (ppgtt->pdp.pagedir[0 ]->daddr); + reg_state[CTX_PDP0_LDW +1 ] =lower_32_bits (ppgtt->pdp.pagedir[0 ]->daddr);if (ring->id ==RCS ) { reg_state[CTX_LRI_HEADER_2 ] =MI_LOAD_REGISTER_IMM (1 ); reg_state[CTX_R_PWR_CLK_STATE ] =0x20c8 ;
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH 10/24 drm/i915: Track GEN6 page table usage
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
Instead of implementing the full tracking + dynamic allocation, this patch does a bit less than half of the work, by tracking and warning on unexpected conditions. The tracking itself follows which PTEs within a page table are currently being used for objects. The next patch will modify this to actually allocate the page tables only when necessary.
With the current patch there isn't much in the way of making a gen agnostic range allocation function. However, in the next patch we'll add more specificity which makes having separate functions a bit easier to manage.
One important change introduced here is that DMA mappings are created/destroyed at the same page directories/tables are allocated/deallocated.
Notice that aliasing PPGTT is not managed here. The patch which actually begins dynamic allocation/teardown explains the reasoning for this.
v2: s/pdp.pagedir/pdp.pagedirs
Make a scratch page allocation helper
v3: Rebase and expand commit message.
--- drivers/gpu/drm/i915/i915_gem_gtt.c | 300 ++++++++++++++++++++++++++---------- drivers/gpu/drm/i915/i915_gem_gtt.h | 151 +++++++++++++----- 2 files changed, 333 insertions(+), 118 deletions(-)
drivers/gpu/drm/i915/i915_gem_gtt.c +217 -83
@@ -138,10 +138,9 @@ static int sanitize_enable_ppgtt(struct drm_device *dev, int enable_ppgtt)return has_aliasing_ppgtt ?1 :0 ; } - -static void ppgtt_bind_vma (struct i915_vma *vma, -enum i915_cache_level cache_level, -u32 flags); +static int ppgtt_bind_vma (struct i915_vma *vma, +enum i915_cache_level cache_level, +u32 flags);static void ppgtt_unbind_vma (struct i915_vma *vma); static inline gen8_gtt_pte_t gen8_pte_encode(dma_addr_t addr,
@@ -275,27 +274,99 @@ static gen6_gtt_pte_t iris_pte_encode(dma_addr_t addr,return pte; } -static void free_pt_single (struct i915_pagetab *pt) -{ +#define i915_dma_unmap_single (px, dev) do { \ + pci_unmap_page((dev)->pdev, (px)->daddr, 4096, PCI_DMA_BIDIRECTIONAL); \ +} while (0); + +/** +* i915_dma_map_px_single() - Create a dma mapping for a page table/dir/etc. +* @px: Page table/dir/etc to get a DMA map for +* @dev: drm device +* +* Page table allocations are unified across all gens. They always require a +* single 4k allocation, as well as a DMA mapping. If we keep the structs +* symmetric here, the simple macro covers us for every page table type. +* +* Return: 0 if success. +*/ +#define i915_dma_map_px_single (px, dev) \ + pci_dma_mapping_error((dev)->pdev, \ + (px)->daddr = pci_map_page((dev)->pdev, \ + (px)->page, 0, 4096, \ + PCI_DMA_BIDIRECTIONAL)) + +static void __free_pt_single (struct i915_pagetab *pt,struct drm_device *dev, +int scratch) +{ +if (WARN (scratch ^ pt->scratch, +"Tried to free scratch = %d. Is scratch = %d\n" , + scratch, pt->scratch)) +return ; +if (WARN_ON (!pt->page))return ; + +if (!scratch) { +const size_t count =INTEL_INFO (dev)->gen >=8 ? +GEN8_PTES_PER_PAGE :I915_PPGTT_PT_ENTRIES ; +WARN (!bitmap_empty (pt->used_ptes, count), +"Free page table with %d used pages\n" , +bitmap_weight (pt->used_ptes, count)); + } + +i915_dma_unmap_single (pt, dev);__free_page (pt->page); +kfree (pt->used_ptes);kfree (pt); } -static struct i915_pagetab *alloc_pt_single (void ) +#define free_pt_single (pt, dev) \ + __free_pt_single(pt, dev, false) +#define free_pt_scratch (pt, dev) \ + __free_pt_single(pt, dev, true) + +static struct i915_pagetab *alloc_pt_single (struct drm_device *dev) {struct i915_pagetab *pt; +const size_t count =INTEL_INFO (dev)->gen >=8 ? +GEN8_PTES_PER_PAGE :I915_PPGTT_PT_ENTRIES ; +int ret = -ENOMEM ; pt =kzalloc (sizeof (*pt),GFP_KERNEL );if (!pt)return ERR_PTR (-ENOMEM ); + pt->used_ptes =kcalloc (BITS_TO_LONGS (count),sizeof (*pt->used_ptes), +GFP_KERNEL ); + +if (!pt->used_ptes) + goto fail_bitmap; + pt->page =alloc_page (GFP_KERNEL | __GFP_ZERO); -if (!pt->page) { -kfree (pt); -return ERR_PTR (-ENOMEM ); - } +if (!pt->page) + goto fail_page; + + ret =i915_dma_map_px_single (pt, dev); +if (ret) + goto fail_dma; + +return pt; + +fail_dma: +__free_page (pt->page); +fail_page: +kfree (pt->used_ptes); +fail_bitmap: +kfree (pt); + +return ERR_PTR (ret); +} + +static inline struct i915_pagetab *alloc_pt_scratch (struct drm_device *dev) +{ +struct i915_pagetab *pt =alloc_pt_single (dev); +if (!IS_ERR (pt)) + pt->scratch =1 ;return pt; }
@@ -313,7 +384,9 @@ static struct i915_pagetab *alloc_pt_single(void)* * Return: 0 if allocation succeeded. */ -static int alloc_pt_range (struct i915_pagedir *pd,uint16_t pde,size_t count) +static int alloc_pt_range (struct i915_pagedir *pd,uint16_t pde,size_t count, +struct drm_device *dev) + {int i, ret;
@@ -323,7 +396,7 @@ static int alloc_pt_range(struct i915_pagedir *pd, uint16_t pde, size_t count)BUG_ON (pde + count >GEN6_PPGTT_PD_ENTRIES );for (i = pde; i < pde + count; i++) { -struct i915_pagetab *pt =alloc_pt_single (); +struct i915_pagetab *pt =alloc_pt_single (dev);if (IS_ERR (pt)) { ret =PTR_ERR (pt); goto err_out;
@@ -338,7 +411,7 @@ static int alloc_pt_range(struct i915_pagedir *pd, uint16_t pde, size_t count) err_out:while (i--) -free_pt_single (pd->page_tables[i]); +free_pt_single (pd->page_tables[i], dev);return ret; }
@@ -506,7 +579,7 @@ static void gen8_ppgtt_insert_entries(struct i915_address_space *vm, } } -static void gen8_free_page_tables (struct i915_pagedir *pd) +static void gen8_free_page_tables (struct i915_pagedir *pd,struct drm_device *dev) {int i;
@@ -514,7 +587,7 @@ static void gen8_free_page_tables(struct i915_pagedir *pd)return ;for (i =0 ; i <GEN8_PDES_PER_PAGE ; i++) { -free_pt_single (pd->page_tables[i]); +free_pt_single (pd->page_tables[i], dev); pd->page_tables[i] =NULL ; } }
@@ -524,7 +597,7 @@ static void gen8_ppgtt_free(struct i915_hw_ppgtt *ppgtt)int i;for (i =0 ; i < ppgtt->num_pd_pages; i++) { -gen8_free_page_tables (ppgtt->pdp.pagedir[i]); +gen8_free_page_tables (ppgtt->pdp.pagedir[i], ppgtt->base.dev);free_pd_single (ppgtt->pdp.pagedir[i]); } }
@@ -569,7 +642,7 @@ static int gen8_ppgtt_allocate_page_tables(struct i915_hw_ppgtt *ppgtt)for (i =0 ; i < ppgtt->num_pd_pages; i++) { ret =alloc_pt_range (ppgtt->pdp.pagedir[i], -0 ,GEN8_PDES_PER_PAGE ); +0 ,GEN8_PDES_PER_PAGE , ppgtt->base.dev);if (ret) goto unwind_out; }
@@ -578,7 +651,7 @@ static int gen8_ppgtt_allocate_page_tables(struct i915_hw_ppgtt *ppgtt) unwind_out:while (i--) -gen8_free_page_tables (ppgtt->pdp.pagedir[i]); +gen8_free_page_tables (ppgtt->pdp.pagedir[i], ppgtt->base.dev);return -ENOMEM ; }
@@ -808,26 +881,36 @@ static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m) } } -static void gen6_write_pdes (struct i915_hw_ppgtt *ppgtt) +/* Write pde (index) from the page directory @pd to the page table @pt */ +static void gen6_write_pdes (struct i915_pagedir *pd, +const int pde,struct i915_pagetab *pt) { -struct drm_i915_private *dev_priv = ppgtt->base.dev->dev_private; -gen6_gtt_pte_t __iomem *pd_addr; -uint32_t pd_entry; -int i; +struct i915_hw_ppgtt *ppgtt = + container_of(pd, struct i915_hw_ppgtt, pd); +u32 pd_entry; -WARN_ON (ppgtt->pd.pd_offset &0x3f ); - pd_addr = (gen6_gtt_pte_t __iomem *)dev_priv->gtt.gsm + - ppgtt->pd.pd_offset /sizeof (gen6_gtt_pte_t); -for (i =0 ; i < ppgtt->num_pd_entries; i++) { -dma_addr_t pt_addr; + pd_entry =GEN6_PDE_ADDR_ENCODE (pt->daddr); + pd_entry |=GEN6_PDE_VALID ; - pt_addr = ppgtt->pd.page_tables[i]->daddr; - pd_entry =GEN6_PDE_ADDR_ENCODE (pt_addr); - pd_entry |=GEN6_PDE_VALID ; +writel (pd_entry, ppgtt->pd_addr + pde); -writel (pd_entry, pd_addr + i); - } -readl (pd_addr); +/* XXX: Caller needs to make sure the write completes if necessary */ +} + +/* Write all the page tables found in the ppgtt structure to incrementing page +* directories. */ +static void gen6_write_page_range (struct drm_i915_private *dev_priv, +struct i915_pagedir *pd,uint32_t start,uint32_t length) +{ +struct i915_pagetab *pt; +uint32_t pde, temp; + +gen6_for_each_pde (pt, pd, start, length, temp, pde) +gen6_write_pdes (pd, pde, pt); + +/* Make sure write is complete before other code can use this page +* table. Also require for WC mapped PTEs */ +readl (dev_priv->gtt.gsm); }static uint32_t get_pd_offset (struct i915_hw_ppgtt *ppgtt)
@@ -1043,13 +1126,59 @@ static void gen6_ppgtt_unmap_pages(struct i915_hw_ppgtt *ppgtt) 4096, PCI_DMA_BIDIRECTIONAL); } +static int gen6_alloc_va_range (struct i915_address_space *vm, +uint64_t start,uint64_t length) +{ +struct i915_hw_ppgtt *ppgtt = + container_of(vm, struct i915_hw_ppgtt, base); +struct i915_pagetab *pt; +uint32_t pde, temp; + +gen6_for_each_pde (pt, &ppgtt->pd, start, length, temp, pde) { +int j; + +DECLARE_BITMAP (tmp_bitmap,I915_PPGTT_PT_ENTRIES ); +bitmap_zero (tmp_bitmap,I915_PPGTT_PT_ENTRIES ); +bitmap_set (tmp_bitmap,gen6_pte_index (start), +gen6_pte_count (start, length)); + +/* TODO: To be done in the next patch. Map the page/insert +* entries here */ +for_each_set_bit (j, tmp_bitmap,I915_PPGTT_PT_ENTRIES ) { +if (test_bit (j, pt->used_ptes)) { +/* Check that we're changing cache levels */ + } + } + +bitmap_or (pt->used_ptes, pt->used_ptes, tmp_bitmap, +I915_PPGTT_PT_ENTRIES ); + } + +return 0 ; +} + +static void gen6_teardown_va_range (struct i915_address_space *vm, +uint64_t start,uint64_t length) +{ +struct i915_hw_ppgtt *ppgtt = + container_of(vm, struct i915_hw_ppgtt, base); +struct i915_pagetab *pt; +uint32_t pde, temp; + +gen6_for_each_pde (pt, &ppgtt->pd, start, length, temp, pde) { +bitmap_clear (pt->used_ptes,gen6_pte_index (start), +gen6_pte_count (start, length)); + } +} +static void gen6_ppgtt_free (struct i915_hw_ppgtt *ppgtt) {int i;for (i =0 ; i < ppgtt->num_pd_entries; i++) -free_pt_single (ppgtt->pd.page_tables[i]); +free_pt_single (ppgtt->pd.page_tables[i], ppgtt->base.dev); +free_pt_scratch (ppgtt->scratch_pt, ppgtt->base.dev);free_pd_single (&ppgtt->pd); }
@@ -1076,6 +1205,9 @@ static int gen6_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt)* size. We allocate at the top of the GTT to avoid fragmentation. */ BUG_ON (!drm_mm_initialized (&dev_priv->gtt.base.mm)); + ppgtt->scratch_pt =alloc_pt_scratch (ppgtt->base.dev); +if (IS_ERR (ppgtt->scratch_pt)) +return PTR_ERR (ppgtt->scratch_pt); alloc: ret = drm_mm_insert_node_in_range_generic(&dev_priv->gtt.base.mm, &ppgtt->node, GEN6_PD_SIZE,
@@ -1089,20 +1221,25 @@ alloc:0 , dev_priv->gtt.base.total, 0);if (ret) -return ret; + goto err_out; retried = true; goto alloc; }if (ret) -return ret; + goto err_out; +if (ppgtt->node.start < dev_priv->gtt.mappable_end)DRM_DEBUG ("Forced to use aperture for PDEs\n" ); ppgtt->num_pd_entries =GEN6_PPGTT_PD_ENTRIES ;return 0 ; + +err_out: +free_pt_scratch (ppgtt->scratch_pt, ppgtt->base.dev); +return ret; }static int gen6_ppgtt_alloc (struct i915_hw_ppgtt *ppgtt)
@@ -1113,7 +1250,9 @@ static int gen6_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt)if (ret)return ret; - ret =alloc_pt_range (&ppgtt->pd,0 , ppgtt->num_pd_entries); + ret =alloc_pt_range (&ppgtt->pd,0 , ppgtt->num_pd_entries, + ppgtt->base.dev); +if (ret) {drm_mm_remove_node (&ppgtt->node);return ret;
@@ -1122,30 +1261,6 @@ static int gen6_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt)return 0 ; } -static int gen6_ppgtt_setup_page_tables (struct i915_hw_ppgtt *ppgtt) -{ -struct drm_device *dev = ppgtt->base.dev; -int i; - -for (i =0 ; i < ppgtt->num_pd_entries; i++) { -struct page *page; -dma_addr_t pt_addr; - - page = ppgtt->pd.page_tables[i]->page; - pt_addr =pci_map_page (dev->pdev, page,0 ,4096 , -PCI_DMA_BIDIRECTIONAL ); - -if (pci_dma_mapping_error (dev->pdev, pt_addr)) { -gen6_ppgtt_unmap_pages (ppgtt); -return -EIO ; - } - - ppgtt->pd.page_tables[i]->daddr = pt_addr; - } - -return 0 ; -} -static int gen6_ppgtt_init (struct i915_hw_ppgtt *ppgtt) {struct drm_device *dev = ppgtt->base.dev;
@@ -1166,12 +1281,8 @@ static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt)if (ret)return ret; - ret =gen6_ppgtt_setup_page_tables (ppgtt); -if (ret) { -gen6_ppgtt_free (ppgtt); -return ret; - } - + ppgtt->base.allocate_va_range = gen6_alloc_va_range; + ppgtt->base.teardown_va_range = gen6_teardown_va_range; ppgtt->base.clear_range = gen6_ppgtt_clear_range; ppgtt->base.insert_entries = gen6_ppgtt_insert_entries; ppgtt->base.cleanup = gen6_ppgtt_cleanup;
@@ -1182,11 +1293,15 @@ static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt) ppgtt->pd.pd_offset = ppgtt->node.start /PAGE_SIZE *sizeof (gen6_gtt_pte_t); + ppgtt->pd_addr = (gen6_gtt_pte_t __iomem *)dev_priv->gtt.gsm + + ppgtt->pd.pd_offset /sizeof (gen6_gtt_pte_t); + +gen6_write_page_range (dev_priv, &ppgtt->pd,0 , ppgtt->base.total); +DRM_DEBUG_DRIVER ("Allocated pde space (%ldM) at GTT entry: %lx\n" , ppgtt->node.size >>20 , ppgtt->node.start /PAGE_SIZE ); -gen6_write_pdes (ppgtt);DRM_DEBUG ("Adding PPGTT at offset %x\n" , ppgtt->pd.pd_offset <<10 );
@@ -1301,17 +1416,28 @@ void i915_ppgtt_release(struct kref *kref)kfree (ppgtt); } -static void +static int ppgtt_bind_vma (struct i915_vma *vma,enum i915_cache_level cache_level,u32 flags) { +int ret; +/* Currently applicable only to VLV */ if (vma->obj->gt_ro) flags |=PTE_READ_ONLY ; +if (vma->vm->allocate_va_range) { + ret = vma->vm->allocate_va_range (vma->vm, + vma->node.start, + vma->node.size); +if (ret) +return ret; + } + vma->vm->insert_entries (vma->vm, vma->obj->pages, vma->node.start, cache_level, flags); +return 0 ; }static void ppgtt_unbind_vma (struct i915_vma *vma)
@@ -1320,6 +1446,9 @@ static void ppgtt_unbind_vma(struct i915_vma *vma) vma->node.start, vma->obj->base.size, true); +if (vma->vm->teardown_va_range) + vma->vm->teardown_va_range (vma->vm, + vma->node.start, vma->node.size); }extern int intel_iommu_gfx_mapped;
@@ -1463,13 +1592,14 @@ void i915_gem_restore_gtt_mappings(struct drm_device *dev)list_for_each_entry (vm, &dev_priv->vm_list, global_link) {/* TODO: Perhaps it shouldn't be gen6 specific */ -if (i915_is_ggtt (vm)) { -if (dev_priv->mm.aliasing_ppgtt) -gen6_write_pdes (dev_priv->mm.aliasing_ppgtt); -continue ; - } -gen6_write_pdes (container_of(vm ,struct i915_hw_ppgtt ,base )); +struct i915_hw_ppgtt *ppgtt = + container_of(vm, struct i915_hw_ppgtt, base); + +if (i915_is_ggtt (vm)) + ppgtt = dev_priv->mm.aliasing_ppgtt; + +gen6_write_page_range (dev_priv, &ppgtt->pd,0 , ppgtt->num_pd_entries); }i915_ggtt_flush (dev_priv);
@@ -1634,9 +1764,9 @@ static void gen6_ggtt_clear_range(struct i915_address_space *vm, } -static void i915_ggtt_bind_vma (struct i915_vma *vma, -enum i915_cache_level cache_level, -u32 unused) +static int i915_ggtt_bind_vma (struct i915_vma *vma, +enum i915_cache_level cache_level, +u32 unused) {const unsigned long entry = vma->node.start >>PAGE_SHIFT ; unsigned int flags = (cache_level == I915_CACHE_NONE) ?
@@ -1645,6 +1775,8 @@ static void i915_ggtt_bind_vma(struct i915_vma *vma,BUG_ON (!i915_is_ggtt (vma->vm));intel_gtt_insert_sg_entries (vma->ggtt_view.pages, entry, flags); vma->bound =GLOBAL_BIND ; + +return 0 ; } static void i915_ggtt_clear_range(struct i915_address_space *vm,
@@ -1667,9 +1799,9 @@ static void i915_ggtt_unbind_vma(struct i915_vma *vma)intel_gtt_clear_range (first, size); } -static void ggtt_bind_vma (struct i915_vma *vma, -enum i915_cache_level cache_level, -u32 flags) +static int ggtt_bind_vma (struct i915_vma *vma, +enum i915_cache_level cache_level, +u32 flags) {struct drm_device *dev = vma->vm->dev;struct drm_i915_private *dev_priv = dev->dev_private;
@@ -1710,6 +1842,8 @@ static void ggtt_bind_vma(struct i915_vma *vma, cache_level, flags); vma->bound |=LOCAL_BIND ; } + +return 0 ; }static void ggtt_unbind_vma (struct i915_vma *vma)
drivers/gpu/drm/i915/i915_gem_gtt.h +116 -35
188 lines not on this page · raw message
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
Re: [PATCH 10/24] drm/i915: Track GEN6 page table usage
Ok this is just a very coarse high level comment. The only thing I really looked for is very the dynamic pagetable allocation point happens and how out-of-memory is handled in there.
But I've noticed that while trying to look for that that the patches and code have a lot of history and often code comments and commit message don't really agree with the code any more. I think that should be carefully reviewed to make it less confusing.
Quoted 16 lines · Michel Thierry
drivers/gpu/drm/i915/i915_gem_gtt.h +3
-0@@ -54,7 +54,10 @@ typedef gen8_gtt_pte_t gen8_ppgtt_pde_t;#define GEN6_PPGTT_PD_ENTRIES 512#define GEN6_PD_SIZE (GEN6_PPGTT_PD_ENTRIES * PAGE_SIZE)#define GEN6_PD_ALIGN (PAGE_SIZE * 16) +#define GEN6_PDE_SHIFT 22#define GEN6_PDE_VALID (1 << 0) +#define GEN6_PDE_MASK (GEN6_PPGTT_PD_ENTRIES-1) +#define NUM_PTE (pde_shift) (1 << (pde_shift - PAGE_SHIFT))#define GEN7_PTE_CACHE_L3_LLC (3 << 1)
drivers/gpu/drm/i915/i915_gem_gtt.h struct i915_vma { · line 191
Quoted 10 lines
drivers/gpu/drm/i915/i915_gem_gtt.h +4
-3@@ -182,9 +185,33 @@ struct i915_vma {* setting the valid PTE entries to a reserved scratch page. */ void (*unbind_vma)(struct i915_vma *vma);/* Map an object into an address space with the given cache flags. */ -void (*bind_vma)(struct i915_vma *vma, -enum i915_cache_level cache_level, -u32 flags); +int (*bind_vma)(struct i915_vma *vma, +enum i915_cache_level cache_level, +u32 flags); +};
So this patch changes the interface of vma->bind_vma to return errors when the pagetable alloc fails. Afaics none of the callers get updated, and I didn't see those adjustments in any other patch in this series. This doesn't work (or I'm blind).
Also I'm not too happy with smashing this into ->bind_vma: This function is also called in places where we know that the pagetables must be there already, namely when changing pte bits in i915_gem_object_set_cache_level.
Imo we should add an explicit call to allocate required pagetables in i915_gem_object_bind_to_vm, which is the only place which actually needs this (I think).
-Daniel
Signature · 7 lines
Daniel Vetter Software Engineer, Intel Corporation +41 (0) 79 365 57 48 - http://blog.ffwll.ch _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH 11/24 drm/i915: Extract context switch skip logic
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
We have some fanciness coming up. This patch just breaks out the logic.
--- drivers/gpu/drm/i915/i915_gem_context.c | 10 ++++++++++ 1 file changed, 10 insertions(+)
drivers/gpu/drm/i915/i915_gem_context.c +10 -0
@@ -726,6 +726,16 @@ unpin_out:return ret; } +static inline bool should_skip_switch (struct intel_engine_cs *ring, +struct intel_context *from, +struct intel_context *to) +{ +if (from == to && !to->remap_slice) +return true; + +return false; +} +/** * i915_switch_context() - perform a GPU context switch. * @ring: ring for which we'll execute the context switch
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
Re: [PATCH 11/24] drm/i915: Extract context switch skip logic
Quoted 6 lines · Michel Thierry
From: Ben Widawsky <[email protected]>
We have some fanciness coming up. This patch just breaks out the logic.
Signed-off-by Ben Widawsky <[email protected]>Signed-off-by Michel Thierry <[email protected]>
The next patch moves around this function again which means the diff is actually bigger. Also an extracted function should actually replace the logic that's extracted in the same patch. As-is this doesn't help review, but I agree it's a nice split-up idea which would help review. Probably got lost in all the rebases.
-Daniel
Quoted 32 lines
--- drivers/gpu/drm/i915/i915_gem_context.c | 10 ++++++++++ 1 file changed, 10 insertions(+)drivers/gpu/drm/i915/i915_gem_context.c +10
-0@@ -726,6 +726,16 @@ unpin_out:return ret; } +static inline bool should_skip_switch (struct intel_engine_cs *ring, +struct intel_context *from, +struct intel_context *to) +{ +if (from == to && !to->remap_slice) +return true; + +return false; +} +/** * i915_switch_context() - perform a GPU context switch. * @ring: ring for which we'll execute the context switch Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
Signature · 7 lines
Daniel Vetter Software Engineer, Intel Corporation +41 (0) 79 365 57 48 - http://blog.ffwll.ch _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH 12/24 drm/i915: Track page table reload need
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
This patch was formerly known as, "Force pd restore when PDEs change, gen6-7." I had to change the name because it is needed for GEN8 too.
The real issue this is trying to solve is when a new object is mapped into the current address space. The GPU does not snoop the new mapping so we must do the gen specific action to reload the page tables.
GEN8 and GEN7 do differ in the way they load page tables for the RCS. GEN8 does so with the context restore, while GEN7 requires the proper load commands in the command streamer. Non-render is similar for both.
Caveat for GEN7
The docs say you cannot change the PDEs of a currently running context. We never map new PDEs of a running context, and expect them to be present - so I think this is okay. (We can unmap, but this should also be okay since we only unmap unreferenced objects that the GPU shouldn't be tryingto va->pa xlate.) The MI_SET_CONTEXT command does have a flag to signal that even if the context is the same, force a reload. It's unclear exactly what this does, but I have a hunch it's the right thing to do.
The logic assumes that we always emit a context switch after mapping new PDEs, and before we submit a batch. This is the case today, and has been the case since the inception of hardware contexts. A note in the comment let's the user know.
squash! drm/i915: Force pd restore when PDEs change, gen6-7
It's not just for gen8. If the current context has mappings change, we need a context reload to switch
v2: Rebased after ppgtt clean up patches. Split the warning for aliasing and true ppgtt options. And do not break aliasing ppgtt, where to->ppgtt is always null.
--- drivers/gpu/drm/i915/i915_gem_context.c | 67 +++++++++++++++++++++--------- drivers/gpu/drm/i915/i915_gem_execbuffer.c | 11 +++++ drivers/gpu/drm/i915/i915_gem_gtt.c | 15 ++++++- drivers/gpu/drm/i915/i915_gem_gtt.h | 2 + 4 files changed, 75 insertions(+), 20 deletions(-)
drivers/gpu/drm/i915/i915_gem_context.c +48 -19
@@ -563,6 +563,42 @@ mi_set_context(struct intel_engine_cs *ring,return ret; } +static inline bool should_skip_switch (struct intel_engine_cs *ring, +struct intel_context *from, +struct intel_context *to) +{ +struct drm_i915_private *dev_priv = ring->dev->dev_private; + +if (to->remap_slice) +return false; + +if (to->ppgtt) { +if (from == to && !test_bit (ring->id, &to->ppgtt->base.pd_reload_mask)) +return true; + }else { +if (from == to && !test_bit (ring->id, &dev_priv->mm.aliasing_ppgtt->base.pd_reload_mask)) +return true; + } + +return false; +} + +static bool +needs_pd_load_pre (struct intel_engine_cs *ring,struct intel_context *to) +{ +struct drm_i915_private *dev_priv = ring->dev->dev_private; + +return ((INTEL_INFO (ring->dev)->gen <8 ) || + (ring != &dev_priv->ring[RCS ])) && to->ppgtt; +} + +static bool +needs_pd_load_post (struct intel_engine_cs *ring,struct intel_context *to) +{ +return IS_GEN8 (ring->dev) && + (to->ppgtt || &to->ppgtt->base.pd_reload_mask); +} +static int do_switch (struct intel_engine_cs *ring,struct intel_context *to) {
@@ -571,9 +607,6 @@ static int do_switch(struct intel_engine_cs *ring,u32 hw_flags =0 ;bool uninitialized = false;struct i915_vma *vma; -bool needs_pd_load_pre = ((INTEL_INFO (ring->dev)->gen <8 ) || - (ring != &dev_priv->ring[RCS ])) && to->ppgtt; -bool needs_pd_load_post = false;int ret, i;if (from !=NULL && ring == &dev_priv->ring[RCS ]) {
@@ -581,7 +614,7 @@ static int do_switch(struct intel_engine_cs *ring,BUG_ON (!i915_gem_obj_is_pinned (from->legacy_hw_ctx.rcs_state)); } -if (from == to && !to->remap_slice) +if (should_skip_switch (ring, from, to))return 0 ;/* Trying to pin first makes error handling easier. */
@@ -599,7 +632,7 @@ static int do_switch(struct intel_engine_cs *ring,*/ from = ring->last_context; -if (needs_pd_load_pre) { +if (needs_pd_load_pre (ring, to)) {/* Older GENs and non render rings still want the load first, * "PP_DCLV followed by PP_DIR_BASE register through Load * Register Immediate commands in Ring Buffer before submitting
@@ -608,6 +641,12 @@ static int do_switch(struct intel_engine_cs *ring, ret = to->ppgtt->switch_mm (to->ppgtt, ring);if (ret) goto unpin_out; + +/* Doing a PD load always reloads the page dirs */ +if (to->ppgtt) +clear_bit (ring->id, &to->ppgtt->base.pd_reload_mask); +else +clear_bit (ring->id, &dev_priv->mm.aliasing_ppgtt->base.pd_reload_mask); }if (ring != &dev_priv->ring[RCS ]) {
@@ -644,16 +683,16 @@ static int do_switch(struct intel_engine_cs *ring,* XXX: If we implemented page directory eviction code, this * optimization needs to be removed. */ -if (!to->legacy_hw_ctx.initialized ||i915_gem_context_is_default (to)) { +if (!to->legacy_hw_ctx.initialized ||i915_gem_context_is_default (to)) hw_flags |=MI_RESTORE_INHIBIT ; - needs_pd_load_post = to->ppgtt &&IS_GEN8 (ring->dev); - } +else if (to->ppgtt &&test_and_clear_bit (ring->id, &to->ppgtt->base.pd_reload_mask)) + hw_flags |=MI_FORCE_RESTORE ; ret =mi_set_context (ring, to, hw_flags);if (ret) goto unpin_out; -if (needs_pd_load_post) { +if (needs_pd_load_post (ring, to)) { ret = to->ppgtt->switch_mm (to->ppgtt, ring);/* The hardware context switch is emitted, but we haven't * actually changed the state - so it's probably safe to bail
@@ -726,16 +765,6 @@ unpin_out:return ret; } -static inline bool should_skip_switch (struct intel_engine_cs *ring, -struct intel_context *from, -struct intel_context *to) -{ -if (from == to && !to->remap_slice) -return true; - -return false; -} -/** * i915_switch_context() - perform a GPU context switch. * @ring: ring for which we'll execute the context switch
drivers/gpu/drm/i915/i915_gem_execbuffer.c +11 -0
@@ -1199,6 +1199,13 @@ i915_gem_ringbuffer_submission(struct drm_device *dev, struct drm_file *file,if (ret) goto error; +if (ctx->ppgtt) +WARN (ctx->ppgtt->base.pd_reload_mask & (1 <<ring->id), +"%s didn't clear reload\n" , ring->name); +else +WARN (dev_priv->mm.aliasing_ppgtt->base.pd_reload_mask & + (1 <<ring->id),"%s didn't clear reload\n" , ring->name); + instp_mode = args->flags &I915_EXEC_CONSTANTS_MASK ; instp_mask =I915_EXEC_CONSTANTS_MASK ;switch (instp_mode) {
@@ -1446,6 +1453,10 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,if (ret) goto err; +/* XXX: Reserve has possibly change PDEs which means we must do a +* context switch before we can coherently read some of the reserved +* VMAs. */ +/* The objects are in their final locations, apply the relocations. */ if (need_relocs) ret =i915_gem_execbuffer_relocate (eb);
drivers/gpu/drm/i915/i915_gem_gtt.c +14 -1
@@ -1398,6 +1398,15 @@ i915_ppgtt_create(struct drm_device *dev, struct drm_i915_file_private *fpriv)return ppgtt; } +/* PDE TLBs are a pain invalidate pre GEN8. It requires a context reload. If we +* are switching between contexts with the same LRCA, we also must do a force +* restore. +*/ +#define ppgtt_invalidate_tlbs (vm) do {\ +/* If current vm != vm, */ \ + vm->pd_reload_mask =INTEL_INFO (vm->dev)->ring_mask; \ +}while (0 ) + void i915_ppgtt_release(struct kref *kref) { struct i915_hw_ppgtt *ppgtt =
@@ -1433,6 +1442,8 @@ ppgtt_bind_vma(struct i915_vma *vma, vma->node.size);if (ret)return ret; + +ppgtt_invalidate_tlbs (vma->vm); } vma->vm->insert_entries(vma->vm, vma->obj->pages, vma->node.start,
@@ -1446,9 +1457,11 @@ static void ppgtt_unbind_vma(struct i915_vma *vma) vma->node.start, vma->obj->base.size, true); -if (vma->vm->teardown_va_range) +if (vma->vm->teardown_va_range) { vma->vm->teardown_va_range (vma->vm, vma->node.start, vma->node.size); +ppgtt_invalidate_tlbs (vma->vm); + } }extern int intel_iommu_gfx_mapped;
drivers/gpu/drm/i915/i915_gem_gtt.h +2 -0
@@ -226,6 +226,8 @@ struct i915_address_space {struct page *page; } scratch; +unsigned long pd_reload_mask; +/** * List of objects currently involved in rendering. *
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
Re: [PATCH 12/24] drm/i915: Track page table reload need
Another high-level logic comment below.
Quoted 21 lines · Michel Thierry
drivers/gpu/drm/i915/i915_gem_gtt.c +9
-0@@ -1398,6 +1398,15 @@ i915_ppgtt_create(struct drm_device *dev, struct drm_i915_file_private *fpriv)return ppgtt; } +/* PDE TLBs are a pain invalidate pre GEN8. It requires a context reload. If we +* are switching between contexts with the same LRCA, we also must do a force +* restore. +*/ +#define ppgtt_invalidate_tlbs (vm) do {\ +/* If current vm != vm, */ \ + vm->pd_reload_mask =INTEL_INFO (vm->dev)->ring_mask; \ +}while (0 ) + void i915_ppgtt_release(struct kref *kref) { struct i915_hw_ppgtt *ppgtt =
drivers/gpu/drm/i915/i915_gem_gtt.c ppgtt_bind_vma(struct i915_vma *vma, · line 1446
Quoted 5 lines
drivers/gpu/drm/i915/i915_gem_gtt.c +2
-0@@ -1433,6 +1442,8 @@ ppgtt_bind_vma(struct i915_vma *vma, vma->node.size);if (ret)return ret; + +ppgtt_invalidate_tlbs (vma->vm);
Imo we should only set this when we actually allocate new pagetables, and not unconditionally every time we bind.
-Daniel
Signature · 7 lines
Daniel Vetter Software Engineer, Intel Corporation +41 (0) 79 365 57 48 - http://blog.ffwll.ch _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH 13/24 drm/i915: Initialize all contexts
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
The problem is we're going to switch to a new context, which could be the default context. The plan was to use restore inhibit, which would be fine, except if we are using dynamic page tables (which we will). If we use dynamic page tables and we don't load new page tables, the previous page tables might go away, and future operations will fault.
CTXA runs.
switch to default, restore inhibit
CTXA dies and has its address space taken away.
Run CTXB, tries to save using the context A's address space - this fails.
The general solution is to make sure every context has it's own state, and its own address space. For cases when we must restore inhibit, first thing we do is load a valid address space. I thought this would be enough, but apparently there are references within the context itself which will refer to the old address space - therefore, we also must reinitialize.
It was tricky to track this down as we don't have much insight into what happens in a context save.
This is required for the next patch which enables dynamic page tables.
v2: to->ppgtt is only valid in full ppgtt.
--- drivers/gpu/drm/i915/i915_gem_context.c | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-)
drivers/gpu/drm/i915/i915_gem_context.c +11 -14
@@ -592,13 +592,6 @@ needs_pd_load_pre(struct intel_engine_cs *ring, struct intel_context *to) (ring != &dev_priv->ring[RCS])) && to->ppgtt; } -static bool -needs_pd_load_post (struct intel_engine_cs *ring,struct intel_context *to) -{ -return IS_GEN8 (ring->dev) && - (to->ppgtt || &to->ppgtt->base.pd_reload_mask); -} -static int do_switch (struct intel_engine_cs *ring,struct intel_context *to) {
@@ -679,20 +672,24 @@ static int do_switch(struct intel_engine_cs *ring,/* GEN8 does *not* require an explicit reload if the PDPs have been * setup, and we do not wish to move them. -* -* XXX: If we implemented page directory eviction code, this -* optimization needs to be removed. */ -if (!to->legacy_hw_ctx.initialized ||i915_gem_context_is_default (to)) +if (!to->legacy_hw_ctx.initialized) { hw_flags |=MI_RESTORE_INHIBIT ; -else if (to->ppgtt &&test_and_clear_bit (ring->id, &to->ppgtt->base.pd_reload_mask)) +/* NB: If we inhibit the restore, the context is not allowed to +* die because future work may end up depending on valid address +* space. This means we must enforce that a page table load +* occur when this occurs. */ + }else if (to->ppgtt &&test_and_clear_bit (ring->id, &to->ppgtt->base.pd_reload_mask)) hw_flags |=MI_FORCE_RESTORE ; ret =mi_set_context (ring, to, hw_flags);if (ret) goto unpin_out; -if (needs_pd_load_post (ring, to)) { +if (IS_GEN8 (ring->dev) && to->ppgtt && (hw_flags &MI_RESTORE_INHIBIT )) { +/* We have a valid page directory (scratch) to switch to. This +* allows the old VM to be freed. Note that if anything occurs +* between the set context, and here, we are f*cked */ ret = to->ppgtt->switch_mm (to->ppgtt, ring);/* The hardware context switch is emitted, but we haven't * actually changed the state - so it's probably safe to bail
@@ -742,7 +739,7 @@ static int do_switch(struct intel_engine_cs *ring,i915_gem_context_unreference (from); } - uninitialized = !to->legacy_hw_ctx.initialized && from ==NULL ; + uninitialized = !to->legacy_hw_ctx.initialized; to->legacy_hw_ctx.initialized = true; done:
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH 14/24 drm/i915: Finish gen6/7 dynamic page table allocation
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
This patch continues on the idea from the previous patch. From here on, in the steady state, PDEs are all pointing to the scratch page table (as recommended in the spec). When an object is allocated in the VA range, the code will determine if we need to allocate a page for the page table. Similarly when the object is destroyed, we will remove, and free the page table pointing the PDE back to the scratch page.
Following patches will work to unify the code a bit as we bring in GEN8 support. GEN6 and GEN8 are different enough that I had a hard time to get to this point with as much common code as I do.
The aliasing PPGTT must pre-allocate all of the page tables. There are a few reasons for this. Two trivial ones: aliasing ppgtt goes through the ggtt paths, so it's hard to maintain, we currently do not restore the default context (assuming the previous force reload is indeed necessary). Most importantly though, the only way (it seems from empirical evidence) to invalidate the CS TLBs on non-render ring is to either use ring sync (which requires actually stopping the rings in order to synchronize when the sync completes vs. where you are in execution), or to reload DCLV. Since without full PPGTT we do not ever reload the DCLV register, there is no good way to achieve this. The simplest solution is just to not support dynamic page table creation/destruction in the aliasing PPGTT.
We could always reload DCLV, but this seems like quite a bit of excess overhead only to save at most 2MB-4k of memory for the aliasing PPGTT page tables.
v2: Make the page table bitmap declared inside the function (Chris) Simplify the way scratching address space works.
Move the alloc/teardown tracepoints up a level in the call stack so that both all implementations get the trace.
v3: Updated trace event to spit out a name
v4: Aliasing ppgtt is now initialized differently (in setup global gtt)
--- drivers/gpu/drm/i915/i915_debugfs.c | 3 +- drivers/gpu/drm/i915/i915_drv.h | 7 ++ drivers/gpu/drm/i915/i915_gem_gtt.c | 125 +++++++++++++++++++++++++++++++++--- drivers/gpu/drm/i915/i915_trace.h | 116 +++++++++++++++++++++++++++++++++ 4 files changed, 240 insertions(+), 11 deletions(-)
drivers/gpu/drm/i915/i915_debugfs.c +2 -1
@@ -2149,6 +2149,8 @@ static void gen6_ppgtt_info(struct seq_file *m, struct drm_device *dev)seq_printf (m,"PP_DIR_BASE_READ: 0x%08x\n" ,I915_READ (RING_PP_DIR_BASE_READ (ring)));seq_printf (m,"PP_DIR_DCLV: 0x%08x\n" ,I915_READ (RING_PP_DIR_DCLV (ring))); } +seq_printf (m,"ECOCHK: 0x%08x\n\n" ,I915_READ (GAM_ECOCHK )); +if (dev_priv->mm.aliasing_ppgtt) {struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
@@ -2165,7 +2167,6 @@ static void gen6_ppgtt_info(struct seq_file *m, struct drm_device *dev) get_pid_task(file->pid, PIDTYPE_PID)->comm);idr_for_each (&file_priv->context_idr, per_file_ctx, m); } -seq_printf (m,"ECOCHK: 0x%08x\n" ,I915_READ (GAM_ECOCHK )); }static int i915_ppgtt_info (struct seq_file *m,void *data)
drivers/gpu/drm/i915/i915_drv.h +7 -0
@@ -2819,6 +2819,13 @@ static inline bool i915_is_ggtt(struct i915_address_space *vm)return vm == ggtt; } +static inline bool i915_is_aliasing_ppgtt (struct i915_address_space *vm) +{ +struct i915_address_space *appgtt = + &((struct drm_i915_private *)(vm)->dev->dev_private)->mm.aliasing_ppgtt->base; +return vm == appgtt; +} +static inline struct i915_hw_ppgtt *i915_vm_to_ppgtt (struct i915_address_space *vm) {
drivers/gpu/drm/i915/i915_gem_gtt.c +115 -10
230 lines not on this page · raw message
drivers/gpu/drm/i915/i915_trace.h +116 -0
122 lines not on this page · raw message
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
Re: [PATCH 14/24] drm/i915: Finish gen6/7 dynamic page table allocation
Quoted 74 lines · Michel Thierry
From: Ben Widawsky <[email protected]>
This patch continues on the idea from the previous patch. From here on, in the steady state, PDEs are all pointing to the scratch page table (as recommended in the spec). When an object is allocated in the VA range, the code will determine if we need to allocate a page for the page table. Similarly when the object is destroyed, we will remove, and free the page table pointing the PDE back to the scratch page.
Following patches will work to unify the code a bit as we bring in GEN8 support. GEN6 and GEN8 are different enough that I had a hard time to get to this point with as much common code as I do.
The aliasing PPGTT must pre-allocate all of the page tables. There are a few reasons for this. Two trivial ones: aliasing ppgtt goes through the ggtt paths, so it's hard to maintain, we currently do not restore the default context (assuming the previous force reload is indeed necessary). Most importantly though, the only way (it seems from empirical evidence) to invalidate the CS TLBs on non-render ring is to either use ring sync (which requires actually stopping the rings in order to synchronize when the sync completes vs. where you are in execution), or to reload DCLV. Since without full PPGTT we do not ever reload the DCLV register, there is no good way to achieve this. The simplest solution is just to not support dynamic page table creation/destruction in the aliasing PPGTT.
We could always reload DCLV, but this seems like quite a bit of excess overhead only to save at most 2MB-4k of memory for the aliasing PPGTT page tables.
v2: Make the page table bitmap declared inside the function (Chris) Simplify the way scratching address space works.
Move the alloc/teardown tracepoints up a level in the call stack so that both all implementations get the trace.v3: Updated trace event to spit out a name
v4: Aliasing ppgtt is now initialized differently (in setup global gtt)
Signed-off-by Ben Widawsky <[email protected]>Signed-off-by Michel Thierry <[email protected]> (v4)--- drivers/gpu/drm/i915/i915_debugfs.c | 3 +- drivers/gpu/drm/i915/i915_drv.h | 7 ++ drivers/gpu/drm/i915/i915_gem_gtt.c | 125 +++++++++++++++++++++++++++++++++--- drivers/gpu/drm/i915/i915_trace.h | 116 +++++++++++++++++++++++++++++++++ 4 files changed, 240 insertions(+), 11 deletions(-)drivers/gpu/drm/i915/i915_debugfs.c +2
-1@@ -2149,6 +2149,8 @@ static void gen6_ppgtt_info(struct seq_file *m, struct drm_device *dev)seq_printf (m,"PP_DIR_BASE_READ: 0x%08x\n" ,I915_READ (RING_PP_DIR_BASE_READ (ring)));seq_printf (m,"PP_DIR_DCLV: 0x%08x\n" ,I915_READ (RING_PP_DIR_DCLV (ring))); } +seq_printf (m,"ECOCHK: 0x%08x\n\n" ,I915_READ (GAM_ECOCHK )); +if (dev_priv->mm.aliasing_ppgtt) {struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;@@ -2165,7 +2167,6 @@ static void gen6_ppgtt_info(struct seq_file *m, struct drm_device *dev) get_pid_task(file->pid, PIDTYPE_PID)->comm);idr_for_each (&file_priv->context_idr, per_file_ctx, m); } -seq_printf (m,"ECOCHK: 0x%08x\n" ,I915_READ (GAM_ECOCHK )); }static int i915_ppgtt_info (struct seq_file *m,void *data)drivers/gpu/drm/i915/i915_drv.h +0
-0
drivers/gpu/drm/i915/i915_drv.h static inline bool i915_is_ggtt(struct i915_address_space *vm) · line 2827
Quoted 9 lines
drivers/gpu/drm/i915/i915_drv.h +6
-0@@ -2819,6 +2819,13 @@ static inline bool i915_is_ggtt(struct i915_address_space *vm)return vm == ggtt; } +static inline bool i915_is_aliasing_ppgtt (struct i915_address_space *vm) +{ +struct i915_address_space *appgtt = + &((struct drm_i915_private *)(vm)->dev->dev_private)->mm.aliasing_ppgtt->base; +return vm == appgtt; +}
We killed the aliasing ppgtt vm, this condition will never be true when you pass any vma->vm pointer to it.
-Daniel
Signature · 7 lines
Daniel Vetter Software Engineer, Intel Corporation +41 (0) 79 365 57 48 - http://blog.ffwll.ch _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH 15/24 drm/i915/bdw: Use dynamic allocation idioms on free
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
The page directory freer is left here for now as it's still useful given that GEN8 still preallocates. Once the allocation functions are broken up into more discrete chunks, we'll follow suit and destroy this leftover piece.
v2: Match trace_i915_va_teardown params
v3: Multiple rebases.
--- drivers/gpu/drm/i915/i915_gem_gtt.c | 54 +++++++++++++++++++++++-------------- drivers/gpu/drm/i915/i915_gem_gtt.h | 46 +++++++++++++++++++++++++++++++ 2 files changed, 80 insertions(+), 20 deletions(-)
drivers/gpu/drm/i915/i915_gem_gtt.c +34 -20
@@ -579,27 +579,32 @@ static void gen8_ppgtt_insert_entries(struct i915_address_space *vm, } } -static void gen8_free_page_tables (struct i915_pagedir *pd,struct drm_device *dev) +static void gen8_teardown_va_range (struct i915_address_space *vm, +uint64_t start,uint64_t length) { -int i; - -if (!pd->page) -return ; - -for (i =0 ; i <GEN8_PDES_PER_PAGE ; i++) { -free_pt_single (pd->page_tables[i], dev); - pd->page_tables[i] =NULL ; +struct i915_hw_ppgtt *ppgtt = + container_of(vm, struct i915_hw_ppgtt, base); +struct i915_pagedir *pd; +struct i915_pagetab *pt; +uint64_t temp; +uint32_t pdpe, pde; + +gen8_for_each_pdpe (pd, &ppgtt->pdp, start, length, temp, pdpe) { +uint64_t pd_len =gen8_clamp_pd (start, length); +uint64_t pd_start = start; +gen8_for_each_pde (pt, pd, pd_start, pd_len, temp, pde) { +free_pt_single (pt, vm->dev); + } +free_pd_single (pd); } } -static void gen8_ppgtt_free (struct i915_hw_ppgtt *ppgtt) +/* This function will die soon */ +static void gen8_free_full_pagedir (struct i915_hw_ppgtt *ppgtt,int i) { -int i; - -for (i =0 ; i < ppgtt->num_pd_pages; i++) { -gen8_free_page_tables (ppgtt->pdp.pagedir[i], ppgtt->base.dev); -free_pd_single (ppgtt->pdp.pagedir[i]); - } +gen8_teardown_va_range (&ppgtt->base, + i <<GEN8_PDPE_SHIFT , + (1 <<GEN8_PDPE_SHIFT )); }static void gen8_ppgtt_unmap_pages (struct i915_hw_ppgtt *ppgtt)
@@ -614,19 +619,28 @@ static void gen8_ppgtt_unmap_pages(struct i915_hw_ppgtt *ppgtt)continue ;pci_unmap_page (hwdev, ppgtt->pdp.pagedir[i]->daddr,PAGE_SIZE , -PCI_DMA_BIDIRECTIONAL ); +PCI_DMA_BIDIRECTIONAL );for (j =0 ; j <GEN8_PDES_PER_PAGE ; j++) {struct i915_pagedir *pd = ppgtt->pdp.pagedir[i]; -struct i915_pagetab *pt = pd->page_tables[j]; +struct i915_pagetab *pt = pd->page_tables[j];dma_addr_t addr = pt->daddr;if (addr)pci_unmap_page (hwdev, addr,PAGE_SIZE , -PCI_DMA_BIDIRECTIONAL ); +PCI_DMA_BIDIRECTIONAL ); } } } +static void gen8_ppgtt_free (struct i915_hw_ppgtt *ppgtt) +{ +trace_i915_va_teardown (&ppgtt->base, + ppgtt->base.start, ppgtt->base.total, +VM_TO_TRACE_NAME (&ppgtt->base)); +gen8_teardown_va_range (&ppgtt->base, + ppgtt->base.start, ppgtt->base.total); +} +static void gen8_ppgtt_cleanup (struct i915_address_space *vm) { struct i915_hw_ppgtt *ppgtt =
@@ -651,7 +665,7 @@ static int gen8_ppgtt_allocate_page_tables(struct i915_hw_ppgtt *ppgtt) unwind_out:while (i--) -gen8_free_page_tables (ppgtt->pdp.pagedir[i], ppgtt->base.dev); +gen8_free_full_pagedir (ppgtt, i);return -ENOMEM ; }
drivers/gpu/drm/i915/i915_gem_gtt.h +46 -0
52 lines not on this page · raw message
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH 16/24 drm/i915/bdw: pagedirs rework allocation
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
Start using gen8_for_each_pdpe macro to allocate the page directories.
--- drivers/gpu/drm/i915/i915_gem_gtt.c | 43 ++++++++++++++++++++++++++----------- 1 file changed, 31 insertions(+), 12 deletions(-)
drivers/gpu/drm/i915/i915_gem_gtt.c +31 -12
@@ -594,8 +594,10 @@ static void gen8_teardown_va_range(struct i915_address_space *vm,uint64_t pd_start = start;gen8_for_each_pde (pt, pd, pd_start, pd_len, temp, pde) {free_pt_single (pt, vm->dev); + pd->page_tables[pde] =NULL ; }free_pd_single (pd); + ppgtt->pdp.pagedir[pdpe] =NULL ; } }
@@ -670,25 +672,39 @@ unwind_out:return -ENOMEM ; } -static int gen8_ppgtt_allocate_page_directories (struct i915_hw_ppgtt *ppgtt, -const int max_pdp) +static int gen8_ppgtt_alloc_pagedirs (struct i915_pagedirpo *pdp, +uint64_t start, +uint64_t length) { -int i; +struct i915_hw_ppgtt *ppgtt = + container_of(pdp, struct i915_hw_ppgtt, pdp); +struct i915_pagedir *unused; +uint64_t temp; +uint32_t pdpe; -for (i =0 ; i < max_pdp; i++) { - ppgtt->pdp.pagedir[i] =alloc_pd_single (); -if (IS_ERR (ppgtt->pdp.pagedir[i])) +/* FIXME: PPGTT container_of won't work for 64b */ +BUG_ON ((start + length) >0x800000000ULL ); + +gen8_for_each_pdpe (unused, pdp, start, length, temp, pdpe) { +BUG_ON (unused); + pdp->pagedir[pdpe] =alloc_pd_single (); +if (IS_ERR (ppgtt->pdp.pagedir[pdpe])) goto unwind_out; + + ppgtt->num_pd_pages++; } - ppgtt->num_pd_pages = max_pdp;BUG_ON (ppgtt->num_pd_pages >GEN8_LEGACY_PDPES );return 0 ; unwind_out: -while (i--) -free_pd_single (ppgtt->pdp.pagedir[i]); +while (pdpe--) { +free_pd_single (ppgtt->pdp.pagedir[pdpe]); + ppgtt->num_pd_pages--; + } + +WARN_ON (ppgtt->num_pd_pages);return -ENOMEM ; }
@@ -698,7 +714,8 @@ static int gen8_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt, {int ret; - ret =gen8_ppgtt_allocate_page_directories (ppgtt, max_pdp); + ret =gen8_ppgtt_alloc_pagedirs (&ppgtt->pdp, ppgtt->base.start, + ppgtt->base.total);if (ret)return ret;
@@ -775,6 +792,10 @@ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size)if (size % (1 <<30 ))DRM_INFO ("Pages will be wasted unless GTT size (%llu) is divisible by 1GB\n" , size); + ppgtt->base.start =0 ; + ppgtt->base.total = size; +BUG_ON (ppgtt->base.total ==0 ); +/* 1. Do all our allocations for page directories and page tables. */ ret =gen8_ppgtt_alloc (ppgtt, max_pdp);if (ret)
@@ -822,8 +843,6 @@ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size) ppgtt->base.clear_range = gen8_ppgtt_clear_range; ppgtt->base.insert_entries = gen8_ppgtt_insert_entries; ppgtt->base.cleanup = gen8_ppgtt_cleanup; - ppgtt->base.start =0 ; - ppgtt->base.total = ppgtt->num_pd_entries *GEN8_PTES_PER_PAGE *PAGE_SIZE ;DRM_DEBUG_DRIVER ("Allocated %d pages for page directories (%d wasted)\n" , ppgtt->num_pd_pages, ppgtt->num_pd_pages - max_pdp);
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH 17/24 drm/i915/bdw: pagetable allocation rework
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
Start using gen8_for_each_pde macro to allocate page tables.
--- drivers/gpu/drm/i915/i915_gem_gtt.c | 54 ++++++++++++++++++++----------------- drivers/gpu/drm/i915/i915_gem_gtt.h | 10 +++++++ 2 files changed, 39 insertions(+), 25 deletions(-)
drivers/gpu/drm/i915/i915_gem_gtt.c +29 -25
@@ -601,14 +601,6 @@ static void gen8_teardown_va_range(struct i915_address_space *vm, } } -/* This function will die soon */ -static void gen8_free_full_pagedir (struct i915_hw_ppgtt *ppgtt,int i) -{ -gen8_teardown_va_range (&ppgtt->base, - i <<GEN8_PDPE_SHIFT , - (1 <<GEN8_PDPE_SHIFT )); -} -static void gen8_ppgtt_unmap_pages (struct i915_hw_ppgtt *ppgtt) {struct pci_dev *hwdev = ppgtt->base.dev->pdev;
@@ -652,22 +644,27 @@ static void gen8_ppgtt_cleanup(struct i915_address_space *vm)gen8_ppgtt_free (ppgtt); } -static int gen8_ppgtt_allocate_page_tables (struct i915_hw_ppgtt *ppgtt) +static int gen8_ppgtt_alloc_pagetabs (struct i915_pagedir *pd, +uint64_t start, +uint64_t length, +struct drm_device *dev) { -int i, ret; +struct i915_pagetab *unused; +uint64_t temp; +uint32_t pde; -for (i =0 ; i < ppgtt->num_pd_pages; i++) { - ret =alloc_pt_range (ppgtt->pdp.pagedir[i], -0 ,GEN8_PDES_PER_PAGE , ppgtt->base.dev); -if (ret) +gen8_for_each_pde (unused, pd, start, length, temp, pde) { +BUG_ON (unused); + pd->page_tables[pde] =alloc_pt_single (dev); +if (IS_ERR (pd->page_tables[pde])) goto unwind_out; }return 0 ; unwind_out: -while (i--) -gen8_free_full_pagedir (ppgtt, i); +while (pde--) +free_pt_single (pd->page_tables[pde], dev);return -ENOMEM ; }
@@ -710,20 +707,28 @@ unwind_out: }static int gen8_ppgtt_alloc (struct i915_hw_ppgtt *ppgtt, -const int max_pdp) +uint64_t start, +uint64_t length) { +struct i915_pagedir *pd; +uint64_t temp; +uint32_t pdpe;int ret; - ret =gen8_ppgtt_alloc_pagedirs (&ppgtt->pdp, ppgtt->base.start, - ppgtt->base.total); + ret =gen8_ppgtt_alloc_pagedirs (&ppgtt->pdp, start, length);if (ret)return ret; - ret =gen8_ppgtt_allocate_page_tables (ppgtt); -if (ret) - goto err_out; +gen8_for_each_pdpe (pd, &ppgtt->pdp, start, length, temp, pdpe) { + ret =gen8_ppgtt_alloc_pagetabs (pd, start, length, + ppgtt->base.dev); +if (ret) + goto err_out; + + ppgtt->num_pd_entries +=GEN8_PDES_PER_PAGE ; + } - ppgtt->num_pd_entries = max_pdp *GEN8_PDES_PER_PAGE ; +BUG_ON (pdpe > ppgtt->num_pd_pages);return 0 ;
@@ -794,10 +799,9 @@ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size) ppgtt->base.start =0 ; ppgtt->base.total = size; -BUG_ON (ppgtt->base.total ==0 );/* 1. Do all our allocations for page directories and page tables. */ - ret =gen8_ppgtt_alloc (ppgtt, max_pdp); + ret =gen8_ppgtt_alloc (ppgtt, ppgtt->base.start, ppgtt->base.total);if (ret)return ret;
drivers/gpu/drm/i915/i915_gem_gtt.h +10 -0
@@ -403,6 +403,16 @@ static inline uint32_t gen6_pde_index(uint32_t addr) temp = min(temp, length), \ start += temp, length -= temp) +/* Clamp length to the next pagetab boundary */ +static inline uint64_t gen8_clamp_pt (uint64_t start,uint64_t length) +{ +uint64_t next_pt =ALIGN (start +1 ,1 <<GEN8_PDE_SHIFT ); +if (next_pt > (start + length)) +return length; + +return next_pt - start; +} +/* Clamp length to the next pagedir boundary */ static inline uint64_t gen8_clamp_pd (uint64_t start,uint64_t length) {
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH 18/24 drm/i915/bdw: Update pdp switch and point unused PDPs to scratch page
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
One important part of this patch is we now write a scratch page directory into any unused PDP descriptors. This matters for 2 reasons, first, we're not allowed to just use 0, or an invalid pointer, and second, we must wipe out any previous contents from the last context.
The latter point only matters with full PPGTT. The former point only effect platforms with less than 4GB memory.
v2: Updated commit message to point that we must set unused PDPs to the scratch page.
--- drivers/gpu/drm/i915/i915_gem_gtt.c | 29 ++++++++++++++++++----------- drivers/gpu/drm/i915/i915_gem_gtt.h | 5 ++++- 2 files changed, 22 insertions(+), 12 deletions(-)
drivers/gpu/drm/i915/i915_gem_gtt.c +18 -11
@@ -445,8 +445,9 @@ static struct i915_pagedir *alloc_pd_single(void) }/* Broadwell Page Directory Pointer Descriptors */ -static int gen8_write_pdp (struct intel_engine_cs *ring,unsigned entry, -uint64_t val) +static int gen8_write_pdp (struct intel_engine_cs *ring, +unsigned entry, +dma_addr_t addr) {int ret;
@@ -458,10 +459,10 @@ static int gen8_write_pdp(struct intel_engine_cs *ring, unsigned entry,intel_ring_emit (ring,MI_LOAD_REGISTER_IMM (1 ));intel_ring_emit (ring,GEN8_RING_PDP_UDW (ring, entry)); -intel_ring_emit (ring, (u32)(val >>32 )); +intel_ring_emit (ring,upper_32_bits (addr));intel_ring_emit (ring,MI_LOAD_REGISTER_IMM (1 ));intel_ring_emit (ring,GEN8_RING_PDP_LDW (ring, entry)); -intel_ring_emit (ring, (u32)(val)); +intel_ring_emit (ring,lower_32_bits (addr));intel_ring_advance (ring);return 0 ;
@@ -472,12 +473,12 @@ static int gen8_mm_switch(struct i915_hw_ppgtt *ppgtt, {int i, ret; -/* bit of a hack to find the actual last used pd */ -int used_pd = ppgtt->num_pd_entries /GEN8_PDES_PER_PAGE ; - -for (i = used_pd -1 ; i >=0 ; i--) { -dma_addr_t addr = ppgtt->pdp.pagedir[i]->daddr; - ret =gen8_write_pdp (ring, i, addr); +for (i =GEN8_LEGACY_PDPES -1 ; i >=0 ; i--) { +struct i915_pagedir *pd = ppgtt->pdp.pagedir[i]; +dma_addr_t pd_daddr = pd ? pd->daddr : ppgtt->scratch_pd->daddr; +/* The page directory might be NULL, but we need to clear out +* whatever the previous context might have used. */ + ret =gen8_write_pdp (ring, i, pd_daddr);if (ret)return ret; }
@@ -800,10 +801,16 @@ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size) ppgtt->base.start =0 ; ppgtt->base.total = size; + ppgtt->scratch_pd =alloc_pt_scratch (ppgtt->base.dev); +if (IS_ERR (ppgtt->scratch_pd)) +return PTR_ERR (ppgtt->scratch_pd); +/* 1. Do all our allocations for page directories and page tables. */ ret =gen8_ppgtt_alloc (ppgtt, ppgtt->base.start, ppgtt->base.total); -if (ret) +if (ret) { +free_pt_scratch (ppgtt->scratch_pd, ppgtt->base.dev);return ret; + }/* * 2. Create DMA mappings for the page directories and page tables.
drivers/gpu/drm/i915/i915_gem_gtt.h +4 -1
@@ -283,7 +283,10 @@ struct i915_hw_ppgtt {struct i915_pagedir pd; }; -struct i915_pagetab *scratch_pt; +union { +struct i915_pagetab *scratch_pt; +struct i915_pagetab *scratch_pd;/* Just need the daddr */ + };struct drm_i915_file_private *file_priv;
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH 19/24 drm/i915: num_pd_pages/num_pd_entries isn't useful
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
These values are never quite useful for dynamic allocations of the page tables. Getting rid of them will help prevent later confusion.
--- drivers/gpu/drm/i915/i915_debugfs.c | 2 -- drivers/gpu/drm/i915/i915_gem_gtt.c | 68 ++++++++++++------------------------- drivers/gpu/drm/i915/i915_gem_gtt.h | 7 ++-- 3 files changed, 27 insertions(+), 50 deletions(-)
drivers/gpu/drm/i915/i915_debugfs.c +0 -2
@@ -2117,8 +2117,6 @@ static void gen8_ppgtt_info(struct seq_file *m, struct drm_device *dev)if (!ppgtt)return ; -seq_printf (m,"Page directories: %d\n" , ppgtt->num_pd_pages); -seq_printf (m,"Page tables: %d\n" , ppgtt->num_pd_entries);for_each_ring (ring, dev_priv, unused) {seq_printf (m,"%s\n" , ring->name);for (i =0 ; i <4 ; i++) {
drivers/gpu/drm/i915/i915_gem_gtt.c +22 -46
160 lines not on this page · raw message
drivers/gpu/drm/i915/i915_gem_gtt.h +5 -2
@@ -276,8 +276,6 @@ struct i915_hw_ppgtt {struct i915_address_space base;struct kref ref;struct drm_mm_node node; -unsigned num_pd_entries; -unsigned num_pd_pages;/* gen8+ */ union {struct i915_pagedirpo pdp;struct i915_pagedir pd;
@@ -343,6 +341,11 @@ struct i915_gtt { temp = min(temp, (unsigned)length), \ start += temp, length -= temp) +#define gen6_for_all_pdes (pt, ppgtt, iter) \ + for (iter = 0, pt = ppgtt->pd.page_tables[iter]; \ + iter < gen6_pde_index(ppgtt->base.total); \ + pt = ppgtt->pd.page_tables[++iter]) +static inline uint32_t i915_pte_index (uint64_t address,uint32_t pde_shift) {const uint32_t mask =NUM_PTE (pde_shift) -1 ;
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH 20/24 drm/i915: Extract PPGTT param from pagedir alloc
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
Now that we don't need to trace num_pd_pages, we may as well kill all need for the PPGTT structure in the alloc_pagedirs. This is very useful for when we move to 48b addressing, and the PDP isn't the root of the page table structure.
The param is replaced with drm_device, which is an unavoidable wart throughout the series. (in other words, not extra flagrant).
--- drivers/gpu/drm/i915/i915_gem_gtt.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-)
drivers/gpu/drm/i915/i915_gem_gtt.c +2 -4
@@ -674,8 +674,6 @@ static int gen8_ppgtt_alloc_pagedirs(struct i915_pagedirpo *pdp, uint64_t start, uint64_t length) { -struct i915_hw_ppgtt *ppgtt = - container_of(pdp, struct i915_hw_ppgtt, pdp);struct i915_pagedir *unused;uint64_t temp;uint32_t pdpe;
@@ -686,7 +684,7 @@ static int gen8_ppgtt_alloc_pagedirs(struct i915_pagedirpo *pdp,gen8_for_each_pdpe (unused, pdp, start, length, temp, pdpe) {BUG_ON (unused); pdp->pagedir[pdpe] =alloc_pd_single (); -if (IS_ERR (ppgtt->pdp.pagedir[pdpe])) +if (IS_ERR (pdp->pagedir[pdpe])) goto unwind_out; }
@@ -694,7 +692,7 @@ static int gen8_ppgtt_alloc_pagedirs(struct i915_pagedirpo *pdp, unwind_out:while (pdpe--) -free_pd_single (ppgtt->pdp.pagedir[pdpe]); +free_pd_single (pdp->pagedir[pdpe]);return -ENOMEM ; }
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH 21/24 drm/i915/bdw: Split out mappings
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
When we do dynamic page table allocations for gen8, we'll need to have more control over how and when we map page tables, similar to gen6. In particular, DMA mappings for page directories/tables occur at allocation time.
This patch adds the functionality and calls it at init, which should have no functional change.
The PDPEs are still a special case for now. We'll need a function for that in the future as well.
--- drivers/gpu/drm/i915/i915_gem_gtt.c | 186 ++++++++++++++---------------------- 1 file changed, 72 insertions(+), 114 deletions(-)
drivers/gpu/drm/i915/i915_gem_gtt.c +72 -114
@@ -415,21 +415,23 @@ err_out:return ret; } -static void __free_pd_single (struct i915_pagedir *pd) +static void __free_pd_single (struct i915_pagedir *pd,struct drm_device *dev) { +i915_dma_unmap_single (pd, dev);__free_page (pd->page);kfree (pd); } -#define free_pd_single (pd) do { \ +#define free_pd_single (pd, dev) do { \ if ((pd)->page) { \ - __free_pd_single(pd); \ + __free_pd_single(pd, dev); \ } \ } while (0) -static struct i915_pagedir *alloc_pd_single (void ) +static struct i915_pagedir *alloc_pd_single (struct drm_device *dev) {struct i915_pagedir *pd; +int ret; pd =kzalloc (sizeof (*pd),GFP_KERNEL );if (!pd)
@@ -441,6 +443,13 @@ static struct i915_pagedir *alloc_pd_single(void)return ERR_PTR (-ENOMEM ); } + ret =i915_dma_map_px_single (pd, dev); +if (ret) { +__free_page (pd->page); +kfree (pd); +return ERR_PTR (ret); + } +return pd; }
@@ -580,6 +589,36 @@ static void gen8_ppgtt_insert_entries(struct i915_address_space *vm, } } +static void __gen8_do_map_pt (gen8_ppgtt_pde_t *pde, +struct i915_pagetab *pt, +struct drm_device *dev) +{ +gen8_ppgtt_pde_t entry = +gen8_pde_encode (dev, pt->daddr,I915_CACHE_LLC ); + *pde = entry; +} + +/* It's likely we'll map more than one pagetable at a time. This function will +* save us unnecessary kmap calls, but do no more functionally than multiple +* calls to map_pt. */ +static void gen8_map_pagetable_range (struct i915_pagedir *pd, +uint64_t start, +uint64_t length, +struct drm_device *dev) +{ +gen8_ppgtt_pde_t *pagedir =kmap_atomic (pd->page); +struct i915_pagetab *pt; +uint64_t temp, pde; + +gen8_for_each_pde (pt, pd, start, length, temp, pde) +__gen8_do_map_pt (pagedir + pde, pt, dev); + +if (!HAS_LLC (dev)) +drm_clflush_virt_range (pagedir,PAGE_SIZE ); + +kunmap_atomic (pagedir); +} +static void gen8_teardown_va_range (struct i915_address_space *vm,uint64_t start,uint64_t length) {
@@ -597,7 +636,7 @@ static void gen8_teardown_va_range(struct i915_address_space *vm,free_pt_single (pt, vm->dev); pd->page_tables[pde] =NULL ; } -free_pd_single (pd); +free_pd_single (pd, vm->dev); ppgtt->pdp.pagedir[pdpe] =NULL ; } }
@@ -629,9 +668,6 @@ static void gen8_ppgtt_unmap_pages(struct i915_hw_ppgtt *ppgtt)static void gen8_ppgtt_free (struct i915_hw_ppgtt *ppgtt) { -trace_i915_va_teardown (&ppgtt->base, - ppgtt->base.start, ppgtt->base.total, -VM_TO_TRACE_NAME (&ppgtt->base));gen8_teardown_va_range (&ppgtt->base, ppgtt->base.start, ppgtt->base.total); }
@@ -672,7 +708,8 @@ unwind_out:static int gen8_ppgtt_alloc_pagedirs (struct i915_pagedirpo *pdp,uint64_t start, -uint64_t length) +uint64_t length, +struct drm_device *dev) {struct i915_pagedir *unused;uint64_t temp;
@@ -683,7 +720,7 @@ static int gen8_ppgtt_alloc_pagedirs(struct i915_pagedirpo *pdp,gen8_for_each_pdpe (unused, pdp, start, length, temp, pdpe) {BUG_ON (unused); - pdp->pagedir[pdpe] =alloc_pd_single (); + pdp->pagedir[pdpe] =alloc_pd_single (dev);if (IS_ERR (pdp->pagedir[pdpe])) goto unwind_out; }
@@ -692,21 +729,25 @@ static int gen8_ppgtt_alloc_pagedirs(struct i915_pagedirpo *pdp, unwind_out:while (pdpe--) -free_pd_single (pdp->pagedir[pdpe]); +free_pd_single (pdp->pagedir[pdpe], dev);return -ENOMEM ; } -static int gen8_ppgtt_alloc (struct i915_hw_ppgtt *ppgtt, -uint64_t start, -uint64_t length) +static int gen8_alloc_va_range (struct i915_address_space *vm, +uint64_t start, +uint64_t length) { +struct i915_hw_ppgtt *ppgtt = + container_of(vm, struct i915_hw_ppgtt, base);struct i915_pagedir *pd; +const uint64_t orig_start = start;uint64_t temp;uint32_t pdpe;int ret; - ret =gen8_ppgtt_alloc_pagedirs (&ppgtt->pdp, start, length); + ret =gen8_ppgtt_alloc_pagedirs (&ppgtt->pdp, start, length, + ppgtt->base.dev);if (ret)return ret;
@@ -719,133 +760,50 @@ static int gen8_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt,return 0 ; -/* TODO: Check this for all cases */ err_out: -gen8_ppgtt_free (ppgtt); +gen8_teardown_va_range (vm, orig_start, start);return ret; } -static int gen8_ppgtt_setup_page_directories (struct i915_hw_ppgtt *ppgtt, -const int pd) -{ -dma_addr_t pd_addr; -int ret; - - pd_addr =pci_map_page (ppgtt->base.dev->pdev, - ppgtt->pdp.pagedir[pd]->page,0 , -PAGE_SIZE ,PCI_DMA_BIDIRECTIONAL ); - - ret =pci_dma_mapping_error (ppgtt->base.dev->pdev, pd_addr); -if (ret) -return ret; - - ppgtt->pdp.pagedir[pd]->daddr = pd_addr; - -return 0 ; -} - -static int gen8_ppgtt_setup_page_tables (struct i915_hw_ppgtt *ppgtt, -const int pd, -const int pt) -{ -dma_addr_t pt_addr; -struct i915_pagedir *pdir = ppgtt->pdp.pagedir[pd]; -struct i915_pagetab *ptab = pdir->page_tables[pt]; -struct page *p = ptab->page; -int ret; - - pt_addr =pci_map_page (ppgtt->base.dev->pdev, - p,0 ,PAGE_SIZE ,PCI_DMA_BIDIRECTIONAL ); - ret =pci_dma_mapping_error (ppgtt->base.dev->pdev, pt_addr); -if (ret) -return ret; - - ptab->daddr = pt_addr; - -return 0 ; -} -/** * GEN8 legacy ppgtt programming is accomplished through a max 4 PDP registers * with a net effect resembling a 2-level page table in normal x86 terms. Each * PDP represents 1GB of memory 4 * 512 * 512 * 4096 = 4GB legacy 32b address * space. * -* FIXME: split allocation into smaller pieces. For now we only ever do this -* once, but with full PPGTT, the multiple contiguous allocations will be bad. -* TODO: Do something with the size parameter */ static int gen8_ppgtt_init (struct i915_hw_ppgtt *ppgtt,uint64_t size) { -const int max_pdp =DIV_ROUND_UP (size,1 <<30 ); -int i, j, ret; - -if (size % (1 <<30 )) -DRM_INFO ("Pages will be wasted unless GTT size (%llu) is divisible by 1GB\n" , size); +struct i915_pagedir *pd; +uint64_t temp, start =0 ; +const uint64_t orig_length = size; +uint32_t pdpe; +int ret; ppgtt->base.start =0 ; ppgtt->base.total = size; + ppgtt->base.clear_range = gen8_ppgtt_clear_range; + ppgtt->base.insert_entries = gen8_ppgtt_insert_entries; + ppgtt->base.cleanup = gen8_ppgtt_cleanup; + ppgtt->switch_mm = gen8_mm_switch; ppgtt->scratch_pd =alloc_pt_scratch (ppgtt->base.dev);if (IS_ERR (ppgtt->scratch_pd))return PTR_ERR (ppgtt->scratch_pd); -/* 1. Do all our allocations for page directories and page tables. */ - ret =gen8_ppgtt_alloc (ppgtt, ppgtt->base.start, ppgtt->base.total); + ret =gen8_alloc_va_range (&ppgtt->base, start, size);if (ret) {free_pt_scratch (ppgtt->scratch_pd, ppgtt->base.dev);return ret; } -/* -* 2. Create DMA mappings for the page directories and page tables. -*/ -for (i =0 ; i < max_pdp; i++) { - ret =gen8_ppgtt_setup_page_directories (ppgtt, i); -if (ret) - goto bail; - -for (j =0 ; j <GEN8_PDES_PER_PAGE ; j++) { - ret =gen8_ppgtt_setup_page_tables (ppgtt, i, j); -if (ret) - goto bail; - } - } - -/* -* 3. Map all the page directory entires to point to the page tables -* we've allocated. -* -* For now, the PPGTT helper functions all require that the PDEs are -* plugged in correctly. So we do that now/here. For aliasing PPGTT, we -* will never need to touch the PDEs again. -*/ -for (i =0 ; i < max_pdp; i++) { -struct i915_pagedir *pd = ppgtt->pdp.pagedir[i]; -gen8_ppgtt_pde_t *pd_vaddr; - pd_vaddr =kmap_atomic (ppgtt->pdp.pagedir[i]->page); -for (j =0 ; j <GEN8_PDES_PER_PAGE ; j++) { -struct i915_pagetab *pt = pd->page_tables[j]; -dma_addr_t addr = pt->daddr; - pd_vaddr[j] =gen8_pde_encode (ppgtt->base.dev, addr, -I915_CACHE_LLC ); - } -if (!HAS_LLC (ppgtt->base.dev)) -drm_clflush_virt_range (pd_vaddr,PAGE_SIZE ); -kunmap_atomic (pd_vaddr); - } + start =0 ; + size = orig_length; - ppgtt->switch_mm = gen8_mm_switch; - ppgtt->base.clear_range = gen8_ppgtt_clear_range; - ppgtt->base.insert_entries = gen8_ppgtt_insert_entries; - ppgtt->base.cleanup = gen8_ppgtt_cleanup; +gen8_for_each_pdpe (pd, &ppgtt->pdp, start, size, temp, pdpe) +gen8_map_pagetable_range (pd, start, size, ppgtt->base.dev);return 0 ; - -bail: -gen8_ppgtt_unmap_pages (ppgtt); -gen8_ppgtt_free (ppgtt); -return ret; }static void gen6_dump_ppgtt (struct i915_hw_ppgtt *ppgtt,struct seq_file *m)
@@ -1281,7 +1239,7 @@ static void gen6_ppgtt_free(struct i915_hw_ppgtt *ppgtt)/* Consider putting this as part of pd free. */ free_pt_scratch (ppgtt->scratch_pt, ppgtt->base.dev); -free_pd_single (&ppgtt->pd); +free_pd_single (&ppgtt->pd, ppgtt->base.dev); }static void gen6_ppgtt_cleanup (struct i915_address_space *vm)
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH 22/24 drm/i915/bdw: begin bitmap tracking
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
Like with gen6/7, we can enable bitmap tracking with all the preallocations to make sure things actually don't blow up.
--- drivers/gpu/drm/i915/i915_gem_gtt.c | 121 +++++++++++++++++++++++++----------- drivers/gpu/drm/i915/i915_gem_gtt.h | 24 +++++++ 2 files changed, 108 insertions(+), 37 deletions(-)
drivers/gpu/drm/i915/i915_gem_gtt.c +84 -37
@@ -417,8 +417,12 @@ err_out:static void __free_pd_single (struct i915_pagedir *pd,struct drm_device *dev) { +WARN (!bitmap_empty (pd->used_pdes,GEN8_PDES_PER_PAGE ), +"Free page directory with %d used pages\n" , +bitmap_weight (pd->used_pdes,GEN8_PDES_PER_PAGE ));i915_dma_unmap_single (pd, dev);__free_page (pd->page); +kfree (pd->used_pdes);kfree (pd); }
@@ -431,26 +435,35 @@ static void __free_pd_single(struct i915_pagedir *pd, struct drm_device *dev)static struct i915_pagedir *alloc_pd_single (struct drm_device *dev) {struct i915_pagedir *pd; -int ret; +int ret = -ENOMEM ; pd =kzalloc (sizeof (*pd),GFP_KERNEL );if (!pd)return ERR_PTR (-ENOMEM ); + pd->used_pdes =kcalloc (BITS_TO_LONGS (GEN8_PDES_PER_PAGE ), +sizeof (*pd->used_pdes),GFP_KERNEL ); +if (!pd->used_pdes) + goto free_pd; + pd->page =alloc_page (GFP_KERNEL | __GFP_ZERO); -if (!pd->page) { -kfree (pd); -return ERR_PTR (-ENOMEM ); - } +if (!pd->page) + goto free_bitmap; ret =i915_dma_map_px_single (pd, dev); -if (ret) { -__free_page (pd->page); -kfree (pd); -return ERR_PTR (ret); - } +if (ret) + goto free_page;return pd; + +free_page: +__free_page (pd->page); +free_bitmap: +kfree (pd->used_pdes); +free_pd: +kfree (pd); + +return ERR_PTR (ret); }/* Broadwell Page Directory Pointer Descriptors */
@@ -632,36 +645,47 @@ static void gen8_teardown_va_range(struct i915_address_space *vm,gen8_for_each_pdpe (pd, &ppgtt->pdp, start, length, temp, pdpe) {uint64_t pd_len =gen8_clamp_pd (start, length);uint64_t pd_start = start; -gen8_for_each_pde (pt, pd, pd_start, pd_len, temp, pde) { -free_pt_single (pt, vm->dev); - pd->page_tables[pde] =NULL ; - } -free_pd_single (pd, vm->dev); - ppgtt->pdp.pagedir[pdpe] =NULL ; - } -} -static void gen8_ppgtt_unmap_pages (struct i915_hw_ppgtt *ppgtt) -{ -struct pci_dev *hwdev = ppgtt->base.dev->pdev; -int i, j; - -for (i =0 ; i <GEN8_PDES_PER_PAGE ; i++) { -/* TODO: In the future we'll support sparse mappings, so this -* will have to change. */ -if (!ppgtt->pdp.pagedir[i]->daddr) +/* Page directories might not be present since the macro rounds +* down, and up. +*/ +if (!pd) { +WARN (test_bit (pdpe, ppgtt->pdp.used_pdpes), +"PDPE %d is not allocated, but is reserved (%p)\n" , + pdpe, vm);continue ; + }else { +WARN (!test_bit (pdpe, ppgtt->pdp.used_pdpes), +"PDPE %d not reserved, but is allocated (%p)" , + pdpe, vm); + } -pci_unmap_page (hwdev, ppgtt->pdp.pagedir[i]->daddr,PAGE_SIZE , -PCI_DMA_BIDIRECTIONAL ); +gen8_for_each_pde (pt, pd, pd_start, pd_len, temp, pde) { +if (!pt) { +WARN (test_bit (pde, pd->used_pdes), +"PDE %d is not allocated, but is reserved (%p)\n" , + pde, vm); +continue ; + }else +WARN (!test_bit (pde, pd->used_pdes), +"PDE %d not reserved, but is allocated (%p)" , + pde, vm); + +bitmap_clear (pt->used_ptes, +gen8_pte_index (pd_start), +gen8_pte_count (pd_start, pd_len)); + +if (bitmap_empty (pt->used_ptes,GEN8_PTES_PER_PAGE )) { +free_pt_single (pt, vm->dev); + pd->page_tables[pde] =NULL ; +WARN_ON (!test_and_clear_bit (pde, pd->used_pdes)); + } + } -for (j =0 ; j <GEN8_PDES_PER_PAGE ; j++) { -struct i915_pagedir *pd = ppgtt->pdp.pagedir[i]; -struct i915_pagetab *pt = pd->page_tables[j]; -dma_addr_t addr = pt->daddr; -if (addr) -pci_unmap_page (hwdev, addr,PAGE_SIZE , -PCI_DMA_BIDIRECTIONAL ); +if (bitmap_empty (pd->used_pdes,GEN8_PDES_PER_PAGE )) { +free_pd_single (pd, vm->dev); + ppgtt->pdp.pagedir[pdpe] =NULL ; +WARN_ON (!test_and_clear_bit (pdpe, ppgtt->pdp.used_pdpes)); } } }
@@ -677,7 +701,6 @@ static void gen8_ppgtt_cleanup(struct i915_address_space *vm)struct i915_hw_ppgtt *ppgtt = container_of(vm, struct i915_hw_ppgtt, base); -gen8_ppgtt_unmap_pages (ppgtt);gen8_ppgtt_free (ppgtt); }
@@ -706,6 +729,7 @@ unwind_out:return -ENOMEM ; } +/* bitmap of new pagedirs */ static int gen8_ppgtt_alloc_pagedirs(struct i915_pagedirpo *pdp, uint64_t start, uint64_t length,
@@ -721,6 +745,7 @@ static int gen8_ppgtt_alloc_pagedirs(struct i915_pagedirpo *pdp,gen8_for_each_pdpe (unused, pdp, start, length, temp, pdpe) {BUG_ON (unused); pdp->pagedir[pdpe] =alloc_pd_single (dev); +if (IS_ERR (pdp->pagedir[pdpe])) goto unwind_out; }
@@ -742,10 +767,12 @@ static int gen8_alloc_va_range(struct i915_address_space *vm, container_of(vm, struct i915_hw_ppgtt, base);struct i915_pagedir *pd;const uint64_t orig_start = start; +const uint64_t orig_length = length;uint64_t temp;uint32_t pdpe;int ret; +/* Do the allocations first so we can easily bail out */ ret =gen8_ppgtt_alloc_pagedirs (&ppgtt->pdp, start, length, ppgtt->base.dev);if (ret)
@@ -758,6 +785,26 @@ static int gen8_alloc_va_range(struct i915_address_space *vm, goto err_out; } +/* Now mark everything we've touched as used. This doesn't allow for +* robust error checking, but it makes the code a hell of a lot simpler. +*/ + start = orig_start; + length = orig_length; + +gen8_for_each_pdpe (pd, &ppgtt->pdp, start, length, temp, pdpe) { +struct i915_pagetab *pt; +uint64_t pd_len =gen8_clamp_pd (start, length); +uint64_t pd_start = start; +uint32_t pde; +gen8_for_each_pde (pt, &ppgtt->pd, pd_start, pd_len, temp, pde) { +bitmap_set (pd->page_tables[pde]->used_ptes, +gen8_pte_index (start), +gen8_pte_count (start, length)); +set_bit (pde, pd->used_pdes); + } +set_bit (pdpe, ppgtt->pdp.used_pdpes); + } +return 0 ; err_out:
drivers/gpu/drm/i915/i915_gem_gtt.h +24 -0
@@ -206,11 +206,13 @@ struct i915_pagedir {dma_addr_t daddr; }; +unsigned long *used_pdes;struct i915_pagetab *page_tables[GEN6_PPGTT_PD_ENTRIES ]; };struct i915_pagedirpo {/* struct page *page; */ + DECLARE_BITMAP(used_pdpes, GEN8_LEGACY_PDPES);struct i915_pagedir *pagedir[GEN8_LEGACY_PDPES ]; };
@@ -449,6 +451,28 @@ static inline uint32_t gen8_pml4e_index(uint64_t address)BUG ();/* For 64B */ } +static inline size_t gen8_pte_count (uint64_t addr,uint64_t length) +{ +return i915_pte_count (addr, length,GEN8_PDE_SHIFT ); +} + +static inline size_t gen8_pde_count (uint64_t addr,uint64_t length) +{ +const uint32_t pdp_shift =GEN8_PDE_SHIFT +9 ; +const uint64_t mask = ~((1 << pdp_shift) -1 ); +uint64_t end; + +BUG_ON (length ==0 ); +BUG_ON (offset_in_page (addr|length)); + + end = addr + length; + +if ((addr & mask) != (end & mask)) +return GEN8_PDES_PER_PAGE -i915_pde_index (addr,GEN8_PDE_SHIFT ); + +return i915_pde_index (end,GEN8_PDE_SHIFT ) -i915_pde_index (addr,GEN8_PDE_SHIFT ); +} +int i915_gem_gtt_init (struct drm_device *dev);void i915_gem_init_global_gtt (struct drm_device *dev);void i915_global_gtt_cleanup (struct drm_device *dev);
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH 23/24 drm/i915/bdw: Dynamic page table allocations
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
This finishes off the dynamic page tables allocations, in the legacy 3 level style that already exists. Most everything has already been setup to this point, the patch finishes off the enabling by setting the appropriate function pointers.
Zombie tracking:
This could be a separate patch, but I found it helpful for debugging. Since we write page tables asynchronously with respect to the GPU using them, we can't actually free the page tables until we know the GPU won't use them. With this patch, that is always when the context dies. It would be possible to write a reaper to go through zombies and clean them up when under memory pressure. That exercise is left for the reader.
Scratch unused pages:
The object pages can get freed even if a page table still points to them. Like the zombie fix, we need to make sure we don't let our GPU access arbitrary memory when we've unmapped things.
v2: Update aliasing/true ppgtt allocate/teardown/clear functions for gen 6 & 7.
--- drivers/gpu/drm/i915/i915_gem_gtt.c | 378 +++++++++++++++++++++++++++++------- drivers/gpu/drm/i915/i915_gem_gtt.h | 16 +- 2 files changed, 327 insertions(+), 67 deletions(-)
drivers/gpu/drm/i915/i915_gem_gtt.c +312 -66
@@ -602,7 +602,7 @@ static void gen8_ppgtt_insert_entries(struct i915_address_space *vm, } } -static void __gen8_do_map_pt (gen8_ppgtt_pde_t *pde, +static void __gen8_do_map_pt (gen8_ppgtt_pde_t *const pde,struct i915_pagetab *pt,struct drm_device *dev) {
@@ -619,7 +619,7 @@ static void gen8_map_pagetable_range(struct i915_pagedir *pd,uint64_t length, struct drm_device *dev) { - gen8_ppgtt_pde_t *pagedir = kmap_atomic(pd->page); + gen8_ppgtt_pde_t * const pagedir = kmap_atomic(pd->page);struct i915_pagetab *pt;uint64_t temp, pde;
@@ -632,8 +632,9 @@ static void gen8_map_pagetable_range(struct i915_pagedir *pd,kunmap_atomic (pagedir); } -static void gen8_teardown_va_range (struct i915_address_space *vm, -uint64_t start,uint64_t length) +static void __gen8_teardown_va_range (struct i915_address_space *vm, +uint64_t start,uint64_t length, +bool dead) {struct i915_hw_ppgtt *ppgtt = container_of(vm, struct i915_hw_ppgtt, base);
@@ -655,6 +656,13 @@ static void gen8_teardown_va_range(struct i915_address_space *vm, pdpe, vm);continue ; }else { +if (dead && pd->zombie) { +WARN_ON (test_bit (pdpe, ppgtt->pdp.used_pdpes)); +free_pd_single (pd, vm->dev); + ppgtt->pdp.pagedir[pdpe] =NULL ; +continue ; + } +WARN (!test_bit (pdpe, ppgtt->pdp.used_pdpes),"PDPE %d not reserved, but is allocated (%p)" , pdpe, vm);
@@ -666,34 +674,64 @@ static void gen8_teardown_va_range(struct i915_address_space *vm,"PDE %d is not allocated, but is reserved (%p)\n" , pde, vm);continue ; - }else + }else { +if (dead && pt->zombie) { +WARN_ON (test_bit (pde, pd->used_pdes)); +free_pt_single (pt, vm->dev); + pd->page_tables[pde] =NULL ; +continue ; + }WARN (!test_bit (pde, pd->used_pdes),"PDE %d not reserved, but is allocated (%p)" , pde, vm); + }bitmap_clear (pt->used_ptes,gen8_pte_index (pd_start),gen8_pte_count (pd_start, pd_len));if (bitmap_empty (pt->used_ptes,GEN8_PTES_PER_PAGE )) { +WARN_ON (!test_and_clear_bit (pde, pd->used_pdes)); +if (!dead) { + pt->zombie =1 ; +continue ; + }free_pt_single (pt, vm->dev); pd->page_tables[pde] =NULL ; -WARN_ON (!test_and_clear_bit (pde, pd->used_pdes)); + } } +gen8_ppgtt_clear_range (vm, pd_start, pd_len, true); +if (bitmap_empty (pd->used_pdes,GEN8_PDES_PER_PAGE )) { +WARN_ON (!test_and_clear_bit (pdpe, ppgtt->pdp.used_pdpes)); +if (!dead) { +/* We've unmapped a possibly live context. Make +* note of it so we can clean it up later. */ + pd->zombie =1 ; +continue ; + }free_pd_single (pd, vm->dev); ppgtt->pdp.pagedir[pdpe] =NULL ; -WARN_ON (!test_and_clear_bit (pdpe, ppgtt->pdp.used_pdpes)); } } } +static void gen8_teardown_va_range (struct i915_address_space *vm, +uint64_t start,uint64_t length) +{ +__gen8_teardown_va_range (vm, start, length, false); +} +static void gen8_ppgtt_free (struct i915_hw_ppgtt *ppgtt) { -gen8_teardown_va_range (&ppgtt->base, - ppgtt->base.start, ppgtt->base.total); +trace_i915_va_teardown (&ppgtt->base, + ppgtt->base.start, ppgtt->base.total, +VM_TO_TRACE_NAME (&ppgtt->base)); +__gen8_teardown_va_range (&ppgtt->base, + ppgtt->base.start, ppgtt->base.total, + true); }static void gen8_ppgtt_cleanup (struct i915_address_space *vm)
@@ -704,58 +742,167 @@ static void gen8_ppgtt_cleanup(struct i915_address_space *vm)gen8_ppgtt_free (ppgtt); } -static int gen8_ppgtt_alloc_pagetabs (struct i915_pagedir *pd, +/** +* gen8_ppgtt_alloc_pagetabs() - Allocate page tables for VA range. +* @ppgtt: Master ppgtt structure. +* @pd: Page directory for this address range. +* @start: Starting virtual address to begin allocations. +* @length Size of the allocations. +* @new_pts: Bitmap set by function with new allocations. Likely used by the +* caller to free on error. +* +* Allocate the required number of page tables. Extremely similar to +* gen8_ppgtt_alloc_pagedirs(). The main difference is here we are limited by +* the page directory boundary (instead of the page directory pointer). That +* boundary is 1GB virtual. Therefore, unlike gen8_ppgtt_alloc_pagedirs(), it is +* possible, and likely that the caller will need to use multiple calls of this +* function to achieve the appropriate allocation. +* +* Return: 0 if success; negative error code otherwise. +*/ +static int gen8_ppgtt_alloc_pagetabs (struct i915_hw_ppgtt *ppgtt, +struct i915_pagedir *pd,uint64_t start,uint64_t length, -struct drm_device *dev) +unsigned long *new_pts) { -struct i915_pagetab *unused; +struct i915_pagetab *pt;uint64_t temp;uint32_t pde; -gen8_for_each_pde (unused, pd, start, length, temp, pde) { -BUG_ON (unused); - pd->page_tables[pde] =alloc_pt_single (dev); -if (IS_ERR (pd->page_tables[pde])) +gen8_for_each_pde (pt, pd, start, length, temp, pde) { +/* Don't reallocate page tables */ +if (pt) { +/* Scratch is never allocated this way */ +WARN_ON (pt->scratch); +/* If there is a zombie, we can reuse it and save time +* on the allocation. If we clear the zombie status and +* the caller somehow fails, we'll probably hit some +* assertions, so it's up to them to fix up the bitmaps. +*/ +continue ; + } + + pt =alloc_pt_single (ppgtt->base.dev); +if (IS_ERR (pt)) goto unwind_out; + + pd->page_tables[pde] = pt; +set_bit (pde, new_pts); }return 0 ; unwind_out: -while (pde--) -free_pt_single (pd->page_tables[pde], dev); +for_each_set_bit (pde, new_pts,GEN8_PDES_PER_PAGE ) +free_pt_single (pd->page_tables[pde], ppgtt->base.dev);return -ENOMEM ; } -/* bitmap of new pagedirs */ -static int gen8_ppgtt_alloc_pagedirs (struct i915_pagedirpo *pdp, +/** +* gen8_ppgtt_alloc_pagedirs() - Allocate page directories for VA range. +* @ppgtt: Master ppgtt structure. +* @pdp: Page directory pointer for this address range. +* @start: Starting virtual address to begin allocations. +* @length Size of the allocations. +* @new_pds Bitmap set by function with new allocations. Likely used by the +* caller to free on error. +* +* Allocate the required number of page directories starting at the pde index of +* @start, and ending at the pde index @start + @length. This function will skip +* over already allocated page directories within the range, and only allocate +* new ones, setting the appropriate pointer within the pdp as well as the +* correct position in the bitmap @new_pds. +* +* The function will only allocate the pages within the range for a give page +* directory pointer. In other words, if @start + @length straddles a virtually +* addressed PDP boundary (512GB for 4k pages), there will be more allocations +* required by the caller, This is not currently possible, and the BUG in the +* code will prevent it. +* +* Return: 0 if success; negative error code otherwise. +*/ +static int gen8_ppgtt_alloc_pagedirs (struct i915_hw_ppgtt *ppgtt, +struct i915_pagedirpo *pdp,uint64_t start,uint64_t length, -struct drm_device *dev) +unsigned long *new_pds) { -struct i915_pagedir *unused; +struct i915_pagedir *pd;uint64_t temp;uint32_t pdpe; +BUG_ON (!bitmap_empty (new_pds,GEN8_LEGACY_PDPES )); +/* FIXME: PPGTT container_of won't work for 64b */ BUG_ON ((start + length) >0x800000000ULL ); -gen8_for_each_pdpe (unused, pdp, start, length, temp, pdpe) { -BUG_ON (unused); - pdp->pagedir[pdpe] =alloc_pd_single (dev); +gen8_for_each_pdpe (pd, pdp, start, length, temp, pdpe) { +if (pd) +continue ; -if (IS_ERR (pdp->pagedir[pdpe])) + pd =alloc_pd_single (ppgtt->base.dev); +if (IS_ERR (pd)) goto unwind_out; + + pdp->pagedir[pdpe] = pd; +set_bit (pdpe, new_pds); }return 0 ; unwind_out: -while (pdpe--) -free_pd_single (pdp->pagedir[pdpe], dev); +for_each_set_bit (pdpe, new_pds,GEN8_LEGACY_PDPES ) +free_pd_single (pdp->pagedir[pdpe], ppgtt->base.dev); + +return -ENOMEM ; +} + +static inline void +free_gen8_temp_bitmaps (unsigned long *new_pds,unsigned long **new_pts) +{ +int i; +for (i =0 ; i <GEN8_LEGACY_PDPES ; i++) +kfree (new_pts[i]); +kfree (new_pts); +kfree (new_pds); +} + +/* Fills in the page directory bitmap, ant the array of page tables bitmap. Both +* of these are based on the number of PDPEs in the system. +*/ +int __must_check alloc_gen8_temp_bitmaps (unsigned long **new_pds, +unsigned long ***new_pts) +{ +int i; +unsigned long *pds; +unsigned long **pts; + pds =kcalloc (BITS_TO_LONGS (GEN8_LEGACY_PDPES ),sizeof (unsigned long ),GFP_KERNEL ); +if (!pds) +return -ENOMEM ; + + pts =kcalloc (GEN8_PDES_PER_PAGE ,sizeof (unsigned long *),GFP_KERNEL ); +if (!pts) { +kfree (pds); +return -ENOMEM ; + } + +for (i =0 ; i <GEN8_LEGACY_PDPES ; i++) { + pts[i] =kcalloc (BITS_TO_LONGS (GEN8_PDES_PER_PAGE ), +sizeof (unsigned long ),GFP_KERNEL ); +if (!pts[i]) + goto err_out; + } + + *new_pds = pds; + *new_pts = (unsigned long **)pts; + +return 0 ; + +err_out: +free_gen8_temp_bitmaps (pds, pts);return -ENOMEM ; }
@@ -765,6 +912,7 @@ static int gen8_alloc_va_range(struct i915_address_space *vm, {struct i915_hw_ppgtt *ppgtt = container_of(vm, struct i915_hw_ppgtt, base); +unsigned long *new_page_dirs, **new_page_tables;struct i915_pagedir *pd;const uint64_t orig_start = start;const uint64_t orig_length = length;
@@ -772,43 +920,103 @@ static int gen8_alloc_va_range(struct i915_address_space *vm,uint32_t pdpe;int ret; -/* Do the allocations first so we can easily bail out */ - ret =gen8_ppgtt_alloc_pagedirs (&ppgtt->pdp, start, length, - ppgtt->base.dev); +#ifndef CONFIG_64BIT +/* Disallow 64b address on 32b platforms. Nothing is wrong with doing +* this in hardware, but a lot of the drm code is not prepared to handle +* 64b offset on 32b platforms. */ +if (start + length >0x100000000ULL ) +return -E2BIG ; +#endif + +/* Wrap is never okay since we can only represent 48b, and we don't +* actually use the other side of the canonical address space. +*/ +if (WARN_ON (start + length < start)) +return -ERANGE ; + + ret =alloc_gen8_temp_bitmaps (&new_page_dirs, &new_page_tables);if (ret)return ret; +/* Do the allocations first so we can easily bail out */ + ret =gen8_ppgtt_alloc_pagedirs (ppgtt, &ppgtt->pdp, start, length, + new_page_dirs); +if (ret) { +free_gen8_temp_bitmaps (new_page_dirs, new_page_tables); +return ret; + } + +/* For every page directory referenced, allocate page tables */ gen8_for_each_pdpe (pd, &ppgtt->pdp, start, length, temp, pdpe) { - ret =gen8_ppgtt_alloc_pagetabs (pd, start, length, - ppgtt->base.dev); +bitmap_zero (new_page_tables[pdpe],GEN8_PDES_PER_PAGE ); + ret =gen8_ppgtt_alloc_pagetabs (ppgtt, pd, start, length, + new_page_tables[pdpe]);if (ret) goto err_out; } -/* Now mark everything we've touched as used. This doesn't allow for -* robust error checking, but it makes the code a hell of a lot simpler. -*/ start = orig_start; length = orig_length; +/* Allocations have completed successfully, so set the bitmaps, and do +* the mappings. */ gen8_for_each_pdpe (pd, &ppgtt->pdp, start, length, temp, pdpe) { +gen8_ppgtt_pde_t *const pagedir =kmap_atomic (pd->page);struct i915_pagetab *pt;uint64_t pd_len =gen8_clamp_pd (start, length);uint64_t pd_start = start;uint32_t pde; -gen8_for_each_pde (pt, &ppgtt->pd, pd_start, pd_len, temp, pde) { -bitmap_set (pd->page_tables[pde]->used_ptes, -gen8_pte_index (start), -gen8_pte_count (start, length)); + +/* Every pd should be allocated, we just did that above. */ +BUG_ON (!pd); + +gen8_for_each_pde (pt, pd, pd_start, pd_len, temp, pde) { +/* Same reasoning as pd */ +BUG_ON (!pt); +BUG_ON (!pd_len); +BUG_ON (!gen8_pte_count (pd_start, pd_len)); + +/* Set our used ptes within the page table */ +bitmap_set (pt->used_ptes, +gen8_pte_index (pd_start), +gen8_pte_count (pd_start, pd_len)); + +/* Our pde is now pointing to the pagetable, pt */ set_bit (pde, pd->used_pdes); + +/* Map the PDE to the page table */ +__gen8_do_map_pt (pagedir + pde, pt, vm->dev); + +/* NB: We haven't yet mapped ptes to pages. At this +* point we're still relying on insert_entries() */ + +/* No longer possible this page table is a zombie */ + pt->zombie =0 ; } + +if (!HAS_LLC (vm->dev)) +drm_clflush_virt_range (pagedir,PAGE_SIZE ); + +kunmap_atomic (pagedir); +set_bit (pdpe, ppgtt->pdp.used_pdpes); +/* This pd is officially not a zombie either */ + ppgtt->pdp.pagedir[pdpe]->zombie =0 ; } +free_gen8_temp_bitmaps (new_page_dirs, new_page_tables);return 0 ; err_out: -gen8_teardown_va_range (vm, orig_start, start); +while (pdpe--) { +for_each_set_bit (temp, new_page_tables[pdpe],GEN8_PDES_PER_PAGE ) +free_pt_single (pd->page_tables[temp], vm->dev); + } + +for_each_set_bit (pdpe, new_page_dirs,GEN8_LEGACY_PDPES ) +free_pd_single (ppgtt->pdp.pagedir[pdpe], vm->dev); + +free_gen8_temp_bitmaps (new_page_dirs, new_page_tables);return ret; }
@@ -819,37 +1027,68 @@ err_out:* space. * */ -static int gen8_ppgtt_init (struct i915_hw_ppgtt *ppgtt,uint64_t size) +static int gen8_ppgtt_init_common (struct i915_hw_ppgtt *ppgtt,uint64_t size) { -struct i915_pagedir *pd; -uint64_t temp, start =0 ; -const uint64_t orig_length = size; -uint32_t pdpe; -int ret; + ppgtt->scratch_pd =alloc_pt_scratch (ppgtt->base.dev); +if (IS_ERR (ppgtt->scratch_pd)) +return PTR_ERR (ppgtt->scratch_pd); ppgtt->base.start =0 ; ppgtt->base.total = size; - ppgtt->base.clear_range = gen8_ppgtt_clear_range; - ppgtt->base.insert_entries = gen8_ppgtt_insert_entries; ppgtt->base.cleanup = gen8_ppgtt_cleanup; + ppgtt->base.insert_entries = gen8_ppgtt_insert_entries; + ppgtt->switch_mm = gen8_mm_switch; - ppgtt->scratch_pd =alloc_pt_scratch (ppgtt->base.dev); -if (IS_ERR (ppgtt->scratch_pd)) -return PTR_ERR (ppgtt->scratch_pd); +return 0 ; +} +static int gen8_aliasing_ppgtt_init (struct i915_hw_ppgtt *ppgtt) +{ +struct drm_device *dev = ppgtt->base.dev; +struct drm_i915_private *dev_priv = dev->dev_private; +struct i915_pagedir *pd; +uint64_t temp, start =0 , size = dev_priv->gtt.base.total; +uint32_t pdpe; +int ret; + + ret =gen8_ppgtt_init_common (ppgtt, dev_priv->gtt.base.total); +if (ret) +return ret; + +/* Aliasing PPGTT has to always work and be mapped because of the way we +* use RESTORE_INHIBIT in the context switch. This will be fixed +* eventually. */ ret =gen8_alloc_va_range (&ppgtt->base, start, size);if (ret) {free_pt_scratch (ppgtt->scratch_pd, ppgtt->base.dev);return ret; } - start =0 ; - size = orig_length; -gen8_for_each_pdpe (pd, &ppgtt->pdp, start, size, temp, pdpe)gen8_map_pagetable_range (pd, start, size, ppgtt->base.dev); + ppgtt->base.allocate_va_range =NULL ; + ppgtt->base.teardown_va_range =NULL ; + ppgtt->base.clear_range = gen8_ppgtt_clear_range; + +return 0 ; +} + +static int gen8_ppgtt_init (struct i915_hw_ppgtt *ppgtt) +{ +struct drm_device *dev = ppgtt->base.dev; +struct drm_i915_private *dev_priv = dev->dev_private; +int ret; + + ret =gen8_ppgtt_init_common (ppgtt, dev_priv->gtt.base.total); +if (ret) +return ret; + + ppgtt->base.allocate_va_range = gen8_alloc_va_range; + ppgtt->base.teardown_va_range = gen8_teardown_va_range; + ppgtt->base.clear_range =NULL ; +return 0 ; }
@@ -1399,9 +1638,9 @@ static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt, bool aliasing)if (ret)return ret; - ppgtt->base.allocate_va_range = gen6_alloc_va_range; - ppgtt->base.teardown_va_range = gen6_teardown_va_range; - ppgtt->base.clear_range = gen6_ppgtt_clear_range; + ppgtt->base.allocate_va_range = aliasing ?NULL : gen6_alloc_va_range; + ppgtt->base.teardown_va_range = aliasing ?NULL : gen6_teardown_va_range; + ppgtt->base.clear_range = aliasing ? gen6_ppgtt_clear_range :NULL ; ppgtt->base.insert_entries = gen6_ppgtt_insert_entries; ppgtt->base.cleanup = gen6_ppgtt_cleanup; ppgtt->base.start =0 ;
@@ -1439,8 +1678,10 @@ static int __hw_ppgtt_init(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt,if (INTEL_INFO (dev)->gen <8 )return gen6_ppgtt_init (ppgtt, aliasing); +else if ((IS_GEN8 (dev) ||IS_GEN9 (dev)) && aliasing) +return gen8_aliasing_ppgtt_init (ppgtt);else if (IS_GEN8 (dev) ||IS_GEN9 (dev)) -return gen8_ppgtt_init (ppgtt, dev_priv->gtt.base.total); +return gen8_ppgtt_init (ppgtt);else BUG (); }
@@ -1454,8 +1695,9 @@ int i915_ppgtt_init(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt)kref_init (&ppgtt->ref);drm_mm_init (&ppgtt->base.mm, ppgtt->base.start, ppgtt->base.total); - ppgtt->base.clear_range (&ppgtt->base,0 , - ppgtt->base.total, true); +if (ppgtt->base.clear_range) + ppgtt->base.clear_range (&ppgtt->base,0 , + ppgtt->base.total, true);i915_init_vm (dev_priv, &ppgtt->base); }
@@ -1577,10 +1819,7 @@ ppgtt_bind_vma(struct i915_vma *vma,static void ppgtt_unbind_vma (struct i915_vma *vma) { - vma->vm->clear_range (vma->vm, - vma->node.start, - vma->obj->base.size, - true); +WARN_ON (vma->vm->teardown_va_range && vma->vm->clear_range);if (vma->vm->teardown_va_range) { trace_i915_va_teardown(vma->vm, vma->node.start, vma->node.size,
@@ -1589,7 +1828,14 @@ static void ppgtt_unbind_vma(struct i915_vma *vma) vma->vm->teardown_va_range (vma->vm, vma->node.start, vma->node.size);ppgtt_invalidate_tlbs (vma->vm); - } + }else if (vma->vm->clear_range) { + vma->vm->clear_range (vma->vm, + vma->node.start, + vma->obj->base.size, + true); + }else +BUG (); + }extern int intel_iommu_gfx_mapped;
drivers/gpu/drm/i915/i915_gem_gtt.h +15 -1
@@ -190,13 +190,26 @@ struct i915_vma { u32 flags); }; - +/* Zombies. We write page tables with the CPU, and hardware switches them with +* the GPU. As such, the only time we can safely remove a page table is when we +* know the context is idle. Since we have no good way to do this, we use the +* zombie. +* +* Under memory pressure, if the system is idle, zombies may be reaped. +* +* There are 3 states a page table can be in (not including scratch) +* bitmap = 0, zombie = 0: unallocated +* bitmap = 1, zombie = 0: allocated +* bitmap = 0, zombie = 1: zombie +* bitmap = 1, zombie = 1: invalid +*/ struct i915_pagetab {struct page *page;dma_addr_t daddr;unsigned long *used_ptes;unsigned int scratch:1 ; +unsigned zombie:1 ; };struct i915_pagedir {
@@ -208,6 +221,7 @@ struct i915_pagedir {unsigned long *used_pdes;struct i915_pagetab *page_tables[GEN6_PPGTT_PD_ENTRIES ]; + unsigned zombie:1; };struct i915_pagedirpo {
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH 24/24 drm/i915/bdw: Dynamic page table allocations in lrc mode
In reply to Michel ThierryLogic ring contexts need to know the PDPs when they are populated. With dynamic page table allocations, these PDPs may not exist yet.
Check if PDPs have been allocated and use the scratch page if they do not exist yet.
Before submission, update the PDPs in the logic ring context as PDPs have been allocated.
--- drivers/gpu/drm/i915/intel_lrc.c | 80 +++++++++++++++++++++++++++++++++++----- 1 file changed, 70 insertions(+), 10 deletions(-)
drivers/gpu/drm/i915/intel_lrc.c +70 -10
@@ -358,6 +358,7 @@ static void execlists_elsp_write(struct intel_engine_cs *ring,static int execlists_update_context (struct drm_i915_gem_object *ctx_obj,struct drm_i915_gem_object *ring_obj, +struct i915_hw_ppgtt *ppgtt,u32 tail) {struct page *page;
@@ -369,6 +370,40 @@ static int execlists_update_context(struct drm_i915_gem_object *ctx_obj, reg_state[CTX_RING_TAIL +1 ] = tail; reg_state[CTX_RING_BUFFER_START +1 ] =i915_gem_obj_ggtt_offset (ring_obj); +/* True PPGTT with dynamic page allocation: update PDP registers and +* point the unallocated PDPs to the scratch page +*/ +if (ppgtt) { +if (test_bit (3 , ppgtt->pdp.used_pdpes)) { + reg_state[CTX_PDP3_UDW +1 ] =upper_32_bits (ppgtt->pdp.pagedir[3 ]->daddr); + reg_state[CTX_PDP3_LDW +1 ] =lower_32_bits (ppgtt->pdp.pagedir[3 ]->daddr); + }else { + reg_state[CTX_PDP3_UDW +1 ] =upper_32_bits (ppgtt->scratch_pd->daddr); + reg_state[CTX_PDP3_LDW +1 ] =lower_32_bits (ppgtt->scratch_pd->daddr); + } +if (test_bit (2 , ppgtt->pdp.used_pdpes)) { + reg_state[CTX_PDP2_UDW +1 ] =upper_32_bits (ppgtt->pdp.pagedir[2 ]->daddr); + reg_state[CTX_PDP2_LDW +1 ] =lower_32_bits (ppgtt->pdp.pagedir[2 ]->daddr); + }else { + reg_state[CTX_PDP2_UDW +1 ] =upper_32_bits (ppgtt->scratch_pd->daddr); + reg_state[CTX_PDP2_LDW +1 ] =lower_32_bits (ppgtt->scratch_pd->daddr); + } +if (test_bit (1 , ppgtt->pdp.used_pdpes)) { + reg_state[CTX_PDP1_UDW +1 ] =upper_32_bits (ppgtt->pdp.pagedir[1 ]->daddr); + reg_state[CTX_PDP1_LDW +1 ] =lower_32_bits (ppgtt->pdp.pagedir[1 ]->daddr); + }else { + reg_state[CTX_PDP1_UDW +1 ] =upper_32_bits (ppgtt->scratch_pd->daddr); + reg_state[CTX_PDP1_LDW +1 ] =lower_32_bits (ppgtt->scratch_pd->daddr); + } +if (test_bit (0 , ppgtt->pdp.used_pdpes)) { + reg_state[CTX_PDP0_UDW +1 ] =upper_32_bits (ppgtt->pdp.pagedir[0 ]->daddr); + reg_state[CTX_PDP0_LDW +1 ] =lower_32_bits (ppgtt->pdp.pagedir[0 ]->daddr); + }else { + reg_state[CTX_PDP0_UDW +1 ] =upper_32_bits (ppgtt->scratch_pd->daddr); + reg_state[CTX_PDP0_LDW +1 ] =lower_32_bits (ppgtt->scratch_pd->daddr); + } + } +kunmap_atomic (reg_state);return 0 ;
@@ -387,7 +422,7 @@ static void execlists_submit_contexts(struct intel_engine_cs *ring,WARN_ON (!i915_gem_obj_is_pinned (ctx_obj0));WARN_ON (!i915_gem_obj_is_pinned (ringbuf0->obj)); -execlists_update_context (ctx_obj0, ringbuf0->obj, tail0); +execlists_update_context (ctx_obj0, ringbuf0->obj, to0->ppgtt, tail0);if (to1) { ringbuf1 = to1->engine[ring->id].ringbuf;
@@ -396,7 +431,7 @@ static void execlists_submit_contexts(struct intel_engine_cs *ring,WARN_ON (!i915_gem_obj_is_pinned (ctx_obj1));WARN_ON (!i915_gem_obj_is_pinned (ringbuf1->obj)); -execlists_update_context (ctx_obj1, ringbuf1->obj, tail1); +execlists_update_context (ctx_obj1, ringbuf1->obj, to1->ppgtt, tail1); }execlists_elsp_write (ring, ctx_obj0, ctx_obj1);
@@ -1731,14 +1766,39 @@ populate_lr_context(struct intel_context *ctx, struct drm_i915_gem_object *ctx_o reg_state[CTX_PDP1_LDW ] =GEN8_RING_PDP_LDW (ring,1 ); reg_state[CTX_PDP0_UDW ] =GEN8_RING_PDP_UDW (ring,0 ); reg_state[CTX_PDP0_LDW ] =GEN8_RING_PDP_LDW (ring,0 ); - reg_state[CTX_PDP3_UDW +1 ] =upper_32_bits (ppgtt->pdp.pagedir[3 ]->daddr); - reg_state[CTX_PDP3_LDW +1 ] =lower_32_bits (ppgtt->pdp.pagedir[3 ]->daddr); - reg_state[CTX_PDP2_UDW +1 ] =upper_32_bits (ppgtt->pdp.pagedir[2 ]->daddr); - reg_state[CTX_PDP2_LDW +1 ] =lower_32_bits (ppgtt->pdp.pagedir[2 ]->daddr); - reg_state[CTX_PDP1_UDW +1 ] =upper_32_bits (ppgtt->pdp.pagedir[1 ]->daddr); - reg_state[CTX_PDP1_LDW +1 ] =lower_32_bits (ppgtt->pdp.pagedir[1 ]->daddr); - reg_state[CTX_PDP0_UDW +1 ] =upper_32_bits (ppgtt->pdp.pagedir[0 ]->daddr); - reg_state[CTX_PDP0_LDW +1 ] =lower_32_bits (ppgtt->pdp.pagedir[0 ]->daddr); + +/* With dynamic page allocation, PDPs may not be allocated at this point, +* Point the unallocated PDPs to the scratch page +*/ +if (test_bit (3 , ppgtt->pdp.used_pdpes)) { + reg_state[CTX_PDP3_UDW +1 ] =upper_32_bits (ppgtt->pdp.pagedir[3 ]->daddr); + reg_state[CTX_PDP3_LDW +1 ] =lower_32_bits (ppgtt->pdp.pagedir[3 ]->daddr); + }else { + reg_state[CTX_PDP3_UDW +1 ] =upper_32_bits (ppgtt->scratch_pd->daddr); + reg_state[CTX_PDP3_LDW +1 ] =lower_32_bits (ppgtt->scratch_pd->daddr); + } +if (test_bit (2 , ppgtt->pdp.used_pdpes)) { + reg_state[CTX_PDP2_UDW +1 ] =upper_32_bits (ppgtt->pdp.pagedir[2 ]->daddr); + reg_state[CTX_PDP2_LDW +1 ] =lower_32_bits (ppgtt->pdp.pagedir[2 ]->daddr); + }else { + reg_state[CTX_PDP2_UDW +1 ] =upper_32_bits (ppgtt->scratch_pd->daddr); + reg_state[CTX_PDP2_LDW +1 ] =lower_32_bits (ppgtt->scratch_pd->daddr); + } +if (test_bit (1 , ppgtt->pdp.used_pdpes)) { + reg_state[CTX_PDP1_UDW +1 ] =upper_32_bits (ppgtt->pdp.pagedir[1 ]->daddr); + reg_state[CTX_PDP1_LDW +1 ] =lower_32_bits (ppgtt->pdp.pagedir[1 ]->daddr); + }else { + reg_state[CTX_PDP1_UDW +1 ] =upper_32_bits (ppgtt->scratch_pd->daddr); + reg_state[CTX_PDP1_LDW +1 ] =lower_32_bits (ppgtt->scratch_pd->daddr); + } +if (test_bit (0 , ppgtt->pdp.used_pdpes)) { + reg_state[CTX_PDP0_UDW +1 ] =upper_32_bits (ppgtt->pdp.pagedir[0 ]->daddr); + reg_state[CTX_PDP0_LDW +1 ] =lower_32_bits (ppgtt->pdp.pagedir[0 ]->daddr); + }else { + reg_state[CTX_PDP0_UDW +1 ] =upper_32_bits (ppgtt->scratch_pd->daddr); + reg_state[CTX_PDP0_LDW +1 ] =lower_32_bits (ppgtt->scratch_pd->daddr); + } +if (ring->id ==RCS ) { reg_state[CTX_LRI_HEADER_2 ] =MI_LOAD_REGISTER_IMM (1 ); reg_state[CTX_R_PWR_CLK_STATE ] =0x20c8 ;
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
Quoted 39 lines · Michel Thierry
This new version tries to remove as many unnecessary changes as possible from the previous RFC.
For GEN8, it has also been extended to work in logical ring submission (lrc) mode, as it will be the preferred mode of operation.
I also tried to update the lrc code at the same time the ppgtt refactoring occurred, leaving only one patch that is exclusively for lrc.This list can be seen in 3 parts:
[01-10] Include code rework for PPGTT (all GENs).
[11-14] Adds page table allocation for GEN6/GEN7
[15-24] Enables dynamic allocation in GEN8. It is enabled for both legacy and execlist submission modes.Ben Widawsky (23):
drm/i915: Add some extra guards inevict_vmdrm/i915/trace: Fix offsets for 64b
drm/i915: Rename toGEN8_LEGACY_PDPES
drm/i915: Setup less PPGTT on failed pagedir drm/i915/gen8: Un-hardcode number of page directories drm/i915: Range clearing is PPGTT agnostic
drm/i915: page table abstractions
drm/i915: Complete page table structures
drm/i915: Create page table allocators
drm/i915: Track GEN6 page table usage
drm/i915: Extract context switch skip logic
drm/i915: Track page table reload need
drm/i915: Initialize all contexts
drm/i915: Finish gen6/7 dynamic page table allocation drm/i915/bdw: Use dynamic allocation idioms on free drm/i915/bdw: pagedirs rework allocation
drm/i915/bdw: pagetable allocation rework
drm/i915/bdw: Update pdp switch and point unused PDPs to scratch page drm/i915: num_pd_pages/num_pd_entries isn't useful drm/i915: Extract PPGTT param from pagedir alloc drm/i915/bdw: Split out mappings
drm/i915/bdw: begin bitmap tracking
drm/i915/bdw: Dynamic page table allocations
Quoted 2 lines
Michel Thierry (1):
drm/i915/bdw: Dynamic page table allocations in lrc mode
Ok, I've tried to read through this series again and I definitely see a bit clearer, but it's still fairly confusing to me. I think that's just the long history of this patch series - often it seems to do something and then undo it again in some later patch. Which doesn't help understanding it.
I've replied with a few comments. Imo the way forward with this is to read, understand and review it from the beginning and merge while that's happening. It will probably take a few rounds until all the confusion is cleared up and we've reached the last patch.
-Daniel
Quoted 19 lines
drivers/gpu/drm/i915/i915_debugfs.c | 7 +- drivers/gpu/drm/i915/i915_drv.h | 7 + drivers/gpu/drm/i915/i915_gem_context.c | 62 +- drivers/gpu/drm/i915/i915_gem_evict.c | 3 + drivers/gpu/drm/i915/i915_gem_execbuffer.c | 11 + drivers/gpu/drm/i915/i915_gem_gtt.c | 1224 ++++++++++++++++++++-------- drivers/gpu/drm/i915/i915_gem_gtt.h | 252 +++++- drivers/gpu/drm/i915/i915_trace.h | 124 ++- drivers/gpu/drm/i915/intel_lrc.c | 80 +- 9 files changed, 1378 insertions(+), 392 deletions(-)Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
Signature · 7 lines
Daniel Vetter Software Engineer, Intel Corporation +41 (0) 79 365 57 48 - http://blog.ffwll.ch _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
Quoted 49 lines · Daniel Vetter
Quoted 39 lines · Michel Thierry
This new version tries to remove as many unnecessary changes as possible from the previous RFC.
For GEN8, it has also been extended to work in logical ring submission (lrc) mode, as it will be the preferred mode of operation.
I also tried to update the lrc code at the same time the ppgtt refactoring occurred, leaving only one patch that is exclusively for lrc.This list can be seen in 3 parts:
[01-10] Include code rework for PPGTT (all GENs).
[11-14] Adds page table allocation for GEN6/GEN7
[15-24] Enables dynamic allocation in GEN8. It is enabled for both legacy and execlist submission modes.Ben Widawsky (23):
drm/i915: Add some extra guards inevict_vmdrm/i915/trace: Fix offsets for 64b
drm/i915: Rename toGEN8_LEGACY_PDPES
drm/i915: Setup less PPGTT on failed pagedir drm/i915/gen8: Un-hardcode number of page directories drm/i915: Range clearing is PPGTT agnostic
drm/i915: page table abstractions
drm/i915: Complete page table structures
drm/i915: Create page table allocators
drm/i915: Track GEN6 page table usage
drm/i915: Extract context switch skip logic
drm/i915: Track page table reload need
drm/i915: Initialize all contexts
drm/i915: Finish gen6/7 dynamic page table allocation drm/i915/bdw: Use dynamic allocation idioms on free drm/i915/bdw: pagedirs rework allocation
drm/i915/bdw: pagetable allocation rework
drm/i915/bdw: Update pdp switch and point unused PDPs to scratch page drm/i915: num_pd_pages/num_pd_entries isn't useful drm/i915: Extract PPGTT param from pagedir alloc drm/i915/bdw: Split out mappings
drm/i915/bdw: begin bitmap tracking
drm/i915/bdw: Dynamic page table allocationsQuoted 2 lines
Michel Thierry (1):
drm/i915/bdw: Dynamic page table allocations in lrc modeOk, I've tried to read through this series again and I definitely see a bit clearer, but it's still fairly confusing to me. I think that's just the long history of this patch series - often it seems to do something and then undo it again in some later patch. Which doesn't help understanding it.
Quoted 4 lines
I've replied with a few comments. Imo the way forward with this is to read, understand and review it from the beginning and merge while that's happening. It will probably take a few rounds until all the confusion is cleared up and we've reached the last patch.
I honestly think this is starting off in the wrong direction. The first task, imo, is to make the current PD swappable. Then, we can introduce infrastructure to do deferred page allocation, hopefully combining with an approach to allow userspace to similarly defer their page allocation. -Chris
Signature · 5 lines
Chris Wilson, Intel Open Source Technology Centre _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
Quoted 55 lines · Chris Wilson
Quoted 49 lines · Daniel Vetter
Quoted 39 lines · Michel Thierry
This new version tries to remove as many unnecessary changes as possible from the previous RFC.
For GEN8, it has also been extended to work in logical ring submission (lrc) mode, as it will be the preferred mode of operation.
I also tried to update the lrc code at the same time the ppgtt refactoring occurred, leaving only one patch that is exclusively for lrc.This list can be seen in 3 parts:
[01-10] Include code rework for PPGTT (all GENs).
[11-14] Adds page table allocation for GEN6/GEN7
[15-24] Enables dynamic allocation in GEN8. It is enabled for both legacy and execlist submission modes.Ben Widawsky (23):
drm/i915: Add some extra guards inevict_vmdrm/i915/trace: Fix offsets for 64b
drm/i915: Rename toGEN8_LEGACY_PDPES
drm/i915: Setup less PPGTT on failed pagedir drm/i915/gen8: Un-hardcode number of page directories drm/i915: Range clearing is PPGTT agnostic
drm/i915: page table abstractions
drm/i915: Complete page table structures
drm/i915: Create page table allocators
drm/i915: Track GEN6 page table usage
drm/i915: Extract context switch skip logic
drm/i915: Track page table reload need
drm/i915: Initialize all contexts
drm/i915: Finish gen6/7 dynamic page table allocation drm/i915/bdw: Use dynamic allocation idioms on free drm/i915/bdw: pagedirs rework allocation
drm/i915/bdw: pagetable allocation rework
drm/i915/bdw: Update pdp switch and point unused PDPs to scratch page drm/i915: num_pd_pages/num_pd_entries isn't useful drm/i915: Extract PPGTT param from pagedir alloc drm/i915/bdw: Split out mappings
drm/i915/bdw: begin bitmap tracking
drm/i915/bdw: Dynamic page table allocationsQuoted 2 lines
Michel Thierry (1):
drm/i915/bdw: Dynamic page table allocations in lrc modeOk, I've tried to read through this series again and I definitely see a bit clearer, but it's still fairly confusing to me. I think that's just the long history of this patch series - often it seems to do something and then undo it again in some later patch. Which doesn't help understanding it.
Quoted 4 lines
I've replied with a few comments. Imo the way forward with this is to read, understand and review it from the beginning and merge while that's happening. It will probably take a few rounds until all the confusion is cleared up and we've reached the last patch.
Quoted 4 lines
I honestly think this is starting off in the wrong direction. The first task, imo, is to make the current PD swappable. Then, we can introduce infrastructure to do deferred page allocation, hopefully combining with an approach to allow userspace to similarly defer their page allocation.
Thus far things started to blow up because we need a bit too much memory in testcases so we need this. We'll probably need the pd swapping eventually, too. But only on gen7, and atm it doesn't look good for ppgtt on gen7 still. Given that I think the priorities are right for now.
But yeah we need to keep this in mind for gen7 full ppgtt. -Daniel
Signature · 7 lines
Daniel Vetter Software Engineer, Intel Corporation +41 (0) 79 365 57 48 - http://blog.ffwll.ch _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
Quoted 61 lines · Daniel Vetter
Quoted 55 lines · Chris Wilson
Quoted 49 lines · Daniel Vetter
Quoted 39 lines · Michel Thierry
This new version tries to remove as many unnecessary changes as possible from the previous RFC.
For GEN8, it has also been extended to work in logical ring submission (lrc) mode, as it will be the preferred mode of operation.
I also tried to update the lrc code at the same time the ppgtt refactoring occurred, leaving only one patch that is exclusively for lrc.This list can be seen in 3 parts:
[01-10] Include code rework for PPGTT (all GENs).
[11-14] Adds page table allocation for GEN6/GEN7
[15-24] Enables dynamic allocation in GEN8. It is enabled for both legacy and execlist submission modes.Ben Widawsky (23):
drm/i915: Add some extra guards inevict_vmdrm/i915/trace: Fix offsets for 64b
drm/i915: Rename toGEN8_LEGACY_PDPES
drm/i915: Setup less PPGTT on failed pagedir drm/i915/gen8: Un-hardcode number of page directories drm/i915: Range clearing is PPGTT agnostic
drm/i915: page table abstractions
drm/i915: Complete page table structures
drm/i915: Create page table allocators
drm/i915: Track GEN6 page table usage
drm/i915: Extract context switch skip logic
drm/i915: Track page table reload need
drm/i915: Initialize all contexts
drm/i915: Finish gen6/7 dynamic page table allocation drm/i915/bdw: Use dynamic allocation idioms on free drm/i915/bdw: pagedirs rework allocation
drm/i915/bdw: pagetable allocation rework
drm/i915/bdw: Update pdp switch and point unused PDPs to scratch page drm/i915: num_pd_pages/num_pd_entries isn't useful drm/i915: Extract PPGTT param from pagedir alloc drm/i915/bdw: Split out mappings
drm/i915/bdw: begin bitmap tracking
drm/i915/bdw: Dynamic page table allocationsQuoted 2 lines
Michel Thierry (1):
drm/i915/bdw: Dynamic page table allocations in lrc modeOk, I've tried to read through this series again and I definitely see a bit clearer, but it's still fairly confusing to me. I think that's just the long history of this patch series - often it seems to do something and then undo it again in some later patch. Which doesn't help understanding it.
Quoted 4 lines
I've replied with a few comments. Imo the way forward with this is to read, understand and review it from the beginning and merge while that's happening. It will probably take a few rounds until all the confusion is cleared up and we've reached the last patch.
Quoted 4 lines
I honestly think this is starting off in the wrong direction. The first task, imo, is to make the current PD swappable. Then, we can introduce infrastructure to do deferred page allocation, hopefully combining with an approach to allow userspace to similarly defer their page allocation.
Quoted 6 lines
Thus far things started to blow up because we need a bit too much memory in testcases so we need this. We'll probably need the pd swapping eventually, too. But only on gen7, and atm it doesn't look good for ppgtt on gen7 still. Given that I think the priorities are right for now.
But yeah we need to keep this in mind for gen7 full ppgtt.
Certainly making the toplevel PD swappable on BDW is less of a priority, since that should also be already done as part of execlists (and it is so much smaller). I would like to see the gen6/7 deferred allocation removed from this series, as I still consider it to be the wrong initial approach (and making the current gen7 full-ppgtt swappable is rather trivial).
-Chris
Signature · 5 lines
Chris Wilson, Intel Open Source Technology Centre _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
Quoted 67 lines · Chris Wilson
Quoted 66 lines · Daniel Vetter
Quoted 55 lines · Chris Wilson
Quoted 49 lines · Daniel Vetter
Quoted 39 lines · Michel Thierry
This new version tries to remove as many unnecessary changes as possible from the previous RFC.
For GEN8, it has also been extended to work in logical ring submission (lrc) mode, as it will be the preferred mode of operation.
I also tried to update the lrc code at the same time the ppgtt refactoring occurred, leaving only one patch that is exclusively for lrc.This list can be seen in 3 parts:
[01-10] Include code rework for PPGTT (all GENs).
[11-14] Adds page table allocation for GEN6/GEN7
[15-24] Enables dynamic allocation in GEN8. It is enabled for both legacy and execlist submission modes.Ben Widawsky (23):
drm/i915: Add some extra guards inevict_vmdrm/i915/trace: Fix offsets for 64b
drm/i915: Rename toGEN8_LEGACY_PDPES
drm/i915: Setup less PPGTT on failed pagedir drm/i915/gen8: Un-hardcode number of page directories drm/i915: Range clearing is PPGTT agnostic
drm/i915: page table abstractions
drm/i915: Complete page table structures
drm/i915: Create page table allocators
drm/i915: Track GEN6 page table usage
drm/i915: Extract context switch skip logic
drm/i915: Track page table reload need
drm/i915: Initialize all contexts
drm/i915: Finish gen6/7 dynamic page table allocation drm/i915/bdw: Use dynamic allocation idioms on free drm/i915/bdw: pagedirs rework allocation
drm/i915/bdw: pagetable allocation rework
drm/i915/bdw: Update pdp switch and point unused PDPs to scratch page drm/i915: num_pd_pages/num_pd_entries isn't useful drm/i915: Extract PPGTT param from pagedir alloc drm/i915/bdw: Split out mappings
drm/i915/bdw: begin bitmap tracking
drm/i915/bdw: Dynamic page table allocationsQuoted 2 lines
Michel Thierry (1):
drm/i915/bdw: Dynamic page table allocations in lrc modeOk, I've tried to read through this series again and I definitely see a bit clearer, but it's still fairly confusing to me. I think that's just the long history of this patch series - often it seems to do something and then undo it again in some later patch. Which doesn't help understanding it.
Quoted 4 lines
I've replied with a few comments. Imo the way forward with this is to read, understand and review it from the beginning and merge while that's happening. It will probably take a few rounds until all the confusion is cleared up and we've reached the last patch.
Quoted 4 lines
I honestly think this is starting off in the wrong direction. The first task, imo, is to make the current PD swappable. Then, we can introduce infrastructure to do deferred page allocation, hopefully combining with an approach to allow userspace to similarly defer their page allocation.
Thus far things started to blow up because we need a bit too much memory in testcases so we need this. We'll probably need the pd swapping eventually, too. But only on gen7, and atm it doesn't look good for ppgtt on gen7 still. Given that I think the priorities are right for now.
Quoted 8 lines
Quoted 1 line
But yeah we need to keep this in mind for gen7 full ppgtt.
Certainly making the toplevel PD swappable on BDW is less of a priority, since that should also be already done as part of execlists (and it is so much smaller). I would like to see the gen6/7 deferred allocation removed from this series, as I still consider it to be the wrong initial approach (and making the current gen7 full-ppgtt swappable is rather trivial).
I don't understand why you'd want to hold up the delayed pagetable alloc until the pd is evictable. Imo these are fairly orthogoanl issues: pd eviction tries to make ggtt space reclaimable, while deferred pagetable alloc ensure that we don't alloc 2M of pagetables (which are system memory) in the lower level when not needed. So imo we can go ahead with both in parallel. There might be some minor conflicts between this series and making pd ggtt blocks evictable around pd reloading, but noting fundamental.
Or are we talking past each another? Just wondering since you call it "swappable pd" and I can't really think of a way we could swap out pds to disk on gen7 (they just block ggtt space which can't be used for real ptes any more).
-Daniel
Signature · 7 lines
Daniel Vetter Software Engineer, Intel Corporation +41 (0) 79 365 57 48 - http://blog.ffwll.ch _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
Quoted 81 lines · Daniel Vetter
Quoted 64 lines · Chris Wilson
Quoted 63 lines · Daniel Vetter
Quoted 49 lines · Chris Wilson
Quoted 48 lines · Daniel Vetter
Quoted 39 lines · Michel Thierry
This new version tries to remove as many unnecessary changes as possible from the previous RFC.
For GEN8, it has also been extended to work in logical ring submission (lrc) mode, as it will be the preferred mode of operation.
I also tried to update the lrc code at the same time the ppgtt refactoring occurred, leaving only one patch that is exclusively for lrc.This list can be seen in 3 parts:
[01-10] Include code rework for PPGTT (all GENs).
[11-14] Adds page table allocation for GEN6/GEN7
[15-24] Enables dynamic allocation in GEN8. It is enabled for both legacy and execlist submission modes.Ben Widawsky (23):
drm/i915: Add some extra guards inevict_vmdrm/i915/trace: Fix offsets for 64b
drm/i915: Rename toGEN8_LEGACY_PDPES
drm/i915: Setup less PPGTT on failed pagedir drm/i915/gen8: Un-hardcode number of page directories drm/i915: Range clearing is PPGTT agnostic
drm/i915: page table abstractions
drm/i915: Complete page table structures
drm/i915: Create page table allocators
drm/i915: Track GEN6 page table usage
drm/i915: Extract context switch skip logic
drm/i915: Track page table reload need
drm/i915: Initialize all contexts
drm/i915: Finish gen6/7 dynamic page table allocation drm/i915/bdw: Use dynamic allocation idioms on free drm/i915/bdw: pagedirs rework allocation
drm/i915/bdw: pagetable allocation rework
drm/i915/bdw: Update pdp switch and point unused PDPs to scratch page drm/i915: num_pd_pages/num_pd_entries isn't useful drm/i915: Extract PPGTT param from pagedir alloc drm/i915/bdw: Split out mappings
drm/i915/bdw: begin bitmap tracking
drm/i915/bdw: Dynamic page table allocationsQuoted 2 lines
Michel Thierry (1):
drm/i915/bdw: Dynamic page table allocations in lrc modeOk, I've tried to read through this series again and I definitely see a bit clearer, but it's still fairly confusing to me. I think that's just the long history of this patch series - often it seems to do something and then undo it again in some later patch. Which doesn't help understanding it.
Quoted 8 lines
Quoted 4 lines
I've replied with a few comments. Imo the way forward with this is to read, understand and review it from the beginning and merge while that's happening. It will probably take a few rounds until all the confusion is cleared up and we've reached the last patch.
I honestly think this is starting off in the wrong direction. The first task, imo, is to make the current PD swappable. Then, we can introduce infrastructure to do deferred page allocation, hopefully combining with an approach to allow userspace to similarly defer their page allocation.
Thus far things started to blow up because we need a bit too much memory in testcases so we need this. We'll probably need the pd swapping eventually, too. But only on gen7, and atm it doesn't look good for ppgtt on gen7 still. Given that I think the priorities are right for now.
Quoted 7 lines
Quoted 1 line
But yeah we need to keep this in mind for gen7 full ppgtt.
Certainly making the toplevel PD swappable on BDW is less of a priority, since that should also be already done as part of execlists (and it is so much smaller). I would like to see the gen6/7 deferred allocation removed from this series, as I still consider it to be the wrong initial approach (and making the current gen7 full-ppgtt swappable is rather trivial).
I don't understand why you'd want to hold up the delayed pagetable alloc until the pd is evictable. Imo these are fairly orthogoanl issues: pd eviction tries to make ggtt space reclaimable, while deferred pagetable alloc ensure that we don't alloc 2M of pagetables (which are system memory) in the lower level when not needed. So imo we can go ahead with both in parallel. There might be some minor conflicts between this series and making pd ggtt blocks evictable around pd reloading, but noting fundamental.
Quoted 5 lines
Or are we talking past each another? Just wondering since you call it "swappable pd" and I can't really think of a way we could swap out pds to disk on gen7 (they just block ggtt space which can't be used for real ptes any more).
-Daniel
Thanks for the quick review.
I'll make the changes (and try to tidy up the commit messages). In particular, I agree it's better to decouple alloc and bind_vma.
-Michel
_______________________________________________
Intel-gfx mailing list
[email protected]
http://lists.freedesktop.org/mailman/listinfo/intel-gfx
Quoted 77 lines · Daniel Vetter
Quoted 67 lines · Chris Wilson
Quoted 66 lines · Daniel Vetter
Quoted 55 lines · Chris Wilson
Quoted 49 lines · Daniel Vetter
Quoted 39 lines · Michel Thierry
This new version tries to remove as many unnecessary changes as possible from the previous RFC.
For GEN8, it has also been extended to work in logical ring submission (lrc) mode, as it will be the preferred mode of operation.
I also tried to update the lrc code at the same time the ppgtt refactoring occurred, leaving only one patch that is exclusively for lrc.This list can be seen in 3 parts:
[01-10] Include code rework for PPGTT (all GENs).
[11-14] Adds page table allocation for GEN6/GEN7
[15-24] Enables dynamic allocation in GEN8. It is enabled for both legacy and execlist submission modes.Ben Widawsky (23):
drm/i915: Add some extra guards inevict_vmdrm/i915/trace: Fix offsets for 64b
drm/i915: Rename toGEN8_LEGACY_PDPES
drm/i915: Setup less PPGTT on failed pagedir drm/i915/gen8: Un-hardcode number of page directories drm/i915: Range clearing is PPGTT agnostic
drm/i915: page table abstractions
drm/i915: Complete page table structures
drm/i915: Create page table allocators
drm/i915: Track GEN6 page table usage
drm/i915: Extract context switch skip logic
drm/i915: Track page table reload need
drm/i915: Initialize all contexts
drm/i915: Finish gen6/7 dynamic page table allocation drm/i915/bdw: Use dynamic allocation idioms on free drm/i915/bdw: pagedirs rework allocation
drm/i915/bdw: pagetable allocation rework
drm/i915/bdw: Update pdp switch and point unused PDPs to scratch page drm/i915: num_pd_pages/num_pd_entries isn't useful drm/i915: Extract PPGTT param from pagedir alloc drm/i915/bdw: Split out mappings
drm/i915/bdw: begin bitmap tracking
drm/i915/bdw: Dynamic page table allocationsQuoted 2 lines
Michel Thierry (1):
drm/i915/bdw: Dynamic page table allocations in lrc modeOk, I've tried to read through this series again and I definitely see a bit clearer, but it's still fairly confusing to me. I think that's just the long history of this patch series - often it seems to do something and then undo it again in some later patch. Which doesn't help understanding it.
Quoted 4 lines
I've replied with a few comments. Imo the way forward with this is to read, understand and review it from the beginning and merge while that's happening. It will probably take a few rounds until all the confusion is cleared up and we've reached the last patch.
Quoted 4 lines
I honestly think this is starting off in the wrong direction. The first task, imo, is to make the current PD swappable. Then, we can introduce infrastructure to do deferred page allocation, hopefully combining with an approach to allow userspace to similarly defer their page allocation.
Thus far things started to blow up because we need a bit too much memory in testcases so we need this. We'll probably need the pd swapping eventually, too. But only on gen7, and atm it doesn't look good for ppgtt on gen7 still. Given that I think the priorities are right for now.
Quoted 8 lines
Quoted 1 line
But yeah we need to keep this in mind for gen7 full ppgtt.
Certainly making the toplevel PD swappable on BDW is less of a priority, since that should also be already done as part of execlists (and it is so much smaller). I would like to see the gen6/7 deferred allocation removed from this series, as I still consider it to be the wrong initial approach (and making the current gen7 full-ppgtt swappable is rather trivial).
Quoted 5 lines
I don't understand why you'd want to hold up the delayed pagetable alloc until the pd is evictable. Imo these are fairly orthogoanl issues: pd eviction tries to make ggtt space reclaimable, while deferred pagetable alloc ensure that we don't alloc 2M of pagetables (which are system memory) in the lower level when not needed.
There is only one level of PD on ivb, and it is 2M of pinned memory.
Quoted 4 lines
So imo we can go ahead with
both in parallel. There might be some minor conflicts between this series and making pd ggtt blocks evictable around pd reloading, but noting fundamental.
The trivial way to make it work, is to make the PD an obj. That then ties into the existing vma/obj management and debug/error infrastructure. Having deferred allocation for objects is also on the wishlist.
Quoted 4 lines
Or are we talking past each another? Just wondering since you call it "swappable pd" and I can't really think of a way we could swap out pds to disk on gen7 (they just block ggtt space which can't be used for real ptes any more).
Of course we can swap the inactive page directory tables. -Chris
Signature · 5 lines
Chris Wilson, Intel Open Source Technology Centre _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
Quoted 78 lines · Chris Wilson
Quoted 77 lines · Daniel Vetter
Quoted 75 lines · Chris Wilson
Quoted 61 lines · Daniel Vetter
Quoted 55 lines · Chris Wilson
Quoted 49 lines · Daniel Vetter
Quoted 39 lines · Michel Thierry
This new version tries to remove as many unnecessary changes as possible from the previous RFC.
For GEN8, it has also been extended to work in logical ring submission (lrc) mode, as it will be the preferred mode of operation.
I also tried to update the lrc code at the same time the ppgtt refactoring occurred, leaving only one patch that is exclusively for lrc.This list can be seen in 3 parts:
[01-10] Include code rework for PPGTT (all GENs).
[11-14] Adds page table allocation for GEN6/GEN7
[15-24] Enables dynamic allocation in GEN8. It is enabled for both legacy and execlist submission modes.Ben Widawsky (23):
drm/i915: Add some extra guards inevict_vmdrm/i915/trace: Fix offsets for 64b
drm/i915: Rename toGEN8_LEGACY_PDPES
drm/i915: Setup less PPGTT on failed pagedir drm/i915/gen8: Un-hardcode number of page directories drm/i915: Range clearing is PPGTT agnostic
drm/i915: page table abstractions
drm/i915: Complete page table structures
drm/i915: Create page table allocators
drm/i915: Track GEN6 page table usage
drm/i915: Extract context switch skip logic
drm/i915: Track page table reload need
drm/i915: Initialize all contexts
drm/i915: Finish gen6/7 dynamic page table allocation drm/i915/bdw: Use dynamic allocation idioms on free drm/i915/bdw: pagedirs rework allocation
drm/i915/bdw: pagetable allocation rework
drm/i915/bdw: Update pdp switch and point unused PDPs to scratch page drm/i915: num_pd_pages/num_pd_entries isn't useful drm/i915: Extract PPGTT param from pagedir alloc drm/i915/bdw: Split out mappings
drm/i915/bdw: begin bitmap tracking
drm/i915/bdw: Dynamic page table allocationsQuoted 2 lines
Michel Thierry (1):
drm/i915/bdw: Dynamic page table allocations in lrc modeOk, I've tried to read through this series again and I definitely see a bit clearer, but it's still fairly confusing to me. I think that's just the long history of this patch series - often it seems to do something and then undo it again in some later patch. Which doesn't help understanding it.
Quoted 4 lines
I've replied with a few comments. Imo the way forward with this is to read, understand and review it from the beginning and merge while that's happening. It will probably take a few rounds until all the confusion is cleared up and we've reached the last patch.
Quoted 4 lines
I honestly think this is starting off in the wrong direction. The first task, imo, is to make the current PD swappable. Then, we can introduce infrastructure to do deferred page allocation, hopefully combining with an approach to allow userspace to similarly defer their page allocation.
Quoted 6 lines
Thus far things started to blow up because we need a bit too much memory in testcases so we need this. We'll probably need the pd swapping eventually, too. But only on gen7, and atm it doesn't look good for ppgtt on gen7 still. Given that I think the priorities are right for now.
But yeah we need to keep this in mind for gen7 full ppgtt.
Certainly making the toplevel PD swappable on BDW is less of a priority, since that should also be already done as part of execlists (and it is so much smaller). I would like to see the gen6/7 deferred allocation removed from this series, as I still consider it to be the wrong initial approach (and making the current gen7 full-ppgtt swappable is rather trivial).
Quoted 7 lines
Quoted 5 lines
I don't understand why you'd want to hold up the delayed pagetable alloc until the pd is evictable. Imo these are fairly orthogoanl issues: pd eviction tries to make ggtt space reclaimable, while deferred pagetable alloc ensure that we don't alloc 2M of pagetables (which are system memory) in the lower level when not needed.
There is only one level of PD on ivb, and it is 2M of pinned memory.
I think we have a mixup of nomeclatura here. The thing that takes 512x4k on ivb are imo the pagetables, whereas the page directory entries are the 512 entries in the global gtt table. We can't swap those entries out, but we could evict them from the ggtt.
Quoted 10 lines
Quoted 4 lines
So imo we can go ahead with
both in parallel. There might be some minor conflicts between this series and making pd ggtt blocks evictable around pd reloading, but noting fundamental.The trivial way to make it work, is to make the PD an obj. That then ties into the existing vma/obj management and debug/error infrastructure. Having deferred allocation for objects is also on the wishlist.
Quoted 6 lines
Quoted 4 lines
Or are we talking past each another? Just wondering since you call it "swappable pd" and I can't really think of a way we could swap out pds to disk on gen7 (they just block ggtt space which can't be used for real ptes any more).
Of course we can swap the inactive page directory tables.
See above, I don't think we can. Furthermore the vm doesn't bother with making page tables or directories reclaimable afaik (we don't need to swap them out, we can simply free them when everything in them is evicted). So I don't think that's worth the bother.
If we really have a problem with pagetables we can add it to the oom
shrinker: Walking all vms and dropping all pagetables for completely empty vms would be conceptually really simple. And there shouldn't be anything left really when all objects are evicted for a ppgtt vm. -DAniel
Signature · 7 lines
Daniel Vetter Software Engineer, Intel Corporation +41 (0) 79 365 57 48 - http://blog.ffwll.ch _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
Quoted 85 lines · Daniel Vetter
Quoted 84 lines · Chris Wilson
Quoted 82 lines · Daniel Vetter
Quoted 67 lines · Chris Wilson
Quoted 66 lines · Daniel Vetter
Quoted 55 lines · Chris Wilson
Quoted 49 lines · Daniel Vetter
Quoted 39 lines · Michel Thierry
This new version tries to remove as many unnecessary changes as possible from the previous RFC.
For GEN8, it has also been extended to work in logical ring submission (lrc) mode, as it will be the preferred mode of operation.
I also tried to update the lrc code at the same time the ppgtt refactoring occurred, leaving only one patch that is exclusively for lrc.This list can be seen in 3 parts:
[01-10] Include code rework for PPGTT (all GENs).
[11-14] Adds page table allocation for GEN6/GEN7
[15-24] Enables dynamic allocation in GEN8. It is enabled for both legacy and execlist submission modes.Ben Widawsky (23):
drm/i915: Add some extra guards inevict_vmdrm/i915/trace: Fix offsets for 64b
drm/i915: Rename toGEN8_LEGACY_PDPES
drm/i915: Setup less PPGTT on failed pagedir drm/i915/gen8: Un-hardcode number of page directories drm/i915: Range clearing is PPGTT agnostic
drm/i915: page table abstractions
drm/i915: Complete page table structures
drm/i915: Create page table allocators
drm/i915: Track GEN6 page table usage
drm/i915: Extract context switch skip logic
drm/i915: Track page table reload need
drm/i915: Initialize all contexts
drm/i915: Finish gen6/7 dynamic page table allocation drm/i915/bdw: Use dynamic allocation idioms on free drm/i915/bdw: pagedirs rework allocation
drm/i915/bdw: pagetable allocation rework
drm/i915/bdw: Update pdp switch and point unused PDPs to scratch page drm/i915: num_pd_pages/num_pd_entries isn't useful drm/i915: Extract PPGTT param from pagedir alloc drm/i915/bdw: Split out mappings
drm/i915/bdw: begin bitmap tracking
drm/i915/bdw: Dynamic page table allocationsQuoted 2 lines
Michel Thierry (1):
drm/i915/bdw: Dynamic page table allocations in lrc modeOk, I've tried to read through this series again and I definitely see a bit clearer, but it's still fairly confusing to me. I think that's just the long history of this patch series - often it seems to do something and then undo it again in some later patch. Which doesn't help understanding it.
Quoted 4 lines
I've replied with a few comments. Imo the way forward with this is to read, understand and review it from the beginning and merge while that's happening. It will probably take a few rounds until all the confusion is cleared up and we've reached the last patch.
Quoted 4 lines
I honestly think this is starting off in the wrong direction. The first task, imo, is to make the current PD swappable. Then, we can introduce infrastructure to do deferred page allocation, hopefully combining with an approach to allow userspace to similarly defer their page allocation.
Thus far things started to blow up because we need a bit too much memory in testcases so we need this. We'll probably need the pd swapping eventually, too. But only on gen7, and atm it doesn't look good for ppgtt on gen7 still. Given that I think the priorities are right for now.
Quoted 8 lines
Quoted 1 line
But yeah we need to keep this in mind for gen7 full ppgtt.
Certainly making the toplevel PD swappable on BDW is less of a priority, since that should also be already done as part of execlists (and it is so much smaller). I would like to see the gen6/7 deferred allocation removed from this series, as I still consider it to be the wrong initial approach (and making the current gen7 full-ppgtt swappable is rather trivial).
I don't understand why you'd want to hold up the delayed pagetable alloc until the pd is evictable. Imo these are fairly orthogoanl issues: pd eviction tries to make ggtt space reclaimable, while deferred pagetable alloc ensure that we don't alloc 2M of pagetables (which are system memory) in the lower level when not needed.
Quoted 6 lines
Quoted 1 line
There is only one level of PD on ivb, and it is 2M of pinned memory.
I think we have a mixup of nomeclatura here. The thing that takes 512x4k on ivb are imo the pagetables, whereas the page directory entries are the 512 entries in the global gtt table. We can't swap those entries out, but we could evict them from the ggtt.
Page Directory vs Page Directory Entries. Of course they can be written to a swapfile when idle and repinned before use.
Quoted 22 lines
Quoted 10 lines
Quoted 4 lines
So imo we can go ahead with
both in parallel. There might be some minor conflicts between this series and making pd ggtt blocks evictable around pd reloading, but noting fundamental.The trivial way to make it work, is to make the PD an obj. That then ties into the existing vma/obj management and debug/error infrastructure. Having deferred allocation for objects is also on the wishlist.
Quoted 6 lines
Quoted 4 lines
Or are we talking past each another? Just wondering since you call it "swappable pd" and I can't really think of a way we could swap out pds to disk on gen7 (they just block ggtt space which can't be used for real ptes any more).
Of course we can swap the inactive page directory tables.
See above, I don't think we can. Furthermore the vm doesn't bother with making page tables or directories reclaimable afaik (we don't need to swap them out, we can simply free them when everything in them is evicted). So I don't think that's worth the bother.
Quoted 4 lines
If we really have a problem with pagetables we can add it to the oom
shrinker: Walking all vms and dropping all pagetables for completely empty vms would be conceptually really simple. And there shouldn't be anything left really when all objects are evicted for a ppgtt vm.
It is trivial to do so right now. If you look at the problem, you will see that is a requirement to do things like execlists correctly, rather than the current hodgepodge. In addition, we get a host of debugging insight for free.
-Chris
Signature · 5 lines
Chris Wilson, Intel Open Source Technology Centre _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
To end this particular thread, Daniel made a good point on IRC that his intent is to blow away the contents of the page tables if we need to swap, and then recreate them upon next use.
-Chris
Signature · 5 lines
Chris Wilson, Intel Open Source Technology Centre _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
v2 cover PPGTT dynamic page allocations
In reply to Michel ThierryAddressing comments from v1.
For GEN8, it has also been extended to work in logical ring submission (lrc) mode, as it will be the preferred mode of operation.
I also tried to update the lrc code at the same time the ppgtt refactoring occurred, leaving only one patch that is exclusively for lrc.
This list can be seen in 3 parts:
[01-10] Include code rework for PPGTT (all GENs).
[11-14] Adds page table allocation for GEN6/GEN7
[15-24] Enables dynamic allocation in GEN8. It is enabled for both legacy and execlist submission modes.
Ben Widawsky (23):
drm/i915: Add some extra guards in evict_vm drm/i915/trace: Fix offsets for 64b
drm/i915: Rename to GEN8_LEGACY_PDPES
drm/i915: Setup less PPGTT on failed pagedir drm/i915/gen8: Un-hardcode number of page directories drm/i915: Range clearing is PPGTT agnostic
drm/i915: page table abstractions
drm/i915: Complete page table structures
drm/i915: Create page table allocators
drm/i915: Track GEN6 page table usage
drm/i915: Extract context switch skip and pd load logic drm/i915: Track page table reload need
drm/i915: Initialize all contexts
drm/i915: Finish gen6/7 dynamic page table allocation drm/i915/bdw: Use dynamic allocation idioms on free drm/i915/bdw: pagedirs rework allocation
drm/i915/bdw: pagetable allocation rework
drm/i915/bdw: Update pdp switch and point unused PDPs to scratch page drm/i915: num_pd_pages/num_pd_entries isn't useful drm/i915: Extract PPGTT param from pagedir alloc drm/i915/bdw: Split out mappings
drm/i915/bdw: begin bitmap tracking
drm/i915/bdw: Dynamic page table allocations
Michel Thierry (1):
drm/i915/bdw: Dynamic page table allocations in lrc mode
drivers/gpu/drm/i915/i915_debugfs.c | 7 +- drivers/gpu/drm/i915/i915_gem.c | 11 + drivers/gpu/drm/i915/i915_gem_context.c | 62 +- drivers/gpu/drm/i915/i915_gem_evict.c | 3 + drivers/gpu/drm/i915/i915_gem_execbuffer.c | 11 + drivers/gpu/drm/i915/i915_gem_gtt.c | 1200 ++++++++++++++++++++-------- drivers/gpu/drm/i915/i915_gem_gtt.h | 250 +++++- drivers/gpu/drm/i915/i915_trace.h | 123 ++- drivers/gpu/drm/i915/intel_lrc.c | 80 +- 9 files changed, 1360 insertions(+), 387 deletions(-)
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH v2 1/24 drm/i915: Add some extra guards in evict_vm
From: Ben Widawsky <[email protected]>
v2: Use WARN_ONs (Daniel)
--- drivers/gpu/drm/i915/i915_gem_evict.c | 3 +++ 1 file changed, 3 insertions(+)
drivers/gpu/drm/i915/i915_gem_evict.c +3 -0
@@ -214,6 +214,7 @@ int i915_gem_evict_vm(struct i915_address_space *vm, bool do_idle)struct i915_vma *vma, *next;int ret; +WARN_ON (!mutex_is_locked (&vm->dev->struct_mutex));trace_i915_gem_evict_vm (vm);if (do_idle) {
@@ -222,6 +223,8 @@ int i915_gem_evict_vm(struct i915_address_space *vm, bool do_idle)return ret;i915_gem_retire_requests (vm->dev); + +WARN_ON (!list_empty (&vm->active_list)); }list_for_each_entry_safe (vma, next, &vm->inactive_list, mm_list)
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
Re: [PATCH v2 01/24] drm/i915: Add some extra guards in evict_vm
Quoted 7 lines · Michel Thierry
From: Ben Widawsky <[email protected]>
v2: Use
WARN_ONs(Daniel)Cc Daniel Vetter <[email protected]>Signed-off-by Ben Widawsky <[email protected]>Signed-off-by Michel Thierry <[email protected]>
Nitpick: Please put in some indication about what you've changed since this isn't precisly Ben's patch. Usually we go by putting a (v1) or similar behind the relavant sob line. Just for next time around. -Daniel
Quoted 32 lines
--- drivers/gpu/drm/i915/i915_gem_evict.c | 3 +++ 1 file changed, 3 insertions(+)drivers/gpu/drm/i915/i915_gem_evict.c +3
-0@@ -214,6 +214,7 @@ int i915_gem_evict_vm(struct i915_address_space *vm, bool do_idle)struct i915_vma *vma, *next;int ret; +WARN_ON (!mutex_is_locked (&vm->dev->struct_mutex));trace_i915_gem_evict_vm (vm);if (do_idle) {@@ -222,6 +223,8 @@ int i915_gem_evict_vm(struct i915_address_space *vm, bool do_idle)return ret;i915_gem_retire_requests (vm->dev); + +WARN_ON (!list_empty (&vm->active_list)); }list_for_each_entry_safe (vma, next, &vm->inactive_list, mm_list)Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
Signature · 7 lines
Daniel Vetter Software Engineer, Intel Corporation +41 (0) 79 365 57 48 - http://blog.ffwll.ch _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH v2 2/24 drm/i915/trace: Fix offsets for 64b
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
--- drivers/gpu/drm/i915/i915_trace.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-)
drivers/gpu/drm/i915/i915_trace.h +4 -4
@@ -115,7 +115,7 @@ TRACE_EVENT(i915_vma_bind,TP_STRUCT__entry (__field (struct drm_i915_gem_object *,obj ) __field(struct i915_address_space *, vm) -__field (u32, offset) +__field (u64, offset)__field (u32, size)__field (unsigned, flags) ),
@@ -128,7 +128,7 @@ TRACE_EVENT(i915_vma_bind, __entry->flags = flags; ), -TP_printk ("obj=%p, offset=%08x size=%x%s vm=%p" , +TP_printk ("obj=%p, offset=%016llx size=%x%s vm=%p" , __entry->obj, __entry->offset, __entry->size, __entry->flags &PIN_MAPPABLE ?", mappable" :"" , __entry->vm)
@@ -141,7 +141,7 @@ TRACE_EVENT(i915_vma_unbind,TP_STRUCT__entry (__field (struct drm_i915_gem_object *,obj ) __field(struct i915_address_space *, vm) -__field (u32, offset) +__field (u64, offset)__field (u32, size) ),
@@ -152,7 +152,7 @@ TRACE_EVENT(i915_vma_unbind, __entry->size = vma->node.size; ), -TP_printk ("obj=%p, offset=%08x size=%x vm=%p" , +TP_printk ("obj=%p, offset=%016llx size=%x vm=%p" , __entry->obj, __entry->offset, __entry->size, __entry->vm) );
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH v2 3/24 drm/i915: Rename to GEN8_LEGACY_PDPES
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
In gen8, 32b PPGTT has always had one "pdp" (it doesn't actually have one, but it resembles having one). The #define was confusing as is, and using "PDPE" is a much better description.
sed -i 's/GEN8_LEGACY_PDPS/GEN8_LEGACY_PDPES/' drivers/gpu/drm/i915/*.[ch]
It also matches the x86 pagetable terminology:
PTE = Page Table Entry - pagetable level 1 page PDE = Page Directory Entry - pagetable level 2 page PDPE = Page Directory Pointer Entry - pagetable level 3 page
And in the near future (for 48b addressing):
PML4E = Page Map Level 4 Entry
v2: Expanded information about Page Directory/Table nomenclature.
--- drivers/gpu/drm/i915/i915_gem_gtt.c | 6 +++--- drivers/gpu/drm/i915/i915_gem_gtt.h | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-)
drivers/gpu/drm/i915/i915_gem_gtt.c +3 -3
@@ -375,7 +375,7 @@ static void gen8_ppgtt_insert_entries(struct i915_address_space *vm, pt_vaddr =NULL ;for_each_sg_page (pages->sgl, &sg_iter, pages->nents,0 ) { -if (WARN_ON (pdpe >=GEN8_LEGACY_PDPS )) +if (WARN_ON (pdpe >=GEN8_LEGACY_PDPES ))break ;if (pt_vaddr ==NULL )
@@ -486,7 +486,7 @@ bail:static int gen8_ppgtt_allocate_page_tables (struct i915_hw_ppgtt *ppgtt,const int max_pdp) { -struct page **pt_pages[GEN8_LEGACY_PDPS ]; +struct page **pt_pages[GEN8_LEGACY_PDPES ];int i, ret;for (i =0 ; i < max_pdp; i++) {
@@ -537,7 +537,7 @@ static int gen8_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt,return -ENOMEM ; ppgtt->num_pd_pages =1 <<get_order (max_pdp <<PAGE_SHIFT ); -BUG_ON (ppgtt->num_pd_pages >GEN8_LEGACY_PDPS ); +BUG_ON (ppgtt->num_pd_pages >GEN8_LEGACY_PDPES );return 0 ; }
drivers/gpu/drm/i915/i915_gem_gtt.h +3 -3
@@ -88,7 +88,7 @@ typedef gen8_gtt_pte_t gen8_ppgtt_pde_t;#define GEN8_PDE_MASK 0x1ff#define GEN8_PTE_SHIFT 12#define GEN8_PTE_MASK 0x1ff -#define GEN8_LEGACY_PDPS 4 +#define GEN8_LEGACY_PDPES 4#define GEN8_PTES_PER_PAGE (PAGE_SIZE / sizeof(gen8_gtt_pte_t))#define GEN8_PDES_PER_PAGE (PAGE_SIZE / sizeof(gen8_ppgtt_pde_t))
@@ -273,12 +273,12 @@ struct i915_hw_ppgtt {unsigned num_pd_pages;/* gen8+ */ union {struct page **pt_pages; -struct page **gen8_pt_pages[GEN8_LEGACY_PDPS ]; +struct page **gen8_pt_pages[GEN8_LEGACY_PDPES ]; };struct page *pd_pages;union {uint32_t pd_offset; -dma_addr_t pd_dma_addr[GEN8_LEGACY_PDPS ]; +dma_addr_t pd_dma_addr[GEN8_LEGACY_PDPES ]; };union {dma_addr_t *pt_dma_addr;
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH v2 4/24 drm/i915: Setup less PPGTT on failed pagedir
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
The current code will both potentially print a WARN, and setup part of the PPGTT structure. Neither of these harm the current code, it is simply for clarity, and to perhaps prevent later bugs, or weird debug messages.
--- drivers/gpu/drm/i915/i915_gem_gtt.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-)
drivers/gpu/drm/i915/i915_gem_gtt.c +4 -1
@@ -1032,11 +1032,14 @@ alloc: goto alloc; } +if (ret) +return ret; +if (ppgtt->node.start < dev_priv->gtt.mappable_end)DRM_DEBUG ("Forced to use aperture for PDEs\n" ); ppgtt->num_pd_entries =GEN6_PPGTT_PD_ENTRIES ; -return ret; +return 0 ; }static int gen6_ppgtt_allocate_page_tables (struct i915_hw_ppgtt *ppgtt)
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH v2 5/24 drm/i915/gen8: Un-hardcode number of page directories
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
--- drivers/gpu/drm/i915/i915_gem_gtt.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
drivers/gpu/drm/i915/i915_gem_gtt.h +1 -1
@@ -282,7 +282,7 @@ struct i915_hw_ppgtt { };union {dma_addr_t *pt_dma_addr; -dma_addr_t *gen8_pt_dma_addr[4 ]; +dma_addr_t *gen8_pt_dma_addr[GEN8_LEGACY_PDPES ]; };struct drm_i915_file_private *file_priv;
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH v2 6/24 drm/i915: Range clearing is PPGTT agnostic
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
Therefore we can do it from our general init function. Eventually, I hope to have a lot more commonality like this. It won't arrive yet, but this was a nice easy one.
--- drivers/gpu/drm/i915/i915_gem_gtt.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-)
drivers/gpu/drm/i915/i915_gem_gtt.c +2 -4
@@ -672,8 +672,6 @@ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size) ppgtt->base.start =0 ; ppgtt->base.total = ppgtt->num_pd_entries *GEN8_PTES_PER_PAGE *PAGE_SIZE ; - ppgtt->base.clear_range (&ppgtt->base,0 , ppgtt->base.total, true); -DRM_DEBUG_DRIVER ("Allocated %d pages for page directories (%d wasted)\n" , ppgtt->num_pd_pages, ppgtt->num_pd_pages - max_pdp); DRM_DEBUG_DRIVER("Allocated %d pages for page tables (%lld wasted)\n",
@@ -1146,8 +1144,6 @@ static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt) ppgtt->pd_offset = ppgtt->node.start /PAGE_SIZE *sizeof (gen6_gtt_pte_t); - ppgtt->base.clear_range (&ppgtt->base,0 , ppgtt->base.total, true); -DRM_DEBUG_DRIVER ("Allocated pde space (%ldM) at GTT entry: %lx\n" , ppgtt->node.size >>20 , ppgtt->node.start /PAGE_SIZE );
@@ -1181,6 +1177,8 @@ int i915_ppgtt_init(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt)kref_init (&ppgtt->ref);drm_mm_init (&ppgtt->base.mm, ppgtt->base.start, ppgtt->base.total); + ppgtt->base.clear_range (&ppgtt->base,0 , + ppgtt->base.total, true);i915_init_vm (dev_priv, &ppgtt->base); }
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH v2 7/24 drm/i915: page table abstractions
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
When we move to dynamic page allocation, keeping pagedir and pagetabs as separate structures will help to break actions into simpler tasks.
To help transition the code nicely there is some wasted space in gen6/7. This will be ameliorated shortly.
v2: fixed mismatches after clean-up/rebase.
--- drivers/gpu/drm/i915/i915_gem_gtt.c | 177 ++++++++++++++++++------------------ drivers/gpu/drm/i915/i915_gem_gtt.h | 23 ++++- 2 files changed, 107 insertions(+), 93 deletions(-)
drivers/gpu/drm/i915/i915_gem_gtt.c +89 -88
@@ -334,7 +334,8 @@ static void gen8_ppgtt_clear_range(struct i915_address_space *vm, I915_CACHE_LLC, use_scratch);while (num_entries) { -struct page *page_table = ppgtt->gen8_pt_pages[pdpe][pde]; +struct i915_pagedir *pd = &ppgtt->pdp.pagedir[pdpe]; +struct page *page_table = pd->page_tables[pde].page; last_pte = pte + num_entries;if (last_pte >GEN8_PTES_PER_PAGE )
@@ -378,8 +379,12 @@ static void gen8_ppgtt_insert_entries(struct i915_address_space *vm,if (WARN_ON (pdpe >=GEN8_LEGACY_PDPES ))break ; -if (pt_vaddr ==NULL ) - pt_vaddr =kmap_atomic (ppgtt->gen8_pt_pages[pdpe][pde]); +if (pt_vaddr ==NULL ) { +struct i915_pagedir *pd = &ppgtt->pdp.pagedir[pdpe]; +struct page *page_table = pd->page_tables[pde].page; + + pt_vaddr =kmap_atomic (page_table); + } pt_vaddr[pte] = gen8_pte_encode(sg_page_iter_dma_address(&sg_iter),
@@ -403,29 +408,33 @@ static void gen8_ppgtt_insert_entries(struct i915_address_space *vm, } } -static void gen8_free_page_tables (struct page **pt_pages) +static void gen8_free_page_tables (struct i915_pagedir *pd) {int i; -if (pt_pages ==NULL ) +if (pd->page_tables ==NULL )return ;for (i =0 ; i <GEN8_PDES_PER_PAGE ; i++) -if (pt_pages[i]) -__free_pages (pt_pages[i],0 ); +if (pd->page_tables[i].page) +__free_page (pd->page_tables[i].page); } -static void gen8_ppgtt_free (const struct i915_hw_ppgtt *ppgtt) +static void gen8_free_page_directories (struct i915_pagedir *pd) +{ +kfree (pd->page_tables); +__free_page (pd->page); +} + +static void gen8_ppgtt_free (struct i915_hw_ppgtt *ppgtt) {int i;for (i =0 ; i < ppgtt->num_pd_pages; i++) { -gen8_free_page_tables (ppgtt->gen8_pt_pages[i]); -kfree (ppgtt->gen8_pt_pages[i]); +gen8_free_page_tables (&ppgtt->pdp.pagedir[i]); +gen8_free_page_directories (&ppgtt->pdp.pagedir[i]);kfree (ppgtt->gen8_pt_dma_addr[i]); } - -__free_pages (ppgtt->pd_pages,get_order (ppgtt->num_pd_pages <<PAGE_SHIFT )); }static void gen8_ppgtt_unmap_pages (struct i915_hw_ppgtt *ppgtt)
@@ -460,86 +469,75 @@ static void gen8_ppgtt_cleanup(struct i915_address_space *vm)gen8_ppgtt_free (ppgtt); } -static struct page **__gen8_alloc_page_tables (void ) +static int gen8_ppgtt_allocate_dma (struct i915_hw_ppgtt *ppgtt) { -struct page **pt_pages;int i; - pt_pages =kcalloc (GEN8_PDES_PER_PAGE ,sizeof (struct page *),GFP_KERNEL ); -if (!pt_pages) -return ERR_PTR (-ENOMEM ); - -for (i =0 ; i <GEN8_PDES_PER_PAGE ; i++) { - pt_pages[i] =alloc_page (GFP_KERNEL ); -if (!pt_pages[i]) - goto bail; +for (i =0 ; i < ppgtt->num_pd_pages; i++) { + ppgtt->gen8_pt_dma_addr[i] =kcalloc (GEN8_PDES_PER_PAGE , +sizeof (dma_addr_t), +GFP_KERNEL ); +if (!ppgtt->gen8_pt_dma_addr[i]) +return -ENOMEM ; } -return pt_pages; - -bail: -gen8_free_page_tables (pt_pages); -kfree (pt_pages); -return ERR_PTR (-ENOMEM ); +return 0 ; } -static int gen8_ppgtt_allocate_page_tables (struct i915_hw_ppgtt *ppgtt, -const int max_pdp) +static int gen8_ppgtt_allocate_page_tables (struct i915_hw_ppgtt *ppgtt) { -struct page **pt_pages[GEN8_LEGACY_PDPES ]; -int i, ret; +int i, j; -for (i =0 ; i < max_pdp; i++) { - pt_pages[i] =__gen8_alloc_page_tables (); -if (IS_ERR (pt_pages[i])) { - ret =PTR_ERR (pt_pages[i]); - goto unwind_out; +for (i =0 ; i < ppgtt->num_pd_pages; i++) { +for (j =0 ; j <GEN8_PDES_PER_PAGE ; j++) { +struct i915_pagetab *pt = &ppgtt->pdp.pagedir[i].page_tables[j]; + + pt->page =alloc_page (GFP_KERNEL | __GFP_ZERO); +if (!pt->page) + goto unwind_out; } } -/* NB: Avoid touching gen8_pt_pages until last to keep the allocation, -* "atomic" - for cleanup purposes. -*/ -for (i =0 ; i < max_pdp; i++) - ppgtt->gen8_pt_pages[i] = pt_pages[i]; -return 0 ; unwind_out: -while (i--) { -gen8_free_page_tables (pt_pages[i]); -kfree (pt_pages[i]); - } +while (i--) +gen8_free_page_tables (&ppgtt->pdp.pagedir[i]); -return ret; +return -ENOMEM ; } -static int gen8_ppgtt_allocate_dma (struct i915_hw_ppgtt *ppgtt) +static int gen8_ppgtt_allocate_page_directories (struct i915_hw_ppgtt *ppgtt, +const int max_pdp) {int i; -for (i =0 ; i < ppgtt->num_pd_pages; i++) { - ppgtt->gen8_pt_dma_addr[i] =kcalloc (GEN8_PDES_PER_PAGE , -sizeof (dma_addr_t), -GFP_KERNEL ); -if (!ppgtt->gen8_pt_dma_addr[i]) -return -ENOMEM ; - } +for (i =0 ; i < max_pdp; i++) { +struct i915_pagetab *pt; -return 0 ; -} + pt =kcalloc (GEN8_PDES_PER_PAGE ,sizeof (*pt),GFP_KERNEL ); +if (!pt) + goto unwind_out; -static int gen8_ppgtt_allocate_page_directories (struct i915_hw_ppgtt *ppgtt, -const int max_pdp) -{ - ppgtt->pd_pages =alloc_pages (GFP_KERNEL ,get_order (max_pdp <<PAGE_SHIFT )); -if (!ppgtt->pd_pages) -return -ENOMEM ; + ppgtt->pdp.pagedir[i].page =alloc_page (GFP_KERNEL ); +if (!ppgtt->pdp.pagedir[i].page) + goto unwind_out; + + ppgtt->pdp.pagedir[i].page_tables = pt; + } - ppgtt->num_pd_pages =1 <<get_order (max_pdp <<PAGE_SHIFT ); + ppgtt->num_pd_pages = max_pdp;BUG_ON (ppgtt->num_pd_pages >GEN8_LEGACY_PDPES );return 0 ; + +unwind_out: +while (i--) { +kfree (ppgtt->pdp.pagedir[i].page_tables); +__free_page (ppgtt->pdp.pagedir[i].page); + } + +return -ENOMEM ; } static int gen8_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt,
@@ -551,18 +549,19 @@ static int gen8_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt,if (ret)return ret; - ret =gen8_ppgtt_allocate_page_tables (ppgtt, max_pdp); -if (ret) { -__free_pages (ppgtt->pd_pages,get_order (max_pdp <<PAGE_SHIFT )); -return ret; - } + ret =gen8_ppgtt_allocate_page_tables (ppgtt); +if (ret) + goto err_out; ppgtt->num_pd_entries = max_pdp *GEN8_PDES_PER_PAGE ; ret =gen8_ppgtt_allocate_dma (ppgtt); -if (ret) -gen8_ppgtt_free (ppgtt); +if (!ret) +return ret; +/* TODO: Check this for all cases */ +err_out: +gen8_ppgtt_free (ppgtt);return ret; }
@@ -573,7 +572,7 @@ static int gen8_ppgtt_setup_page_directories(struct i915_hw_ppgtt *ppgtt,int ret; pd_addr =pci_map_page (ppgtt->base.dev->pdev, - &ppgtt->pd_pages[pd],0 , + ppgtt->pdp.pagedir[pd].page,0 ,PAGE_SIZE ,PCI_DMA_BIDIRECTIONAL ); ret =pci_dma_mapping_error (ppgtt->base.dev->pdev, pd_addr);
@@ -593,7 +592,7 @@ static int gen8_ppgtt_setup_page_tables(struct i915_hw_ppgtt *ppgtt,struct page *p;int ret; - p = ppgtt->gen8_pt_pages[pd][pt]; + p = ppgtt->pdp.pagedir[pd].page_tables[pt].page; pt_addr =pci_map_page (ppgtt->base.dev->pdev, p,0 ,PAGE_SIZE ,PCI_DMA_BIDIRECTIONAL ); ret =pci_dma_mapping_error (ppgtt->base.dev->pdev, pt_addr);
@@ -654,7 +653,7 @@ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size)*/ for (i =0 ; i < max_pdp; i++) {gen8_ppgtt_pde_t *pd_vaddr; - pd_vaddr =kmap_atomic (&ppgtt->pd_pages[i]); + pd_vaddr =kmap_atomic (ppgtt->pdp.pagedir[i].page);for (j =0 ; j <GEN8_PDES_PER_PAGE ; j++) {dma_addr_t addr = ppgtt->gen8_pt_dma_addr[i][j]; pd_vaddr[j] = gen8_pde_encode(ppgtt->base.dev, addr,
@@ -715,7 +714,7 @@ static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m) expected);seq_printf (m,"\tPDE: %x\n" , pd_entry); - pt_vaddr =kmap_atomic (ppgtt->pt_pages[pde]); + pt_vaddr =kmap_atomic (ppgtt->pd.page_tables[pde].page);for (pte =0 ; pte <I915_PPGTT_PT_ENTRIES ; pte+=4 ) {unsigned long va = (pde * PAGE_SIZE * I915_PPGTT_PT_ENTRIES) +
@@ -920,7 +919,7 @@ static void gen6_ppgtt_clear_range(struct i915_address_space *vm,if (last_pte >I915_PPGTT_PT_ENTRIES ) last_pte =I915_PPGTT_PT_ENTRIES ; - pt_vaddr =kmap_atomic (ppgtt->pt_pages[act_pt]); + pt_vaddr =kmap_atomic (ppgtt->pd.page_tables[act_pt].page);for (i = first_pte; i < last_pte; i++) pt_vaddr[i] = scratch_pte;
@@ -949,7 +948,7 @@ static void gen6_ppgtt_insert_entries(struct i915_address_space *vm, pt_vaddr =NULL ;for_each_sg_page (pages->sgl, &sg_iter, pages->nents,0 ) {if (pt_vaddr ==NULL ) - pt_vaddr =kmap_atomic (ppgtt->pt_pages[act_pt]); + pt_vaddr =kmap_atomic (ppgtt->pd.page_tables[act_pt].page); pt_vaddr[act_pte] = vm->pte_encode(sg_page_iter_dma_address(&sg_iter),
@@ -984,8 +983,8 @@ static void gen6_ppgtt_free(struct i915_hw_ppgtt *ppgtt)kfree (ppgtt->pt_dma_addr);for (i =0 ; i < ppgtt->num_pd_entries; i++) -__free_page (ppgtt->pt_pages[i]); -kfree (ppgtt->pt_pages); +__free_page (ppgtt->pd.page_tables[i].page); +kfree (ppgtt->pd.page_tables); }static void gen6_ppgtt_cleanup (struct i915_address_space *vm)
@@ -1042,22 +1041,22 @@ alloc:static int gen6_ppgtt_allocate_page_tables (struct i915_hw_ppgtt *ppgtt) { +struct i915_pagetab *pt;int i; - ppgtt->pt_pages =kcalloc (ppgtt->num_pd_entries,sizeof (struct page *), -GFP_KERNEL ); - -if (!ppgtt->pt_pages) + pt =kcalloc (ppgtt->num_pd_entries,sizeof (*pt),GFP_KERNEL ); +if (!pt)return -ENOMEM ;for (i =0 ; i < ppgtt->num_pd_entries; i++) { - ppgtt->pt_pages[i] =alloc_page (GFP_KERNEL ); -if (!ppgtt->pt_pages[i]) { + pt[i].page =alloc_page (GFP_KERNEL ); +if (!pt->page) {gen6_ppgtt_free (ppgtt);return -ENOMEM ; } } + ppgtt->pd.page_tables = pt;return 0 ; }
@@ -1092,9 +1091,11 @@ static int gen6_ppgtt_setup_page_tables(struct i915_hw_ppgtt *ppgtt)int i;for (i =0 ; i < ppgtt->num_pd_entries; i++) { +struct page *page;dma_addr_t pt_addr; - pt_addr =pci_map_page (dev->pdev, ppgtt->pt_pages[i],0 ,4096 , + page = ppgtt->pd.page_tables[i].page; + pt_addr =pci_map_page (dev->pdev, page,0 ,4096 ,PCI_DMA_BIDIRECTIONAL );if (pci_dma_mapping_error (dev->pdev, pt_addr)) {
@@ -1138,7 +1139,7 @@ static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt) ppgtt->base.insert_entries = gen6_ppgtt_insert_entries; ppgtt->base.cleanup = gen6_ppgtt_cleanup; ppgtt->base.start =0 ; - ppgtt->base.total = ppgtt->num_pd_entries *I915_PPGTT_PT_ENTRIES *PAGE_SIZE ; + ppgtt->base.total = ppgtt->num_pd_entries *I915_PPGTT_PT_ENTRIES *PAGE_SIZE ; ppgtt->debug_dump = gen6_dump_ppgtt; ppgtt->pd_offset =
drivers/gpu/drm/i915/i915_gem_gtt.h +18 -5
@@ -265,6 +265,20 @@ struct i915_gtt { unsigned long *mappable_end); }; +struct i915_pagetab { +struct page *page; +}; + +struct i915_pagedir { +struct page *page;/* NULL for GEN6-GEN7 */ +struct i915_pagetab *page_tables; +}; + +struct i915_pagedirpo { +/* struct page *page; */ +struct i915_pagedir pagedir[GEN8_LEGACY_PDPES ]; +}; +struct i915_hw_ppgtt {struct i915_address_space base;struct kref ref;
@@ -272,11 +286,6 @@ struct i915_hw_ppgtt {unsigned num_pd_entries;unsigned num_pd_pages;/* gen8+ */ union { -struct page **pt_pages; -struct page **gen8_pt_pages[GEN8_LEGACY_PDPES ]; - }; -struct page *pd_pages; -union {uint32_t pd_offset;dma_addr_t pd_dma_addr[GEN8_LEGACY_PDPES ]; };
@@ -284,6 +293,10 @@ struct i915_hw_ppgtt {dma_addr_t *pt_dma_addr;dma_addr_t *gen8_pt_dma_addr[GEN8_LEGACY_PDPES ]; }; +union { +struct i915_pagedirpo pdp; +struct i915_pagedir pd; + };struct drm_i915_file_private *file_priv;
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
Re: [PATCH v2 07/24] drm/i915: page table abstractions
Quoted 11 lines · Michel Thierry
drivers/gpu/drm/i915/i915_gem_gtt.h +3
-0@@ -265,6 +265,20 @@ struct i915_gtt { unsigned long *mappable_end); }; +struct i915_pagetab { +struct page *page; +};
drivers/gpu/drm/i915/i915_gem_gtt.h struct i915_gtt { · line 280
Quoted 10 lines
drivers/gpu/drm/i915/i915_gem_gtt.h +10
-0+ +struct i915_pagedir { +struct page *page;/* NULL for GEN6-GEN7 */ +struct i915_pagetab *page_tables; +}; + +struct i915_pagedirpo { +/* struct page *page; */ +struct i915_pagedir pagedir[GEN8_LEGACY_PDPES ]; +};
Well I still think the names assigned by intel in the IA prm are horrible and long-term we should just use the ones used by the core linux vm because they're much saner. But shortening them inconsistently like here really doesn't help. Can you please replace this with the relevant page_directory or page_directory_pointer or whatever to make it clearer?
Since it's only used once saving characters here doesn't seem a good tradeoff.
-Daniel
Quoted 34 lines
drivers/gpu/drm/i915/i915_gem_gtt.h +5
-5+struct i915_hw_ppgtt {struct i915_address_space base;struct kref ref;@@ -272,11 +286,6 @@ struct i915_hw_ppgtt {unsigned num_pd_entries;unsigned num_pd_pages;/* gen8+ */ union { -struct page **pt_pages; -struct page **gen8_pt_pages[GEN8_LEGACY_PDPES ]; - }; -struct page *pd_pages; -union {uint32_t pd_offset;dma_addr_t pd_dma_addr[GEN8_LEGACY_PDPES ]; };@@ -284,6 +293,10 @@ struct i915_hw_ppgtt {dma_addr_t *pt_dma_addr;dma_addr_t *gen8_pt_dma_addr[GEN8_LEGACY_PDPES ]; }; +union { +struct i915_pagedirpo pdp; +struct i915_pagedir pd; + };struct drm_i915_file_private *file_priv;Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
Signature · 7 lines
Daniel Vetter Software Engineer, Intel Corporation +41 (0) 79 365 57 48 - http://blog.ffwll.ch _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH v2 8/24 drm/i915: Complete page table structures
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
Move the remaining members over to the new page table structures.
This can be squashed with the previous commit if desire. The reasoning is the same as that patch. I simply felt it is easier to review if split.
v2: In lrc: s/ppgtt->pd_dma_addr[i]/ppgtt->pdp.pagedir[i].daddr/
--- drivers/gpu/drm/i915/i915_debugfs.c | 2 +- drivers/gpu/drm/i915/i915_gem_gtt.c | 85 +++++++++++++------------------------ drivers/gpu/drm/i915/i915_gem_gtt.h | 14 +++--- drivers/gpu/drm/i915/intel_lrc.c | 16 +++---- 4 files changed, 45 insertions(+), 72 deletions(-)
drivers/gpu/drm/i915/i915_debugfs.c +1 -1
@@ -2153,7 +2153,7 @@ static void gen6_ppgtt_info(struct seq_file *m, struct drm_device *dev)struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;seq_puts (m,"aliasing PPGTT:\n" ); -seq_printf (m,"pd gtt offset: 0x%08x\n" , ppgtt->pd_offset); +seq_printf (m,"pd gtt offset: 0x%08x\n" , ppgtt->pd.pd_offset); ppgtt->debug_dump (ppgtt, m); }
drivers/gpu/drm/i915/i915_gem_gtt.c +30 -55
220 lines not on this page · raw message
drivers/gpu/drm/i915/i915_gem_gtt.h +6 -8
@@ -267,10 +267,16 @@ struct i915_gtt {struct i915_pagetab {struct page *page; +dma_addr_t daddr; };struct i915_pagedir {struct page *page;/* NULL for GEN6-GEN7 */ +union { +uint32_t pd_offset; +dma_addr_t daddr; + }; +struct i915_pagetab *page_tables; };
@@ -286,14 +292,6 @@ struct i915_hw_ppgtt {unsigned num_pd_entries;unsigned num_pd_pages;/* gen8+ */ union { -uint32_t pd_offset; -dma_addr_t pd_dma_addr[GEN8_LEGACY_PDPES ]; - }; -union { -dma_addr_t *pt_dma_addr; -dma_addr_t *gen8_pt_dma_addr[GEN8_LEGACY_PDPES ]; - }; -union {struct i915_pagedirpo pdp;struct i915_pagedir pd; };
drivers/gpu/drm/i915/intel_lrc.c +8 -8
@@ -1731,14 +1731,14 @@ populate_lr_context(struct intel_context *ctx, struct drm_i915_gem_object *ctx_o reg_state[CTX_PDP1_LDW ] =GEN8_RING_PDP_LDW (ring,1 ); reg_state[CTX_PDP0_UDW ] =GEN8_RING_PDP_UDW (ring,0 ); reg_state[CTX_PDP0_LDW ] =GEN8_RING_PDP_LDW (ring,0 ); - reg_state[CTX_PDP3_UDW +1 ] =upper_32_bits (ppgtt->pd_dma_addr[3 ]); - reg_state[CTX_PDP3_LDW +1 ] =lower_32_bits (ppgtt->pd_dma_addr[3 ]); - reg_state[CTX_PDP2_UDW +1 ] =upper_32_bits (ppgtt->pd_dma_addr[2 ]); - reg_state[CTX_PDP2_LDW +1 ] =lower_32_bits (ppgtt->pd_dma_addr[2 ]); - reg_state[CTX_PDP1_UDW +1 ] =upper_32_bits (ppgtt->pd_dma_addr[1 ]); - reg_state[CTX_PDP1_LDW +1 ] =lower_32_bits (ppgtt->pd_dma_addr[1 ]); - reg_state[CTX_PDP0_UDW +1 ] =upper_32_bits (ppgtt->pd_dma_addr[0 ]); - reg_state[CTX_PDP0_LDW +1 ] =lower_32_bits (ppgtt->pd_dma_addr[0 ]); + reg_state[CTX_PDP3_UDW +1 ] =upper_32_bits (ppgtt->pdp.pagedir[3 ].daddr); + reg_state[CTX_PDP3_LDW +1 ] =lower_32_bits (ppgtt->pdp.pagedir[3 ].daddr); + reg_state[CTX_PDP2_UDW +1 ] =upper_32_bits (ppgtt->pdp.pagedir[2 ].daddr); + reg_state[CTX_PDP2_LDW +1 ] =lower_32_bits (ppgtt->pdp.pagedir[2 ].daddr); + reg_state[CTX_PDP1_UDW +1 ] =upper_32_bits (ppgtt->pdp.pagedir[1 ].daddr); + reg_state[CTX_PDP1_LDW +1 ] =lower_32_bits (ppgtt->pdp.pagedir[1 ].daddr); + reg_state[CTX_PDP0_UDW +1 ] =upper_32_bits (ppgtt->pdp.pagedir[0 ].daddr); + reg_state[CTX_PDP0_LDW +1 ] =lower_32_bits (ppgtt->pdp.pagedir[0 ].daddr);if (ring->id ==RCS ) { reg_state[CTX_LRI_HEADER_2 ] =MI_LOAD_REGISTER_IMM (1 ); reg_state[CTX_R_PWR_CLK_STATE ] =0x20c8 ;
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH v2 9/24 drm/i915: Create page table allocators
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
As we move toward dynamic page table allocation, it becomes much easier to manage our data structures if break do things less coarsely by breaking up all of our actions into individual tasks. This makes the code easier to write, read, and verify.
Aside from the dissection of the allocation functions, the patch statically allocates the page table structures without a page directory. This remains the same for all platforms,
The patch itself should not have much functional difference. The primary noticeable difference is the fact that page tables are no longer allocated, but rather statically declared as part of the page directory. This has non-zero overhead, but things gain non-trivial complexity as a result.
This patch exists for a few reasons:
1. Splitting out the functions allows easily combining GEN6 and GEN8 code. Page tables have no difference based on GEN8. As we'll see in a future patch when we add the DMA mappings to the allocations, it requires only one small change to make work, and error handling should just fall into place.
2. Unless we always want to allocate all page tables under a given PDE, we'll have to eventually break this up into an array of pointers (or pointer to pointer).
3. Having the discrete functions is easier to review, and understand. All allocations and frees now take place in just a couple of locations. Reviewing, and catching leaks should be easy.
4. Less important: the GFP flags are confined to one location, which makes playing around with such things trivial.
v2: Updated commit message to explain why this patch exists
v3: For lrc, s/pdp.pagedir[i].daddr/pdp.pagedir[i]->daddr/
--- drivers/gpu/drm/i915/i915_gem_gtt.c | 228 +++++++++++++++++++++++------------- drivers/gpu/drm/i915/i915_gem_gtt.h | 4 +- drivers/gpu/drm/i915/intel_lrc.c | 16 +-- 3 files changed, 155 insertions(+), 93 deletions(-)
drivers/gpu/drm/i915/i915_gem_gtt.c +145 -83
@@ -275,6 +275,102 @@ static gen6_gtt_pte_t iris_pte_encode(dma_addr_t addr,return pte; } +static void free_pt_single (struct i915_pagetab *pt) +{ +if (WARN_ON (!pt->page)) +return ; +__free_page (pt->page); +kfree (pt); +} + +static struct i915_pagetab *alloc_pt_single (void ) +{ +struct i915_pagetab *pt; + + pt =kzalloc (sizeof (*pt),GFP_KERNEL ); +if (!pt) +return ERR_PTR (-ENOMEM ); + + pt->page =alloc_page (GFP_KERNEL | __GFP_ZERO); +if (!pt->page) { +kfree (pt); +return ERR_PTR (-ENOMEM ); + } + +return pt; +} + +/** +* alloc_pt_range() - Allocate a multiple page tables +* @pd: The page directory which will have at least @count entries +* available to point to the allocated page tables. +* @pde: First page directory entry for which we are allocating. +* @count: Number of pages to allocate. +* +* Allocates multiple page table pages and sets the appropriate entries in the +* page table structure within the page directory. Function cleans up after +* itself on any failures. +* +* Return: 0 if allocation succeeded. +*/ +static int alloc_pt_range (struct i915_pagedir *pd,uint16_t pde,size_t count) +{ +int i, ret; + +/* 512 is the max page tables per pagedir on any platform. +* TODO: make WARN after patch series is done +*/ +BUG_ON (pde + count >GEN6_PPGTT_PD_ENTRIES ); + +for (i = pde; i < pde + count; i++) { +struct i915_pagetab *pt =alloc_pt_single (); +if (IS_ERR (pt)) { + ret =PTR_ERR (pt); + goto err_out; + } +WARN (pd->page_tables[i], +"Leaking page directory entry %d (%pa)\n" , + i, pd->page_tables[i]); + pd->page_tables[i] = pt; + } + +return 0 ; + +err_out: +while (i--) +free_pt_single (pd->page_tables[i]); +return ret; +} + +static void __free_pd_single (struct i915_pagedir *pd) +{ +__free_page (pd->page); +kfree (pd); +} + +#define free_pd_single (pd) do { \ + if ((pd)->page) { \ + __free_pd_single(pd); \ + } \ +} while (0) + +static struct i915_pagedir *alloc_pd_single (void ) +{ +struct i915_pagedir *pd; + + pd =kzalloc (sizeof (*pd),GFP_KERNEL ); +if (!pd) +return ERR_PTR (-ENOMEM ); + + pd->page =alloc_page (GFP_KERNEL | __GFP_ZERO); +if (!pd->page) { +kfree (pd); +return ERR_PTR (-ENOMEM ); + } + +return pd; +} +/* Broadwell Page Directory Pointer Descriptors */ static int gen8_write_pdp (struct intel_engine_cs *ring,unsigned entry,uint64_t val)
@@ -307,7 +403,7 @@ static int gen8_mm_switch(struct i915_hw_ppgtt *ppgtt,int used_pd = ppgtt->num_pd_entries /GEN8_PDES_PER_PAGE ;for (i = used_pd -1 ; i >=0 ; i--) { -dma_addr_t addr = ppgtt->pdp.pagedir[i].daddr; +dma_addr_t addr = ppgtt->pdp.pagedir[i]->daddr; ret =gen8_write_pdp (ring, i, addr);if (ret)return ret;
@@ -334,8 +430,9 @@ static void gen8_ppgtt_clear_range(struct i915_address_space *vm, I915_CACHE_LLC, use_scratch);while (num_entries) { -struct i915_pagedir *pd = &ppgtt->pdp.pagedir[pdpe]; -struct page *page_table = pd->page_tables[pde].page; +struct i915_pagedir *pd = ppgtt->pdp.pagedir[pdpe]; +struct i915_pagetab *pt = pd->page_tables[pde]; +struct page *page_table = pt->page; last_pte = pte + num_entries;if (last_pte >GEN8_PTES_PER_PAGE )
@@ -380,8 +477,9 @@ static void gen8_ppgtt_insert_entries(struct i915_address_space *vm,break ;if (pt_vaddr ==NULL ) { -struct i915_pagedir *pd = &ppgtt->pdp.pagedir[pdpe]; -struct page *page_table = pd->page_tables[pde].page; +struct i915_pagedir *pd = ppgtt->pdp.pagedir[pdpe]; +struct i915_pagetab *pt = pd->page_tables[pde]; +struct page *page_table = pt->page; pt_vaddr =kmap_atomic (page_table); }
@@ -412,18 +510,13 @@ static void gen8_free_page_tables(struct i915_pagedir *pd) {int i; -if (pd->page_tables ==NULL ) +if (!pd->page)return ; -for (i =0 ; i <GEN8_PDES_PER_PAGE ; i++) -if (pd->page_tables[i].page) -__free_page (pd->page_tables[i].page); -} - -static void gen8_free_page_directories (struct i915_pagedir *pd) -{ -kfree (pd->page_tables); -__free_page (pd->page); +for (i =0 ; i <GEN8_PDES_PER_PAGE ; i++) { +free_pt_single (pd->page_tables[i]); + pd->page_tables[i] =NULL ; + } }static void gen8_ppgtt_free (struct i915_hw_ppgtt *ppgtt)
@@ -431,8 +524,8 @@ static void gen8_ppgtt_free(struct i915_hw_ppgtt *ppgtt)int i;for (i =0 ; i < ppgtt->num_pd_pages; i++) { -gen8_free_page_tables (&ppgtt->pdp.pagedir[i]); -gen8_free_page_directories (&ppgtt->pdp.pagedir[i]); +gen8_free_page_tables (ppgtt->pdp.pagedir[i]); +free_pd_single (ppgtt->pdp.pagedir[i]); } }
@@ -444,14 +537,16 @@ static void gen8_ppgtt_unmap_pages(struct i915_hw_ppgtt *ppgtt)for (i =0 ; i < ppgtt->num_pd_pages; i++) {/* TODO: In the future we'll support sparse mappings, so this * will have to change. */ -if (!ppgtt->pdp.pagedir[i].daddr) +if (!ppgtt->pdp.pagedir[i]->daddr)continue ; -pci_unmap_page (hwdev, ppgtt->pdp.pagedir[i].daddr,PAGE_SIZE , +pci_unmap_page (hwdev, ppgtt->pdp.pagedir[i]->daddr,PAGE_SIZE ,PCI_DMA_BIDIRECTIONAL );for (j =0 ; j <GEN8_PDES_PER_PAGE ; j++) { -dma_addr_t addr = ppgtt->pdp.pagedir[i].page_tables[j].daddr; +struct i915_pagedir *pd = ppgtt->pdp.pagedir[i]; +struct i915_pagetab *pt = pd->page_tables[j]; +dma_addr_t addr = pt->daddr;if (addr)pci_unmap_page (hwdev, addr,PAGE_SIZE ,PCI_DMA_BIDIRECTIONAL );
@@ -470,25 +565,20 @@ static void gen8_ppgtt_cleanup(struct i915_address_space *vm)static int gen8_ppgtt_allocate_page_tables (struct i915_hw_ppgtt *ppgtt) { -int i, j; +int i, ret;for (i =0 ; i < ppgtt->num_pd_pages; i++) { -struct i915_pagedir *pd = &ppgtt->pdp.pagedir[i]; -for (j =0 ; j <GEN8_PDES_PER_PAGE ; j++) { -struct i915_pagetab *pt = &pd->page_tables[j]; - - pt->page =alloc_page (GFP_KERNEL | __GFP_ZERO); -if (!pt->page) - goto unwind_out; - - } + ret =alloc_pt_range (ppgtt->pdp.pagedir[i], +0 ,GEN8_PDES_PER_PAGE ); +if (ret) + goto unwind_out; }return 0 ; unwind_out:while (i--) -gen8_free_page_tables (&ppgtt->pdp.pagedir[i]); +gen8_free_page_tables (ppgtt->pdp.pagedir[i]);return -ENOMEM ; }
@@ -499,17 +589,9 @@ static int gen8_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt,int i;for (i =0 ; i < max_pdp; i++) { -struct i915_pagetab *pt; - - pt =kcalloc (GEN8_PDES_PER_PAGE ,sizeof (*pt),GFP_KERNEL ); -if (!pt) + ppgtt->pdp.pagedir[i] =alloc_pd_single (); +if (IS_ERR (ppgtt->pdp.pagedir[i])) goto unwind_out; - - ppgtt->pdp.pagedir[i].page =alloc_page (GFP_KERNEL ); -if (!ppgtt->pdp.pagedir[i].page) - goto unwind_out; - - ppgtt->pdp.pagedir[i].page_tables = pt; } ppgtt->num_pd_pages = max_pdp;
@@ -518,10 +600,8 @@ static int gen8_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt,return 0 ; unwind_out: -while (i--) { -kfree (ppgtt->pdp.pagedir[i].page_tables); -__free_page (ppgtt->pdp.pagedir[i].page); - } +while (i--) +free_pd_single (ppgtt->pdp.pagedir[i]);return -ENOMEM ; }
@@ -556,14 +636,14 @@ static int gen8_ppgtt_setup_page_directories(struct i915_hw_ppgtt *ppgtt,int ret; pd_addr =pci_map_page (ppgtt->base.dev->pdev, - ppgtt->pdp.pagedir[pd].page,0 , + ppgtt->pdp.pagedir[pd]->page,0 ,PAGE_SIZE ,PCI_DMA_BIDIRECTIONAL ); ret =pci_dma_mapping_error (ppgtt->base.dev->pdev, pd_addr);if (ret)return ret; - ppgtt->pdp.pagedir[pd].daddr = pd_addr; + ppgtt->pdp.pagedir[pd]->daddr = pd_addr;return 0 ; }
@@ -573,8 +653,8 @@ static int gen8_ppgtt_setup_page_tables(struct i915_hw_ppgtt *ppgtt, const int pt) {dma_addr_t pt_addr; -struct i915_pagedir *pdir = &ppgtt->pdp.pagedir[pd]; -struct i915_pagetab *ptab = &pdir->page_tables[pt]; +struct i915_pagedir *pdir = ppgtt->pdp.pagedir[pd]; +struct i915_pagetab *ptab = pdir->page_tables[pt];struct page *p = ptab->page;int ret;
@@ -637,10 +717,12 @@ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size)* will never need to touch the PDEs again. */ for (i =0 ; i < max_pdp; i++) { +struct i915_pagedir *pd = ppgtt->pdp.pagedir[i];gen8_ppgtt_pde_t *pd_vaddr; - pd_vaddr =kmap_atomic (ppgtt->pdp.pagedir[i].page); + pd_vaddr =kmap_atomic (ppgtt->pdp.pagedir[i]->page);for (j =0 ; j <GEN8_PDES_PER_PAGE ; j++) { -dma_addr_t addr = ppgtt->pdp.pagedir[i].page_tables[j].daddr; +struct i915_pagetab *pt = pd->page_tables[j]; +dma_addr_t addr = pt->daddr; pd_vaddr[j] =gen8_pde_encode (ppgtt->base.dev, addr,I915_CACHE_LLC ); }
@@ -689,7 +771,7 @@ static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m)for (pde =0 ; pde < ppgtt->num_pd_entries; pde++) {u32 expected;gen6_gtt_pte_t *pt_vaddr; -dma_addr_t pt_addr = ppgtt->pd.page_tables[pde].daddr; +dma_addr_t pt_addr = ppgtt->pd.page_tables[pde]->daddr; pd_entry =readl (pd_addr + pde); expected = (GEN6_PDE_ADDR_ENCODE (pt_addr) |GEN6_PDE_VALID );
@@ -700,7 +782,7 @@ static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m) expected);seq_printf (m,"\tPDE: %x\n" , pd_entry); - pt_vaddr =kmap_atomic (ppgtt->pd.page_tables[pde].page); + pt_vaddr =kmap_atomic (ppgtt->pd.page_tables[pde]->page);for (pte =0 ; pte <I915_PPGTT_PT_ENTRIES ; pte+=4 ) {unsigned long va = (pde * PAGE_SIZE * I915_PPGTT_PT_ENTRIES) +
@@ -739,7 +821,7 @@ static void gen6_write_pdes(struct i915_hw_ppgtt *ppgtt)for (i =0 ; i < ppgtt->num_pd_entries; i++) {dma_addr_t pt_addr; - pt_addr = ppgtt->pd.page_tables[i].daddr; + pt_addr = ppgtt->pd.page_tables[i]->daddr; pd_entry =GEN6_PDE_ADDR_ENCODE (pt_addr); pd_entry |=GEN6_PDE_VALID ;
@@ -905,7 +987,7 @@ static void gen6_ppgtt_clear_range(struct i915_address_space *vm,if (last_pte >I915_PPGTT_PT_ENTRIES ) last_pte =I915_PPGTT_PT_ENTRIES ; - pt_vaddr =kmap_atomic (ppgtt->pd.page_tables[act_pt].page); + pt_vaddr =kmap_atomic (ppgtt->pd.page_tables[act_pt]->page);for (i = first_pte; i < last_pte; i++) pt_vaddr[i] = scratch_pte;
@@ -934,7 +1016,7 @@ static void gen6_ppgtt_insert_entries(struct i915_address_space *vm, pt_vaddr =NULL ;for_each_sg_page (pages->sgl, &sg_iter, pages->nents,0 ) {if (pt_vaddr ==NULL ) - pt_vaddr =kmap_atomic (ppgtt->pd.page_tables[act_pt].page); + pt_vaddr =kmap_atomic (ppgtt->pd.page_tables[act_pt]->page); pt_vaddr[act_pte] = vm->pte_encode(sg_page_iter_dma_address(&sg_iter),
@@ -957,7 +1039,7 @@ static void gen6_ppgtt_unmap_pages(struct i915_hw_ppgtt *ppgtt)for (i =0 ; i < ppgtt->num_pd_entries; i++)pci_unmap_page (ppgtt->base.dev->pdev, - ppgtt->pd.page_tables[i].daddr, + ppgtt->pd.page_tables[i]->daddr,4096 ,PCI_DMA_BIDIRECTIONAL ); }
@@ -966,8 +1048,9 @@ static void gen6_ppgtt_free(struct i915_hw_ppgtt *ppgtt)int i;for (i =0 ; i < ppgtt->num_pd_entries; i++) -__free_page (ppgtt->pd.page_tables[i].page); -kfree (ppgtt->pd.page_tables); +free_pt_single (ppgtt->pd.page_tables[i]); + +free_pd_single (&ppgtt->pd); }static void gen6_ppgtt_cleanup (struct i915_address_space *vm)
@@ -1022,27 +1105,6 @@ alloc:return 0 ; } -static int gen6_ppgtt_allocate_page_tables (struct i915_hw_ppgtt *ppgtt) -{ -struct i915_pagetab *pt; -int i; - - pt =kcalloc (ppgtt->num_pd_entries,sizeof (*pt),GFP_KERNEL ); -if (!pt) -return -ENOMEM ; - -for (i =0 ; i < ppgtt->num_pd_entries; i++) { - pt[i].page =alloc_page (GFP_KERNEL ); -if (!pt->page) { -gen6_ppgtt_free (ppgtt); -return -ENOMEM ; - } - } - - ppgtt->pd.page_tables = pt; -return 0 ; -} -static int gen6_ppgtt_alloc (struct i915_hw_ppgtt *ppgtt) {int ret;
@@ -1051,7 +1113,7 @@ static int gen6_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt)if (ret)return ret; - ret =gen6_ppgtt_allocate_page_tables (ppgtt); + ret =alloc_pt_range (&ppgtt->pd,0 , ppgtt->num_pd_entries);if (ret) {drm_mm_remove_node (&ppgtt->node);return ret;
@@ -1069,7 +1131,7 @@ static int gen6_ppgtt_setup_page_tables(struct i915_hw_ppgtt *ppgtt)struct page *page;dma_addr_t pt_addr; - page = ppgtt->pd.page_tables[i].page; + page = ppgtt->pd.page_tables[i]->page; pt_addr =pci_map_page (dev->pdev, page,0 ,4096 ,PCI_DMA_BIDIRECTIONAL );
@@ -1078,7 +1140,7 @@ static int gen6_ppgtt_setup_page_tables(struct i915_hw_ppgtt *ppgtt)return -EIO ; } - ppgtt->pd.page_tables[i].daddr = pt_addr; + ppgtt->pd.page_tables[i]->daddr = pt_addr; }return 0 ;
drivers/gpu/drm/i915/i915_gem_gtt.h +2 -2
@@ -277,12 +277,12 @@ struct i915_pagedir {dma_addr_t daddr; }; -struct i915_pagetab *page_tables; +struct i915_pagetab *page_tables[GEN6_PPGTT_PD_ENTRIES ];/* PDEs */ };struct i915_pagedirpo {/* struct page *page; */ -struct i915_pagedir pagedir[GEN8_LEGACY_PDPES ]; +struct i915_pagedir *pagedir[GEN8_LEGACY_PDPES ]; };struct i915_hw_ppgtt {
drivers/gpu/drm/i915/intel_lrc.c +8 -8
@@ -1731,14 +1731,14 @@ populate_lr_context(struct intel_context *ctx, struct drm_i915_gem_object *ctx_o reg_state[CTX_PDP1_LDW ] =GEN8_RING_PDP_LDW (ring,1 ); reg_state[CTX_PDP0_UDW ] =GEN8_RING_PDP_UDW (ring,0 ); reg_state[CTX_PDP0_LDW ] =GEN8_RING_PDP_LDW (ring,0 ); - reg_state[CTX_PDP3_UDW +1 ] =upper_32_bits (ppgtt->pdp.pagedir[3 ].daddr); - reg_state[CTX_PDP3_LDW +1 ] =lower_32_bits (ppgtt->pdp.pagedir[3 ].daddr); - reg_state[CTX_PDP2_UDW +1 ] =upper_32_bits (ppgtt->pdp.pagedir[2 ].daddr); - reg_state[CTX_PDP2_LDW +1 ] =lower_32_bits (ppgtt->pdp.pagedir[2 ].daddr); - reg_state[CTX_PDP1_UDW +1 ] =upper_32_bits (ppgtt->pdp.pagedir[1 ].daddr); - reg_state[CTX_PDP1_LDW +1 ] =lower_32_bits (ppgtt->pdp.pagedir[1 ].daddr); - reg_state[CTX_PDP0_UDW +1 ] =upper_32_bits (ppgtt->pdp.pagedir[0 ].daddr); - reg_state[CTX_PDP0_LDW +1 ] =lower_32_bits (ppgtt->pdp.pagedir[0 ].daddr); + reg_state[CTX_PDP3_UDW +1 ] =upper_32_bits (ppgtt->pdp.pagedir[3 ]->daddr); + reg_state[CTX_PDP3_LDW +1 ] =lower_32_bits (ppgtt->pdp.pagedir[3 ]->daddr); + reg_state[CTX_PDP2_UDW +1 ] =upper_32_bits (ppgtt->pdp.pagedir[2 ]->daddr); + reg_state[CTX_PDP2_LDW +1 ] =lower_32_bits (ppgtt->pdp.pagedir[2 ]->daddr); + reg_state[CTX_PDP1_UDW +1 ] =upper_32_bits (ppgtt->pdp.pagedir[1 ]->daddr); + reg_state[CTX_PDP1_LDW +1 ] =lower_32_bits (ppgtt->pdp.pagedir[1 ]->daddr); + reg_state[CTX_PDP0_UDW +1 ] =upper_32_bits (ppgtt->pdp.pagedir[0 ]->daddr); + reg_state[CTX_PDP0_LDW +1 ] =lower_32_bits (ppgtt->pdp.pagedir[0 ]->daddr);if (ring->id ==RCS ) { reg_state[CTX_LRI_HEADER_2 ] =MI_LOAD_REGISTER_IMM (1 ); reg_state[CTX_R_PWR_CLK_STATE ] =0x20c8 ;
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH v2 10/24 drm/i915: Track GEN6 page table usage
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
Instead of implementing the full tracking + dynamic allocation, this patch does a bit less than half of the work, by tracking and warning on unexpected conditions. The tracking itself follows which PTEs within a page table are currently being used for objects. The next patch will modify this to actually allocate the page tables only when necessary.
With the current patch there isn't much in the way of making a gen agnostic range allocation function. However, in the next patch we'll add more specificity which makes having separate functions a bit easier to manage.
One important change introduced here is that DMA mappings are created/destroyed at the same page directories/tables are allocated/deallocated.
Notice that aliasing PPGTT is not managed here. The patch which actually begins dynamic allocation/teardown explains the reasoning for this.
v2: s/pdp.pagedir/pdp.pagedirs
Make a scratch page allocation helper
v3: Rebase and expand commit message.
v4: Allocate required pagetables only when it is needed, _bind_to_vm instead of bind_vma (Daniel).
--- drivers/gpu/drm/i915/i915_gem.c | 9 ++ drivers/gpu/drm/i915/i915_gem_gtt.c | 277 ++++++++++++++++++++++++++---------- drivers/gpu/drm/i915/i915_gem_gtt.h | 149 ++++++++++++++----- 3 files changed, 322 insertions(+), 113 deletions(-)
drivers/gpu/drm/i915/i915_gem.c +9 -0
@@ -3597,6 +3597,15 @@ search_free:if (ret) goto err_remove_node; +/* allocate before insert / bind */ +if (vma->vm->allocate_va_range) { + ret = vma->vm->allocate_va_range (vma->vm, + vma->node.start, + vma->node.size); +if (ret) + goto err_remove_node; + } +trace_i915_vma_bind (vma, flags); ret =i915_vma_bind (vma, obj->cache_level, flags &PIN_GLOBAL ?GLOBAL_BIND :0 );
drivers/gpu/drm/i915/i915_gem_gtt.c +198 -79
451 lines not on this page · raw message
drivers/gpu/drm/i915/i915_gem_gtt.h +115 -34
186 lines not on this page · raw message
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
Re: [PATCH v2 10/24] drm/i915: Track GEN6 page table usage
Quoted 23 lines · Michel Thierry
From: Ben Widawsky <[email protected]>
Instead of implementing the full tracking + dynamic allocation, this patch does a bit less than half of the work, by tracking and warning on unexpected conditions. The tracking itself follows which PTEs within a page table are currently being used for objects. The next patch will modify this to actually allocate the page tables only when necessary.
With the current patch there isn't much in the way of making a gen agnostic range allocation function. However, in the next patch we'll add more specificity which makes having separate functions a bit easier to manage.
One important change introduced here is that DMA mappings are created/destroyed at the same page directories/tables are allocated/deallocated.
Notice that aliasing PPGTT is not managed here. The patch which actually begins dynamic allocation/teardown explains the reasoning for this.
v2: s/pdp.pagedir/pdp.pagedirs
Make a scratch page allocation helper
Quoted 8 lines
v3: Rebase and expand commit message.
v4: Allocate required pagetables only when it is needed,
_bind_to_vminstead ofbind_vma(Daniel).Cc Daniel Vetter <[email protected]>Signed-off-by Ben Widawsky <[email protected]>Signed-off-by Michel Thierry <[email protected]> (v3+)
Imo still a bit too much rebase fluff in this patch. I think it would help the patch clarity a lot of we'd split the changes to move around the dma_map/unmap calls from the other parts of the patch to dynamically allocate pagetables.
Bunch more comments below.
-Daniel
Quoted 12 lines
--- drivers/gpu/drm/i915/i915_gem.c | 9 ++ drivers/gpu/drm/i915/i915_gem_gtt.c | 277 ++++++++++++++++++++++++++---------- drivers/gpu/drm/i915/i915_gem_gtt.h | 149 ++++++++++++++----- 3 files changed, 322 insertions(+), 113 deletions(-)drivers/gpu/drm/i915/i915_gem.c +0
-0@@ -3597,6 +3597,15 @@ search_free:if (ret)
drivers/gpu/drm/i915/i915_gem.c search_free: · line 3607
Quoted 10 lines
drivers/gpu/drm/i915/i915_gem.c +8
-0goto err_remove_node; +/* allocate before insert / bind */ +if (vma->vm->allocate_va_range) { + ret = vma->vm->allocate_va_range (vma->vm, + vma->node.start, + vma->node.size); +if (ret) + goto err_remove_node; + }
Is this really the right patch for this hunk? The commit message sounds like dynamic pagetable alloc is only partially implemented here ...
Quoted 37 lines
drivers/gpu/drm/i915/i915_gem.c +1
-0+trace_i915_vma_bind (vma, flags); ret =i915_vma_bind (vma, obj->cache_level, flags &PIN_GLOBAL ?GLOBAL_BIND :0 );drivers/gpu/drm/i915/i915_gem_gtt.c +12
-5@@ -138,10 +138,9 @@ static int sanitize_enable_ppgtt(struct drm_device *dev, int enable_ppgtt)return has_aliasing_ppgtt ?1 :0 ; } -static void ppgtt_bind_vma (struct i915_vma *vma, -enum i915_cache_level cache_level, -u32 flags); +enum i915_cache_level cache_level, +u32 flags);static void ppgtt_unbind_vma (struct i915_vma *vma); static inline gen8_gtt_pte_t gen8_pte_encode(dma_addr_t addr,@@ -275,27 +274,99 @@ static gen6_gtt_pte_t iris_pte_encode(dma_addr_t addr,return pte; } -static void free_pt_single (struct i915_pagetab *pt) -{ +#define i915_dma_unmap_single (px, dev) do { \ + pci_unmap_page((dev)->pdev, (px)->daddr, 4096, PCI_DMA_BIDIRECTIONAL); \ +} while (0); + +/** +* i915_dma_map_px_single() - Create a dma mapping for a page table/dir/etc. +* @px: Page table/dir/etc to get a DMA map for +* @dev: drm device +* +* Page table allocations are unified across all gens. They always require a
drivers/gpu/drm/i915/i915_gem_gtt.c static gen6_gtt_pte_t iris_pte_encode(dma_addr_t addr, · line 296
Quoted 10 lines
drivers/gpu/drm/i915/i915_gem_gtt.c +10
-0+* single 4k allocation, as well as a DMA mapping. If we keep the structs +* symmetric here, the simple macro covers us for every page table type. +* +* Return: 0 if success. +*/ +#define i915_dma_map_px_single (px, dev) \ + pci_dma_mapping_error((dev)->pdev, \ + (px)->daddr = pci_map_page((dev)->pdev, \ + (px)->page, 0, 4096, \ + PCI_DMA_BIDIRECTIONAL))
Linux coding style discourages macro abuse like this, please make this a static inline instead. Otoh I don't really see the value in hiding the pci_map_page call, imo open-codeing this is totally ok.
But while you touch the code please switch away from the pci_map wrappers and use the dma_map functions directly.
Quoted 21 lines
drivers/gpu/drm/i915/i915_gem_gtt.c +19
-0+ +static void __free_pt_single (struct i915_pagetab *pt,struct drm_device *dev, +int scratch) +{ +if (WARN (scratch ^ pt->scratch, +"Tried to free scratch = %d. Is scratch = %d\n" , + scratch, pt->scratch)) +return ; +if (WARN_ON (!pt->page))return ; + +if (!scratch) { +const size_t count =INTEL_INFO (dev)->gen >=8 ? +GEN8_PTES_PER_PAGE :I915_PPGTT_PT_ENTRIES ; +WARN (!bitmap_empty (pt->used_ptes, count), +"Free page table with %d used pages\n" , +bitmap_weight (pt->used_ptes, count)); + } + +i915_dma_unmap_single (pt, dev);
drivers/gpu/drm/i915/i915_gem_gtt.c static gen6_gtt_pte_t iris_pte_encode(dma_addr_t addr, · line 326
Quoted 10 lines
drivers/gpu/drm/i915/i915_gem_gtt.c +5
-1__free_page (pt->page); +kfree (pt->used_ptes);kfree (pt); } -static struct i915_pagetab *alloc_pt_single (void ) +#define free_pt_single (pt, dev) \ + __free_pt_single(pt, dev, false) +#define free_pt_scratch (pt, dev) \ + __free_pt_single(pt, dev, true)
Imo the disdinction between _single and _scracth is confusing. Instead adding a new functions unmap_and_free_pt which calls dma_unmap_page and free_pt_single would make more sense. Then there's also no need for the __ version of the function and the scratch parameter.
It means that we'll need to kill the selfchecks for scratch, but by uncluttering the indirections here a bit as proposed any such bugs should be obvious. All the others will be caught by the dma mapping debugging code (which is super-paranoid afaik).
Quoted 5 lines
drivers/gpu/drm/i915/i915_gem_gtt.c +3
-0+ +static struct i915_pagetab *alloc_pt_single (struct drm_device *dev) {struct i915_pagetab *pt; + const size_t count = INTEL_INFO(dev)->gen >= 8 ?
drivers/gpu/drm/i915/i915_gem_gtt.c static gen6_gtt_pte_t iris_pte_encode(dma_addr_t addr, · line 341
Quoted 10 lines
drivers/gpu/drm/i915/i915_gem_gtt.c +5
-0+ GEN8_PTES_PER_PAGE :I915_PPGTT_PT_ENTRIES ; +int ret = -ENOMEM ; pt =kzalloc (sizeof (*pt),GFP_KERNEL );if (!pt)return ERR_PTR (-ENOMEM ); + pt->used_ptes =kcalloc (BITS_TO_LONGS (count),sizeof (*pt->used_ptes), +GFP_KERNEL ); +
I don't see the value in tracking used_ptes. For debugging we can just look at the pte value itself (which should be in cached memory, so dirt cheap). And since pagetables are the lowest level we can't screw up the allocations/freeing. What do I miss?
Quoted 22 lines
drivers/gpu/drm/i915/i915_gem_gtt.c +17
-4+if (!pt->used_ptes) + goto fail_bitmap; + pt->page =alloc_page (GFP_KERNEL | __GFP_ZERO); -if (!pt->page) { -kfree (pt); -return ERR_PTR (-ENOMEM ); - } +if (!pt->page) + goto fail_page; + + ret =i915_dma_map_px_single (pt, dev); +if (ret) + goto fail_dma; + +return pt; + +fail_dma: +__free_page (pt->page); +fail_page: +kfree (pt->used_ptes); +fail_bitmap:
drivers/gpu/drm/i915/i915_gem_gtt.c static gen6_gtt_pte_t iris_pte_encode(dma_addr_t addr, · line 369
Quoted 10 lines
drivers/gpu/drm/i915/i915_gem_gtt.c +10
-0+kfree (pt); + +return ERR_PTR (ret); +} + +static inline struct i915_pagetab *alloc_pt_scratch (struct drm_device *dev) +{ +struct i915_pagetab *pt =alloc_pt_single (dev); +if (!IS_ERR (pt)) + pt->scratch =1 ;
Shouldn't we fill the scratch pt with scratch pte entries? Or do I miss the point of this? Hard to tell without having users of scratch_pt in the same patch as the patch as the one that adds them. Might be good to split this in yet another patc.
drivers/gpu/drm/i915/i915_gem_gtt.c static struct i915_pagetab *alloc_pt_single(void) · line 388
Quoted 10 lines
drivers/gpu/drm/i915/i915_gem_gtt.c +2
-1return pt; }@@ -313,7 +384,9 @@ static struct i915_pagetab *alloc_pt_single(void)* * Return: 0 if allocation succeeded. */ -static int alloc_pt_range (struct i915_pagedir *pd,uint16_t pde,size_t count) +static int alloc_pt_range (struct i915_pagedir *pd,uint16_t pde,size_t count, +struct drm_device *dev)
Aside: The reason I've suggested to split the patches is so that we can get all the added *dev parameters out of the diff. That should help patch readablity a lot.
Quoted 91 lines
drivers/gpu/drm/i915/i915_gem_gtt.c +16
-17+ {int i, ret;@@ -323,7 +396,7 @@ static int alloc_pt_range(struct i915_pagedir *pd, uint16_t pde, size_t count)BUG_ON (pde + count >GEN6_PPGTT_PD_ENTRIES );for (i = pde; i < pde + count; i++) { -struct i915_pagetab *pt =alloc_pt_single (); +struct i915_pagetab *pt =alloc_pt_single (dev);if (IS_ERR (pt)) { ret =PTR_ERR (pt); goto err_out;@@ -338,7 +411,7 @@ static int alloc_pt_range(struct i915_pagedir *pd, uint16_t pde, size_t count) err_out:while (i--) -free_pt_single (pd->page_tables[i]); +free_pt_single (pd->page_tables[i], dev);return ret; }@@ -506,7 +579,7 @@ static void gen8_ppgtt_insert_entries(struct i915_address_space *vm, } } -static void gen8_free_page_tables (struct i915_pagedir *pd) +static void gen8_free_page_tables (struct i915_pagedir *pd,struct drm_device *dev) {int i;@@ -514,7 +587,7 @@ static void gen8_free_page_tables(struct i915_pagedir *pd)return ;for (i =0 ; i <GEN8_PDES_PER_PAGE ; i++) { -free_pt_single (pd->page_tables[i]); +free_pt_single (pd->page_tables[i], dev); pd->page_tables[i] =NULL ; } }@@ -524,7 +597,7 @@ static void gen8_ppgtt_free(struct i915_hw_ppgtt *ppgtt)int i;for (i =0 ; i < ppgtt->num_pd_pages; i++) { -gen8_free_page_tables (ppgtt->pdp.pagedir[i]); +gen8_free_page_tables (ppgtt->pdp.pagedir[i], ppgtt->base.dev);free_pd_single (ppgtt->pdp.pagedir[i]); } }@@ -569,7 +642,7 @@ static int gen8_ppgtt_allocate_page_tables(struct i915_hw_ppgtt *ppgtt)for (i =0 ; i < ppgtt->num_pd_pages; i++) { ret =alloc_pt_range (ppgtt->pdp.pagedir[i], -0 ,GEN8_PDES_PER_PAGE ); +0 ,GEN8_PDES_PER_PAGE , ppgtt->base.dev);if (ret) goto unwind_out; }@@ -578,7 +651,7 @@ static int gen8_ppgtt_allocate_page_tables(struct i915_hw_ppgtt *ppgtt) unwind_out:while (i--) -gen8_free_page_tables (ppgtt->pdp.pagedir[i]); +gen8_free_page_tables (ppgtt->pdp.pagedir[i], ppgtt->base.dev);return -ENOMEM ; }@@ -808,26 +881,36 @@ static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m) } } -static void gen6_write_pdes (struct i915_hw_ppgtt *ppgtt) +/* Write pde (index) from the page directory @pd to the page table @pt */ +static void gen6_write_pdes (struct i915_pagedir *pd, +const int pde,struct i915_pagetab *pt) { -struct drm_i915_private *dev_priv = ppgtt->base.dev->dev_private; -gen6_gtt_pte_t __iomem *pd_addr; -uint32_t pd_entry; -int i; +struct i915_hw_ppgtt *ppgtt = + container_of(pd, struct i915_hw_ppgtt, pd); +u32 pd_entry; -WARN_ON (ppgtt->pd.pd_offset &0x3f ); - pd_addr = (gen6_gtt_pte_t __iomem *)dev_priv->gtt.gsm + - ppgtt->pd.pd_offset /sizeof (gen6_gtt_pte_t); -for (i =0 ; i < ppgtt->num_pd_entries; i++) { -dma_addr_t pt_addr; + pd_entry =GEN6_PDE_ADDR_ENCODE (pt->daddr); + pd_entry |=GEN6_PDE_VALID ;
drivers/gpu/drm/i915/i915_gem_gtt.c static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m) · line 897
Quoted 10 lines
drivers/gpu/drm/i915/i915_gem_gtt.c +2
-6- pt_addr = ppgtt->pd.page_tables[i]->daddr; - pd_entry =GEN6_PDE_ADDR_ENCODE (pt_addr); - pd_entry |=GEN6_PDE_VALID ; +writel (pd_entry, ppgtt->pd_addr + pde); -writel (pd_entry, pd_addr + i); - } -readl (pd_addr); +/* XXX: Caller needs to make sure the write completes if necessary */
Please take care of such XXX comments. That's the stuff I've meant when I've said that the overall patch series needs a full pass with a critical eye to catch development/rebase leftovers.
Quoted 34 lines
drivers/gpu/drm/i915/i915_gem_gtt.c +27
-0+} + +/* Write all the page tables found in the ppgtt structure to incrementing page +* directories. */ +static void gen6_write_page_range (struct drm_i915_private *dev_priv, +struct i915_pagedir *pd,uint32_t start,uint32_t length) +{ +struct i915_pagetab *pt; +uint32_t pde, temp; + +gen6_for_each_pde (pt, pd, start, length, temp, pde) +gen6_write_pdes (pd, pde, pt); + +/* Make sure write is complete before other code can use this page +* table. Also require for WC mapped PTEs */ +readl (dev_priv->gtt.gsm); }static uint32_t get_pd_offset (struct i915_hw_ppgtt *ppgtt)@@ -1043,13 +1126,59 @@ static void gen6_ppgtt_unmap_pages(struct i915_hw_ppgtt *ppgtt) 4096, PCI_DMA_BIDIRECTIONAL); } +static int gen6_alloc_va_range (struct i915_address_space *vm, +uint64_t start,uint64_t length) +{ +struct i915_hw_ppgtt *ppgtt = + container_of(vm, struct i915_hw_ppgtt, base); +struct i915_pagetab *pt; +uint32_t pde, temp; + +gen6_for_each_pde (pt, &ppgtt->pd, start, length, temp, pde) { +int j; +
drivers/gpu/drm/i915/i915_gem_gtt.c static void gen6_ppgtt_unmap_pages(struct i915_hw_ppgtt *ppgtt) · line 1149
Quoted 10 lines
drivers/gpu/drm/i915/i915_gem_gtt.c +10
-0+DECLARE_BITMAP (tmp_bitmap,I915_PPGTT_PT_ENTRIES ); +bitmap_zero (tmp_bitmap,I915_PPGTT_PT_ENTRIES ); +bitmap_set (tmp_bitmap,gen6_pte_index (start), +gen6_pte_count (start, length)); + +/* TODO: To be done in the next patch. Map the page/insert +* entries here */ +for_each_set_bit (j, tmp_bitmap,I915_PPGTT_PT_ENTRIES ) { +if (test_bit (j, pt->used_ptes)) { +/* Check that we're changing cache levels */
Again something only valid from older revisions since we've taken care of the cache_level changes by only extendeding pagetables where actually needed (in the bind functions). Furthermore with the used_ptes bitmask gone this would all disappear anyway.
Given that the patch justifies itself by adding the dynamic allocation and tracking first to debug it, but doesn't add any self-checks in the pte writing funcs (afaics at least) I don't think this is all that useful any more.
Quoted 149 lines
drivers/gpu/drm/i915/i915_gem_gtt.c +46
-35+ } + } + +bitmap_or (pt->used_ptes, pt->used_ptes, tmp_bitmap, +I915_PPGTT_PT_ENTRIES ); + } + +return 0 ; +} + +static void gen6_teardown_va_range (struct i915_address_space *vm, +uint64_t start,uint64_t length) +{ +struct i915_hw_ppgtt *ppgtt = + container_of(vm, struct i915_hw_ppgtt, base); +struct i915_pagetab *pt; +uint32_t pde, temp; + +gen6_for_each_pde (pt, &ppgtt->pd, start, length, temp, pde) { +bitmap_clear (pt->used_ptes,gen6_pte_index (start), +gen6_pte_count (start, length)); + } +} +static void gen6_ppgtt_free (struct i915_hw_ppgtt *ppgtt) {int i;for (i =0 ; i < ppgtt->num_pd_entries; i++) -free_pt_single (ppgtt->pd.page_tables[i]); +free_pt_single (ppgtt->pd.page_tables[i], ppgtt->base.dev); +free_pt_scratch (ppgtt->scratch_pt, ppgtt->base.dev);free_pd_single (&ppgtt->pd); }@@ -1076,6 +1205,9 @@ static int gen6_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt)* size. We allocate at the top of the GTT to avoid fragmentation. */ BUG_ON (!drm_mm_initialized (&dev_priv->gtt.base.mm)); + ppgtt->scratch_pt =alloc_pt_scratch (ppgtt->base.dev); +if (IS_ERR (ppgtt->scratch_pt)) +return PTR_ERR (ppgtt->scratch_pt); alloc: ret = drm_mm_insert_node_in_range_generic(&dev_priv->gtt.base.mm, &ppgtt->node, GEN6_PD_SIZE,@@ -1089,20 +1221,25 @@ alloc:0 , dev_priv->gtt.base.total, 0);if (ret) -return ret; + goto err_out; retried = true; goto alloc; }if (ret) -return ret; + goto err_out; +if (ppgtt->node.start < dev_priv->gtt.mappable_end)DRM_DEBUG ("Forced to use aperture for PDEs\n" ); ppgtt->num_pd_entries =GEN6_PPGTT_PD_ENTRIES ;return 0 ; + +err_out: +free_pt_scratch (ppgtt->scratch_pt, ppgtt->base.dev); +return ret; }static int gen6_ppgtt_alloc (struct i915_hw_ppgtt *ppgtt)@@ -1113,7 +1250,9 @@ static int gen6_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt)if (ret)return ret; - ret =alloc_pt_range (&ppgtt->pd,0 , ppgtt->num_pd_entries); + ret =alloc_pt_range (&ppgtt->pd,0 , ppgtt->num_pd_entries, + ppgtt->base.dev); +if (ret) {drm_mm_remove_node (&ppgtt->node);return ret;@@ -1122,30 +1261,6 @@ static int gen6_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt)return 0 ; } -static int gen6_ppgtt_setup_page_tables (struct i915_hw_ppgtt *ppgtt) -{ -struct drm_device *dev = ppgtt->base.dev; -int i; - -for (i =0 ; i < ppgtt->num_pd_entries; i++) { -struct page *page; -dma_addr_t pt_addr; - - page = ppgtt->pd.page_tables[i]->page; - pt_addr =pci_map_page (dev->pdev, page,0 ,4096 , -PCI_DMA_BIDIRECTIONAL ); - -if (pci_dma_mapping_error (dev->pdev, pt_addr)) { -gen6_ppgtt_unmap_pages (ppgtt); -return -EIO ; - } - - ppgtt->pd.page_tables[i]->daddr = pt_addr; - } - -return 0 ; -} -static int gen6_ppgtt_init (struct i915_hw_ppgtt *ppgtt) {struct drm_device *dev = ppgtt->base.dev;@@ -1166,12 +1281,8 @@ static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt)if (ret)return ret; - ret =gen6_ppgtt_setup_page_tables (ppgtt); -if (ret) { -gen6_ppgtt_free (ppgtt); -return ret; - } - + ppgtt->base.allocate_va_range = gen6_alloc_va_range; + ppgtt->base.teardown_va_range = gen6_teardown_va_range; ppgtt->base.clear_range = gen6_ppgtt_clear_range; ppgtt->base.insert_entries = gen6_ppgtt_insert_entries; ppgtt->base.cleanup = gen6_ppgtt_cleanup;@@ -1182,11 +1293,15 @@ static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt) ppgtt->pd.pd_offset = ppgtt->node.start /PAGE_SIZE *sizeof (gen6_gtt_pte_t); + ppgtt->pd_addr = (gen6_gtt_pte_t __iomem *)dev_priv->gtt.gsm + + ppgtt->pd.pd_offset /sizeof (gen6_gtt_pte_t); + +gen6_write_page_range (dev_priv, &ppgtt->pd,0 , ppgtt->base.total); +DRM_DEBUG_DRIVER ("Allocated pde space (%ldM) at GTT entry: %lx\n" , ppgtt->node.size >>20 , ppgtt->node.start /PAGE_SIZE ); -gen6_write_pdes (ppgtt);DRM_DEBUG ("Adding PPGTT at offset %x\n" , ppgtt->pd.pd_offset <<10 );
drivers/gpu/drm/i915/i915_gem_gtt.c static void ppgtt_unbind_vma(struct i915_vma *vma) · line 1438
Quoted 6 lines
drivers/gpu/drm/i915/i915_gem_gtt.c +3
-0@@ -1318,6 +1433,9 @@ static void ppgtt_unbind_vma(struct i915_vma *vma) vma->node.start, vma->obj->base.size, true); +if (vma->vm->teardown_va_range) + vma->vm->teardown_va_range (vma->vm, + vma->node.start, vma->node.size);
If we ditch unsed_ptes we can ditch this here too. As per my irc discussion with Chris Wilson I think the best way to actually free pagetables is in our shrinker, by simply freeing them all when a vm contains no bound buffer at all. Much less fuzz and avoids all the complications that ripping out pagetables from underneath active users might entail. On the cpu side the core vm doesn't even bother with that, so I expect we can forgoe that complexity on the gpu side, too.
Quoted 69 lines
drivers/gpu/drm/i915/i915_gem_gtt.c +11
-10}extern int intel_iommu_gfx_mapped;@@ -1461,13 +1579,14 @@ void i915_gem_restore_gtt_mappings(struct drm_device *dev)list_for_each_entry (vm, &dev_priv->vm_list, global_link) {/* TODO: Perhaps it shouldn't be gen6 specific */ -if (i915_is_ggtt (vm)) { -if (dev_priv->mm.aliasing_ppgtt) -gen6_write_pdes (dev_priv->mm.aliasing_ppgtt); -continue ; - } -gen6_write_pdes (container_of(vm ,struct i915_hw_ppgtt ,base )); +struct i915_hw_ppgtt *ppgtt = + container_of(vm, struct i915_hw_ppgtt, base); + +if (i915_is_ggtt (vm)) + ppgtt = dev_priv->mm.aliasing_ppgtt; + +gen6_write_page_range (dev_priv, &ppgtt->pd,0 , ppgtt->num_pd_entries); }i915_ggtt_flush (dev_priv);@@ -1633,8 +1752,8 @@ static void gen6_ggtt_clear_range(struct i915_address_space *vm,static void i915_ggtt_bind_vma (struct i915_vma *vma, -enum i915_cache_level cache_level, -u32 unused) +enum i915_cache_level cache_level, +u32 unused) {const unsigned long entry = vma->node.start >>PAGE_SHIFT ; unsigned int flags = (cache_level == I915_CACHE_NONE) ?@@ -1666,8 +1785,8 @@ static void i915_ggtt_unbind_vma(struct i915_vma *vma) }static void ggtt_bind_vma (struct i915_vma *vma, -enum i915_cache_level cache_level, -u32 flags) +enum i915_cache_level cache_level, +u32 flags) {struct drm_device *dev = vma->vm->dev;struct drm_i915_private *dev_priv = dev->dev_private;drivers/gpu/drm/i915/i915_gem_gtt.h +5
-2@@ -54,7 +54,10 @@ typedef gen8_gtt_pte_t gen8_ppgtt_pde_t;#define GEN6_PPGTT_PD_ENTRIES 512#define GEN6_PD_SIZE (GEN6_PPGTT_PD_ENTRIES * PAGE_SIZE)#define GEN6_PD_ALIGN (PAGE_SIZE * 16) +#define GEN6_PDE_SHIFT 22#define GEN6_PDE_VALID (1 << 0) +#define GEN6_PDE_MASK (GEN6_PPGTT_PD_ENTRIES-1) +#define NUM_PTE (pde_shift) (1 << (pde_shift - PAGE_SHIFT))#define GEN7_PTE_CACHE_L3_LLC (3 << 1)@@ -183,8 +186,32 @@ struct i915_vma {void (*unbind_vma)(struct i915_vma *vma);/* Map an object into an address space with the given cache flags. */ void (*bind_vma)(struct i915_vma *vma, -enum i915_cache_level cache_level, -u32 flags); +enum i915_cache_level cache_level, +u32 flags);
drivers/gpu/drm/i915/i915_gem_gtt.h struct i915_vma { · line 200
Quoted 10 lines
drivers/gpu/drm/i915/i915_gem_gtt.h +10
-0+}; + + +struct i915_pagetab { +struct page *page; +dma_addr_t daddr; + +unsigned long *used_ptes; +unsigned int scratch:1 ; +};
there's a bit of noise in the diff because you move around structures. Imo just using forward decls is better, or reorder the definitions in the patch that adds them.
Quoted 131 lines
drivers/gpu/drm/i915/i915_gem_gtt.h +79
-26+ +struct i915_pagedir { +struct page *page;/* NULL for GEN6-GEN7 */ +union { +uint32_t pd_offset; +dma_addr_t daddr; + }; + +struct i915_pagetab *page_tables[GEN6_PPGTT_PD_ENTRIES ]; +}; + +struct i915_pagedirpo { +/* struct page *page; */ +struct i915_pagedir *pagedir[GEN8_LEGACY_PDPES ]; };struct i915_address_space {@@ -226,6 +253,12 @@ struct i915_address_space {gen6_gtt_pte_t (*pte_encode)(dma_addr_t addr,enum i915_cache_level level,bool valid,u32 flags);/* Create a valid PTE */ +int (*allocate_va_range)(struct i915_address_space *vm, +uint64_t start, +uint64_t length); +void (*teardown_va_range)(struct i915_address_space *vm, +uint64_t start, +uint64_t length); void (*clear_range)(struct i915_address_space *vm, uint64_t start, uint64_t length,@@ -237,6 +270,29 @@ struct i915_address_space {void (*cleanup)(struct i915_address_space *vm); }; +struct i915_hw_ppgtt { +struct i915_address_space base; +struct kref ref; +struct drm_mm_node node; +unsigned num_pd_entries; +unsigned num_pd_pages;/* gen8+ */ +union { +struct i915_pagedirpo pdp; +struct i915_pagedir pd; + }; + +struct i915_pagetab *scratch_pt; + +struct drm_i915_file_private *file_priv; + +gen6_gtt_pte_t __iomem *pd_addr; + +int (*enable)(struct i915_hw_ppgtt *ppgtt); +int (*switch_mm)(struct i915_hw_ppgtt *ppgtt, +struct intel_engine_cs *ring); +void (*debug_dump)(struct i915_hw_ppgtt *ppgtt,struct seq_file *m); +}; +/* The Graphics Translation Table is the way in which GEN hardware translates a * Graphics Virtual Address into a Physical Address. In addition to the normal * collateral associated with any va->pa translations GEN hardware also has a @@ -265,44 +321,69 @@ struct i915_gtt { unsigned long *mappable_end); }; -struct i915_pagetab { -struct page *page; -dma_addr_t daddr; -}; +/* For each pde iterates over every pde between from start until start + length. +* If start, and start+length are not perfectly divisible, the macro will round +* down, and up as needed. The macro modifies pde, start, and length. Dev is +* only used to differentiate shift values. Temp is temp. On gen6/7, start = 0, +* and length = 2G effectively iterates over every PDE in the system. On gen8+ +* it simply iterates over every page directory entry in a page directory. +* +* XXX: temp is not actually needed, but it saves doing the ALIGN operation. +*/ +#define gen6_for_each_pde (pt, pd, start, length, temp, iter) \ + for (iter = gen6_pde_index(start), pt = (pd)->page_tables[iter]; \ + length > 0 && iter < GEN6_PPGTT_PD_ENTRIES; \ + pt = (pd)->page_tables[++iter], \ + temp = ALIGN(start+1, 1 << GEN6_PDE_SHIFT) - start, \ + temp = min(temp, (unsigned)length), \ + start += temp, length -= temp) + +static inline uint32_t i915_pte_index (uint64_t address,uint32_t pde_shift) +{ +const uint32_t mask =NUM_PTE (pde_shift) -1 ; +return (address >>PAGE_SHIFT ) & mask; +} -struct i915_pagedir { -struct page *page;/* NULL for GEN6-GEN7 */ -union { -uint32_t pd_offset; -dma_addr_t daddr; - }; +/* Helper to counts the number of PTEs within the given length. This count does +* not cross a page table boundary, so the max value would be +* I915_PPGTT_PT_ENTRIES for GEN6, and GEN8_PTES_PER_PAGE for GEN8. +*/ +static inline size_t i915_pte_count (uint64_t addr,size_t length, +uint32_t pde_shift) +{ +const uint64_t mask = ~((1 << pde_shift) -1 ); +uint64_t end; -struct i915_pagetab *page_tables[GEN6_PPGTT_PD_ENTRIES ];/* PDEs */ -}; +BUG_ON (length ==0 ); +BUG_ON (offset_in_page (addr|length)); -struct i915_pagedirpo { -/* struct page *page; */ -struct i915_pagedir *pagedir[GEN8_LEGACY_PDPES ]; -}; + end = addr + length; -struct i915_hw_ppgtt { -struct i915_address_space base; -struct kref ref; -struct drm_mm_node node; -unsigned num_pd_entries; -unsigned num_pd_pages;/* gen8+ */ -union { -struct i915_pagedirpo pdp; -struct i915_pagedir pd; - }; +if ((addr & mask) != (end & mask)) +return NUM_PTE (pde_shift) -i915_pte_index (addr, pde_shift);
drivers/gpu/drm/i915/i915_gem_gtt.h struct i915_gtt { · line 368
Quoted 10 lines
drivers/gpu/drm/i915/i915_gem_gtt.h +3
-6-struct drm_i915_file_private *file_priv; +return i915_pte_index (end, pde_shift) -i915_pte_index (addr, pde_shift); +} -int (*enable)(struct i915_hw_ppgtt *ppgtt); -int (*switch_mm)(struct i915_hw_ppgtt *ppgtt, -struct intel_engine_cs *ring); -void (*debug_dump)(struct i915_hw_ppgtt *ppgtt,struct seq_file *m); -}; +static inline uint32_t i915_pde_index (uint64_t addr,uint32_t shift)
Shouldn't this have a gen6 prefix, too?
Quoted 28 lines
drivers/gpu/drm/i915/i915_gem_gtt.h +18
-0+{ +return (addr >> shift) &GEN6_PDE_MASK ; +} + +static inline uint32_t gen6_pte_index (uint32_t addr) +{ +return i915_pte_index (addr,GEN6_PDE_SHIFT ); +} + +static inline size_t gen6_pte_count (uint32_t addr,uint32_t length) +{ +return i915_pte_count (addr, length,GEN6_PDE_SHIFT ); +} + +static inline uint32_t gen6_pde_index (uint32_t addr) +{ +return i915_pde_index (addr,GEN6_PDE_SHIFT ); +}int i915_gem_gtt_init (struct drm_device *dev);void i915_gem_init_global_gtt (struct drm_device *dev);Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
Signature · 7 lines
Daniel Vetter Software Engineer, Intel Corporation +41 (0) 79 365 57 48 - http://blog.ffwll.ch _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH v2 11/24 drm/i915: Extract context switch skip and pd load logic
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
We have some fanciness coming up. This patch just breaks out the logic of context switch skip, pd load pre, and pd load post.
v2: Use new functions to replace the logic right away (Daniel)
--- drivers/gpu/drm/i915/i915_gem_context.c | 40 +++++++++++++++++++++++++-------- 1 file changed, 31 insertions(+), 9 deletions(-)
drivers/gpu/drm/i915/i915_gem_context.c +31 -9
@@ -563,6 +563,33 @@ mi_set_context(struct intel_engine_cs *ring,return ret; } +static inline bool should_skip_switch (struct intel_engine_cs *ring, +struct intel_context *from, +struct intel_context *to) +{ +if (from == to && !to->remap_slice) +return true; + +return false; +} + +static bool +needs_pd_load_pre (struct intel_engine_cs *ring,struct intel_context *to) +{ +struct drm_i915_private *dev_priv = ring->dev->dev_private; + +return ((INTEL_INFO (ring->dev)->gen <8 ) || + (ring != &dev_priv->ring[RCS ])) && to->ppgtt; +} + +static bool +needs_pd_load_post (struct intel_engine_cs *ring,struct intel_context *to) +{ +return (!to->legacy_hw_ctx.initialized || +i915_gem_context_is_default (to)) && + to->ppgtt &&IS_GEN8 (ring->dev); +} +static int do_switch (struct intel_engine_cs *ring,struct intel_context *to) {
@@ -571,9 +598,6 @@ static int do_switch(struct intel_engine_cs *ring,u32 hw_flags =0 ;bool uninitialized = false;struct i915_vma *vma; -bool needs_pd_load_pre = ((INTEL_INFO (ring->dev)->gen <8 ) || - (ring != &dev_priv->ring[RCS ])) && to->ppgtt; -bool needs_pd_load_post = false;int ret, i;if (from !=NULL && ring == &dev_priv->ring[RCS ]) {
@@ -581,7 +605,7 @@ static int do_switch(struct intel_engine_cs *ring,BUG_ON (!i915_gem_obj_is_pinned (from->legacy_hw_ctx.rcs_state)); } -if (from == to && !to->remap_slice) +if (should_skip_switch (ring, from, to))return 0 ;/* Trying to pin first makes error handling easier. */
@@ -599,7 +623,7 @@ static int do_switch(struct intel_engine_cs *ring,*/ from = ring->last_context; -if (needs_pd_load_pre) { +if (needs_pd_load_pre (ring, to)) {/* Older GENs and non render rings still want the load first, * "PP_DCLV followed by PP_DIR_BASE register through Load * Register Immediate commands in Ring Buffer before submitting
@@ -644,16 +668,14 @@ static int do_switch(struct intel_engine_cs *ring,* XXX: If we implemented page directory eviction code, this * optimization needs to be removed. */ -if (!to->legacy_hw_ctx.initialized ||i915_gem_context_is_default (to)) { +if (!to->legacy_hw_ctx.initialized ||i915_gem_context_is_default (to)) hw_flags |=MI_RESTORE_INHIBIT ; - needs_pd_load_post = to->ppgtt &&IS_GEN8 (ring->dev); - } ret =mi_set_context (ring, to, hw_flags);if (ret) goto unpin_out; -if (needs_pd_load_post) { +if (needs_pd_load_post (ring, to)) { ret = to->ppgtt->switch_mm (to->ppgtt, ring);/* The hardware context switch is emitted, but we haven't * actually changed the state - so it's probably safe to bail
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
Re: [PATCH v2 11/24] drm/i915: Extract context switch skip and pd load logic
Quoted 10 lines · Michel Thierry
From: Ben Widawsky <[email protected]>
We have some fanciness coming up. This patch just breaks out the logic of context switch skip, pd load pre, and pd load post.
v2: Use new functions to replace the logic right away (Daniel)
Cc Daniel Vetter <[email protected]>Signed-off-by Ben Widawsky <[email protected]>Signed-off-by Michel Thierry <[email protected]> (v2)
Since we won't ever ship gen8+ without execlist and full ppgtt legacy context switching special-cases for that are kinda moot. But something we could simplify with a follow-up patch once the series has landed, together with some checks to make sure we don't try to reload the ppgtt on gen8+ ever.
-Daniel
Quoted 96 lines
--- drivers/gpu/drm/i915/i915_gem_context.c | 40 +++++++++++++++++++++++++-------- 1 file changed, 31 insertions(+), 9 deletions(-)drivers/gpu/drm/i915/i915_gem_context.c +31
-9@@ -563,6 +563,33 @@ mi_set_context(struct intel_engine_cs *ring,return ret; } +static inline bool should_skip_switch (struct intel_engine_cs *ring, +struct intel_context *from, +struct intel_context *to) +{ +if (from == to && !to->remap_slice) +return true; + +return false; +} + +static bool +needs_pd_load_pre (struct intel_engine_cs *ring,struct intel_context *to) +{ +struct drm_i915_private *dev_priv = ring->dev->dev_private; + +return ((INTEL_INFO (ring->dev)->gen <8 ) || + (ring != &dev_priv->ring[RCS ])) && to->ppgtt; +} + +static bool +needs_pd_load_post (struct intel_engine_cs *ring,struct intel_context *to) +{ +return (!to->legacy_hw_ctx.initialized || +i915_gem_context_is_default (to)) && + to->ppgtt &&IS_GEN8 (ring->dev); +} +static int do_switch (struct intel_engine_cs *ring,struct intel_context *to) {@@ -571,9 +598,6 @@ static int do_switch(struct intel_engine_cs *ring,u32 hw_flags =0 ;bool uninitialized = false;struct i915_vma *vma; -bool needs_pd_load_pre = ((INTEL_INFO (ring->dev)->gen <8 ) || - (ring != &dev_priv->ring[RCS ])) && to->ppgtt; -bool needs_pd_load_post = false;int ret, i;if (from !=NULL && ring == &dev_priv->ring[RCS ]) {@@ -581,7 +605,7 @@ static int do_switch(struct intel_engine_cs *ring,BUG_ON (!i915_gem_obj_is_pinned (from->legacy_hw_ctx.rcs_state)); } -if (from == to && !to->remap_slice) +if (should_skip_switch (ring, from, to))return 0 ;/* Trying to pin first makes error handling easier. */ @@ -599,7 +623,7 @@ static int do_switch(struct intel_engine_cs *ring,*/ from = ring->last_context; -if (needs_pd_load_pre) { +if (needs_pd_load_pre (ring, to)) {/* Older GENs and non render rings still want the load first, * "PP_DCLV followed by PP_DIR_BASE register through Load * Register Immediate commands in Ring Buffer before submitting @@ -644,16 +668,14 @@ static int do_switch(struct intel_engine_cs *ring,* XXX: If we implemented page directory eviction code, this * optimization needs to be removed. */ -if (!to->legacy_hw_ctx.initialized ||i915_gem_context_is_default (to)) { +if (!to->legacy_hw_ctx.initialized ||i915_gem_context_is_default (to)) hw_flags |=MI_RESTORE_INHIBIT ; - needs_pd_load_post = to->ppgtt &&IS_GEN8 (ring->dev); - } ret =mi_set_context (ring, to, hw_flags);if (ret) goto unpin_out; -if (needs_pd_load_post) { +if (needs_pd_load_post (ring, to)) { ret = to->ppgtt->switch_mm (to->ppgtt, ring);/* The hardware context switch is emitted, but we haven't * actually changed the state - so it's probably safe to bail Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
Signature · 7 lines
Daniel Vetter Software Engineer, Intel Corporation +41 (0) 79 365 57 48 - http://blog.ffwll.ch _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH v2 12/24 drm/i915: Track page table reload need
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
This patch was formerly known as, "Force pd restore when PDEs change, gen6-7." I had to change the name because it is needed for GEN8 too.
The real issue this is trying to solve is when a new object is mapped into the current address space. The GPU does not snoop the new mapping so we must do the gen specific action to reload the page tables.
GEN8 and GEN7 do differ in the way they load page tables for the RCS. GEN8 does so with the context restore, while GEN7 requires the proper load commands in the command streamer. Non-render is similar for both.
Caveat for GEN7
The docs say you cannot change the PDEs of a currently running context. We never map new PDEs of a running context, and expect them to be present - so I think this is okay. (We can unmap, but this should also be okay since we only unmap unreferenced objects that the GPU shouldn't be tryingto va->pa xlate.) The MI_SET_CONTEXT command does have a flag to signal that even if the context is the same, force a reload. It's unclear exactly what this does, but I have a hunch it's the right thing to do.
The logic assumes that we always emit a context switch after mapping new PDEs, and before we submit a batch. This is the case today, and has been the case since the inception of hardware contexts. A note in the comment let's the user know.
squash! drm/i915: Force pd restore when PDEs change, gen6-7
It's not just for gen8. If the current context has mappings change, we need a context reload to switch
v2: Rebased after ppgtt clean up patches. Split the warning for aliasing and true ppgtt options. And do not break aliasing ppgtt, where to->ppgtt is always null.
v3: Invalidate PPGTT TLBs inside alloc_va_range and teardown_va_range.
--- drivers/gpu/drm/i915/i915_gem_context.c | 27 ++++++++++++++++++++++----- drivers/gpu/drm/i915/i915_gem_execbuffer.c | 11 +++++++++++ drivers/gpu/drm/i915/i915_gem_gtt.c | 12 ++++++++++++ drivers/gpu/drm/i915/i915_gem_gtt.h | 2 ++ 4 files changed, 47 insertions(+), 5 deletions(-)
drivers/gpu/drm/i915/i915_gem_context.c +22 -5
@@ -567,8 +567,18 @@ static inline bool should_skip_switch(struct intel_engine_cs *ring,struct intel_context *from, struct intel_context *to) { -if (from == to && !to->remap_slice) -return true; + struct drm_i915_private *dev_priv = ring->dev->dev_private; + +if (to->remap_slice) +return false; + +if (to->ppgtt) { +if (from == to && !test_bit (ring->id, &to->ppgtt->base.pd_reload_mask)) +return true; + }else { +if (from == to && !test_bit (ring->id, &dev_priv->mm.aliasing_ppgtt->base.pd_reload_mask)) +return true; + }return false; }
@@ -585,9 +595,8 @@ needs_pd_load_pre(struct intel_engine_cs *ring, struct intel_context *to)static bool needs_pd_load_post (struct intel_engine_cs *ring,struct intel_context *to) { -return (!to->legacy_hw_ctx.initialized || -i915_gem_context_is_default (to)) && - to->ppgtt &&IS_GEN8 (ring->dev); +return IS_GEN8 (ring->dev) && + (to->ppgtt || &to->ppgtt->base.pd_reload_mask); } static int do_switch(struct intel_engine_cs *ring,
@@ -632,6 +641,12 @@ static int do_switch(struct intel_engine_cs *ring, ret = to->ppgtt->switch_mm (to->ppgtt, ring);if (ret) goto unpin_out; + +/* Doing a PD load always reloads the page dirs */ +if (to->ppgtt) +clear_bit (ring->id, &to->ppgtt->base.pd_reload_mask); +else +clear_bit (ring->id, &dev_priv->mm.aliasing_ppgtt->base.pd_reload_mask); }if (ring != &dev_priv->ring[RCS ]) {
@@ -670,6 +685,8 @@ static int do_switch(struct intel_engine_cs *ring,*/ if (!to->legacy_hw_ctx.initialized ||i915_gem_context_is_default (to)) hw_flags |=MI_RESTORE_INHIBIT ; +else if (to->ppgtt &&test_and_clear_bit (ring->id, &to->ppgtt->base.pd_reload_mask)) + hw_flags |=MI_FORCE_RESTORE ; ret =mi_set_context (ring, to, hw_flags);if (ret)
drivers/gpu/drm/i915/i915_gem_execbuffer.c +11 -0
@@ -1199,6 +1199,13 @@ i915_gem_ringbuffer_submission(struct drm_device *dev, struct drm_file *file,if (ret) goto error; +if (ctx->ppgtt) +WARN (ctx->ppgtt->base.pd_reload_mask & (1 <<ring->id), +"%s didn't clear reload\n" , ring->name); +else +WARN (dev_priv->mm.aliasing_ppgtt->base.pd_reload_mask & + (1 <<ring->id),"%s didn't clear reload\n" , ring->name); + instp_mode = args->flags &I915_EXEC_CONSTANTS_MASK ; instp_mask =I915_EXEC_CONSTANTS_MASK ;switch (instp_mode) {
@@ -1446,6 +1453,10 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,if (ret) goto err; +/* XXX: Reserve has possibly change PDEs which means we must do a +* context switch before we can coherently read some of the reserved +* VMAs. */ +/* The objects are in their final locations, apply the relocations. */ if (need_relocs) ret =i915_gem_execbuffer_relocate (eb);
drivers/gpu/drm/i915/i915_gem_gtt.c +12 -0
@@ -1126,6 +1126,15 @@ static void gen6_ppgtt_unmap_pages(struct i915_hw_ppgtt *ppgtt) 4096, PCI_DMA_BIDIRECTIONAL); } +/* PDE TLBs are a pain invalidate pre GEN8. It requires a context reload. If we +* are switching between contexts with the same LRCA, we also must do a force +* restore. +*/ +#define ppgtt_invalidate_tlbs (vm) do {\ +/* If current vm != vm, */ \ + vm->pd_reload_mask =INTEL_INFO (vm->dev)->ring_mask; \ +}while (0 ) + static int gen6_alloc_va_range(struct i915_address_space *vm,uint64_t start,uint64_t length) {
@@ -1154,6 +1163,7 @@ static int gen6_alloc_va_range(struct i915_address_space *vm, I915_PPGTT_PT_ENTRIES); } +ppgtt_invalidate_tlbs (vm);return 0 ; }
@@ -1169,6 +1179,8 @@ static void gen6_teardown_va_range(struct i915_address_space *vm,bitmap_clear (pt->used_ptes,gen6_pte_index (start),gen6_pte_count (start, length)); } + +ppgtt_invalidate_tlbs (vm); }static void gen6_ppgtt_free (struct i915_hw_ppgtt *ppgtt)
drivers/gpu/drm/i915/i915_gem_gtt.h +2 -0
@@ -226,6 +226,8 @@ struct i915_address_space {struct page *page; } scratch; +unsigned long pd_reload_mask; +/** * List of objects currently involved in rendering. *
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
Re: [PATCH v2 12/24] drm/i915: Track page table reload need
Quoted 142 lines · Michel Thierry
From: Ben Widawsky <[email protected]>
This patch was formerly known as, "Force pd restore when PDEs change, gen6-7." I had to change the name because it is needed for GEN8 too.
The real issue this is trying to solve is when a new object is mapped into the current address space. The GPU does not snoop the new mapping so we must do the gen specific action to reload the page tables.
GEN8 and GEN7 do differ in the way they load page tables for the RCS. GEN8 does so with the context restore, while GEN7 requires the proper load commands in the command streamer. Non-render is similar for both.
Caveat for GEN7
The docs say you cannot change the PDEs of a currently running context. We never map new PDEs of a running context, and expect them to be present - so I think this is okay. (We can unmap, but this should also be okay since we only unmap unreferenced objects that the GPU shouldn't be tryingtova->paxlate.) TheMI_SET_CONTEXTcommand does have a flag to signal that even if the context is the same, force a reload. It's unclear exactly what this does, but I have a hunch it's the right thing to do.The logic assumes that we always emit a context switch after mapping new PDEs, and before we submit a batch. This is the case today, and has been the case since the inception of hardware contexts. A note in the comment let's the user know.
Signed-off-by Ben Widawsky <[email protected]>squash! drm/i915: Force pd restore when PDEs change, gen6-7
It's not just for gen8. If the current context has mappings change, we need a context reload to switch
v2: Rebased after ppgtt clean up patches. Split the warning for aliasing and true ppgtt options. And do not break aliasing ppgtt, where
to->ppgttis always null.v3: Invalidate PPGTT TLBs inside
alloc_va_rangeandteardown_va_range.Signed-off-by Michel Thierry <[email protected]> (v2+)--- drivers/gpu/drm/i915/i915_gem_context.c | 27 ++++++++++++++++++++++----- drivers/gpu/drm/i915/i915_gem_execbuffer.c | 11 +++++++++++ drivers/gpu/drm/i915/i915_gem_gtt.c | 12 ++++++++++++ drivers/gpu/drm/i915/i915_gem_gtt.h | 2 ++ 4 files changed, 47 insertions(+), 5 deletions(-)drivers/gpu/drm/i915/i915_gem_context.c +22
-5@@ -567,8 +567,18 @@ static inline bool should_skip_switch(struct intel_engine_cs *ring,struct intel_context *from, struct intel_context *to) { -if (from == to && !to->remap_slice) -return true; + struct drm_i915_private *dev_priv = ring->dev->dev_private; + +if (to->remap_slice) +return false; + +if (to->ppgtt) { +if (from == to && !test_bit (ring->id, &to->ppgtt->base.pd_reload_mask)) +return true; + }else { +if (from == to && !test_bit (ring->id, &dev_priv->mm.aliasing_ppgtt->base.pd_reload_mask)) +return true; + }return false; }@@ -585,9 +595,8 @@ needs_pd_load_pre(struct intel_engine_cs *ring, struct intel_context *to)static bool needs_pd_load_post (struct intel_engine_cs *ring,struct intel_context *to) { -return (!to->legacy_hw_ctx.initialized || -i915_gem_context_is_default (to)) && - to->ppgtt &&IS_GEN8 (ring->dev); +return IS_GEN8 (ring->dev) && + (to->ppgtt || &to->ppgtt->base.pd_reload_mask); } static int do_switch(struct intel_engine_cs *ring,@@ -632,6 +641,12 @@ static int do_switch(struct intel_engine_cs *ring, ret = to->ppgtt->switch_mm (to->ppgtt, ring);if (ret) goto unpin_out; + +/* Doing a PD load always reloads the page dirs */ +if (to->ppgtt) +clear_bit (ring->id, &to->ppgtt->base.pd_reload_mask); +else +clear_bit (ring->id, &dev_priv->mm.aliasing_ppgtt->base.pd_reload_mask); }if (ring != &dev_priv->ring[RCS ]) {@@ -670,6 +685,8 @@ static int do_switch(struct intel_engine_cs *ring,*/ if (!to->legacy_hw_ctx.initialized ||i915_gem_context_is_default (to)) hw_flags |=MI_RESTORE_INHIBIT ; +else if (to->ppgtt &&test_and_clear_bit (ring->id, &to->ppgtt->base.pd_reload_mask)) + hw_flags |=MI_FORCE_RESTORE ; ret =mi_set_context (ring, to, hw_flags);if (ret)drivers/gpu/drm/i915/i915_gem_execbuffer.c +11
-0@@ -1199,6 +1199,13 @@ i915_gem_ringbuffer_submission(struct drm_device *dev, struct drm_file *file,if (ret) goto error; +if (ctx->ppgtt) +WARN (ctx->ppgtt->base.pd_reload_mask & (1 <<ring->id), +"%s didn't clear reload\n" , ring->name); +else +WARN (dev_priv->mm.aliasing_ppgtt->base.pd_reload_mask & + (1 <<ring->id),"%s didn't clear reload\n" , ring->name); + instp_mode = args->flags &I915_EXEC_CONSTANTS_MASK ; instp_mask =I915_EXEC_CONSTANTS_MASK ;switch (instp_mode) {@@ -1446,6 +1453,10 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,if (ret) goto err; +/* XXX: Reserve has possibly change PDEs which means we must do a +* context switch before we can coherently read some of the reserved +* VMAs. */ +/* The objects are in their final locations, apply the relocations. */ if (need_relocs) ret =i915_gem_execbuffer_relocate (eb);drivers/gpu/drm/i915/i915_gem_gtt.c +0
-0@@ -1126,6 +1126,15 @@ static void gen6_ppgtt_unmap_pages(struct i915_hw_ppgtt *ppgtt) 4096, PCI_DMA_BIDIRECTIONAL);
drivers/gpu/drm/i915/i915_gem_gtt.c static void gen6_ppgtt_unmap_pages(struct i915_hw_ppgtt *ppgtt) · line 1136
Quoted 10 lines
drivers/gpu/drm/i915/i915_gem_gtt.c +8
-0} +/* PDE TLBs are a pain invalidate pre GEN8. It requires a context reload. If we +* are switching between contexts with the same LRCA, we also must do a force +* restore. +*/ +#define ppgtt_invalidate_tlbs (vm) do {\ +/* If current vm != vm, */ \ + vm->pd_reload_mask =INTEL_INFO (vm->dev)->ring_mask; \ +}while (0 )
Again this should be a proper static inline. Also maybe call this mark_tlbs_dirty since that's what it does, the invalidate happens later on in the ctx switch code. In the same spirit:
s/pd_realod_mask/pd_dirty_rings/.
-Daniel
Quoted 26 lines
drivers/gpu/drm/i915/i915_gem_gtt.c +4
-0+static int gen6_alloc_va_range (struct i915_address_space *vm,uint64_t start,uint64_t length) {@@ -1154,6 +1163,7 @@ static int gen6_alloc_va_range(struct i915_address_space *vm, I915_PPGTT_PT_ENTRIES); } +ppgtt_invalidate_tlbs (vm);return 0 ; }@@ -1169,6 +1179,8 @@ static void gen6_teardown_va_range(struct i915_address_space *vm,bitmap_clear (pt->used_ptes,gen6_pte_index (start),gen6_pte_count (start, length)); } + +ppgtt_invalidate_tlbs (vm); }static void gen6_ppgtt_free (struct i915_hw_ppgtt *ppgtt)drivers/gpu/drm/i915/i915_gem_gtt.h +0
-0
drivers/gpu/drm/i915/i915_gem_gtt.h struct i915_address_space { · line 229
Quoted 4 lines
drivers/gpu/drm/i915/i915_gem_gtt.h +1
-0@@ -226,6 +226,8 @@ struct i915_address_space {struct page *page; } scratch; +unsigned long pd_reload_mask;
Conceptually this should be in i915_hw_ppgtt, not in struct i915_address_space. Anything holding up that movement that I don't see?
Quoted 11 lines
drivers/gpu/drm/i915/i915_gem_gtt.h +1
-0+/** * List of objects currently involved in rendering. * Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
Signature · 7 lines
Daniel Vetter Software Engineer, Intel Corporation +41 (0) 79 365 57 48 - http://blog.ffwll.ch _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH v2 13/24 drm/i915: Initialize all contexts
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
The problem is we're going to switch to a new context, which could be the default context. The plan was to use restore inhibit, which would be fine, except if we are using dynamic page tables (which we will). If we use dynamic page tables and we don't load new page tables, the previous page tables might go away, and future operations will fault.
CTXA runs.
switch to default, restore inhibit
CTXA dies and has its address space taken away.
Run CTXB, tries to save using the context A's address space - this fails.
The general solution is to make sure every context has it's own state, and its own address space. For cases when we must restore inhibit, first thing we do is load a valid address space. I thought this would be enough, but apparently there are references within the context itself which will refer to the old address space - therefore, we also must reinitialize.
It was tricky to track this down as we don't have much insight into what happens in a context save.
This is required for the next patch which enables dynamic page tables.
v2: to->ppgtt is only valid in full ppgtt.
--- drivers/gpu/drm/i915/i915_gem_context.c | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-)
drivers/gpu/drm/i915/i915_gem_context.c +11 -14
@@ -592,13 +592,6 @@ needs_pd_load_pre(struct intel_engine_cs *ring, struct intel_context *to) (ring != &dev_priv->ring[RCS])) && to->ppgtt; } -static bool -needs_pd_load_post (struct intel_engine_cs *ring,struct intel_context *to) -{ -return IS_GEN8 (ring->dev) && - (to->ppgtt || &to->ppgtt->base.pd_reload_mask); -} -static int do_switch (struct intel_engine_cs *ring,struct intel_context *to) {
@@ -679,20 +672,24 @@ static int do_switch(struct intel_engine_cs *ring,/* GEN8 does *not* require an explicit reload if the PDPs have been * setup, and we do not wish to move them. -* -* XXX: If we implemented page directory eviction code, this -* optimization needs to be removed. */ -if (!to->legacy_hw_ctx.initialized ||i915_gem_context_is_default (to)) +if (!to->legacy_hw_ctx.initialized) { hw_flags |=MI_RESTORE_INHIBIT ; -else if (to->ppgtt &&test_and_clear_bit (ring->id, &to->ppgtt->base.pd_reload_mask)) +/* NB: If we inhibit the restore, the context is not allowed to +* die because future work may end up depending on valid address +* space. This means we must enforce that a page table load +* occur when this occurs. */ + }else if (to->ppgtt &&test_and_clear_bit (ring->id, &to->ppgtt->base.pd_reload_mask)) hw_flags |=MI_FORCE_RESTORE ; ret =mi_set_context (ring, to, hw_flags);if (ret) goto unpin_out; -if (needs_pd_load_post (ring, to)) { +if (IS_GEN8 (ring->dev) && to->ppgtt && (hw_flags &MI_RESTORE_INHIBIT )) { +/* We have a valid page directory (scratch) to switch to. This +* allows the old VM to be freed. Note that if anything occurs +* between the set context, and here, we are f*cked */ ret = to->ppgtt->switch_mm (to->ppgtt, ring);/* The hardware context switch is emitted, but we haven't * actually changed the state - so it's probably safe to bail
@@ -742,7 +739,7 @@ static int do_switch(struct intel_engine_cs *ring,i915_gem_context_unreference (from); } - uninitialized = !to->legacy_hw_ctx.initialized && from ==NULL ; + uninitialized = !to->legacy_hw_ctx.initialized; to->legacy_hw_ctx.initialized = true; done:
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH v2 14/24 drm/i915: Finish gen6/7 dynamic page table allocation
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
This patch continues on the idea from the previous patch. From here on, in the steady state, PDEs are all pointing to the scratch page table (as recommended in the spec). When an object is allocated in the VA range, the code will determine if we need to allocate a page for the page table. Similarly when the object is destroyed, we will remove, and free the page table pointing the PDE back to the scratch page.
Following patches will work to unify the code a bit as we bring in GEN8 support. GEN6 and GEN8 are different enough that I had a hard time to get to this point with as much common code as I do.
The aliasing PPGTT must pre-allocate all of the page tables. There are a few reasons for this. Two trivial ones: aliasing ppgtt goes through the ggtt paths, so it's hard to maintain, we currently do not restore the default context (assuming the previous force reload is indeed necessary). Most importantly though, the only way (it seems from empirical evidence) to invalidate the CS TLBs on non-render ring is to either use ring sync (which requires actually stopping the rings in order to synchronize when the sync completes vs. where you are in execution), or to reload DCLV. Since without full PPGTT we do not ever reload the DCLV register, there is no good way to achieve this. The simplest solution is just to not support dynamic page table creation/destruction in the aliasing PPGTT.
We could always reload DCLV, but this seems like quite a bit of excess overhead only to save at most 2MB-4k of memory for the aliasing PPGTT page tables.
v2: Make the page table bitmap declared inside the function (Chris) Simplify the way scratching address space works.
Move the alloc/teardown tracepoints up a level in the call stack so that both all implementations get the trace.
v3: Updated trace event to spit out a name
v4: Aliasing ppgtt is now initialized differently (in setup global gtt)
v5: Rebase to latest code. Also removed unnecessary aliasing ppgtt check for trace, as it is no longer possible after the PPGTT cleanup patch series of a couple of months ago (Daniel).
--- drivers/gpu/drm/i915/i915_debugfs.c | 3 +- drivers/gpu/drm/i915/i915_gem.c | 2 + drivers/gpu/drm/i915/i915_gem_gtt.c | 128 ++++++++++++++++++++++++++++++++---- drivers/gpu/drm/i915/i915_trace.h | 115 ++++++++++++++++++++++++++++++++ 4 files changed, 236 insertions(+), 12 deletions(-)
drivers/gpu/drm/i915/i915_debugfs.c +2 -1
@@ -2149,6 +2149,8 @@ static void gen6_ppgtt_info(struct seq_file *m, struct drm_device *dev)seq_printf (m,"PP_DIR_BASE_READ: 0x%08x\n" ,I915_READ (RING_PP_DIR_BASE_READ (ring)));seq_printf (m,"PP_DIR_DCLV: 0x%08x\n" ,I915_READ (RING_PP_DIR_DCLV (ring))); } +seq_printf (m,"ECOCHK: 0x%08x\n\n" ,I915_READ (GAM_ECOCHK )); +if (dev_priv->mm.aliasing_ppgtt) {struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
@@ -2165,7 +2167,6 @@ static void gen6_ppgtt_info(struct seq_file *m, struct drm_device *dev) get_pid_task(file->pid, PIDTYPE_PID)->comm);idr_for_each (&file_priv->context_idr, per_file_ctx, m); } -seq_printf (m,"ECOCHK: 0x%08x\n" ,I915_READ (GAM_ECOCHK )); }static int i915_ppgtt_info (struct seq_file *m,void *data)
drivers/gpu/drm/i915/i915_gem.c +2 -0
@@ -3599,6 +3599,8 @@ search_free:/* allocate before insert / bind */ if (vma->vm->allocate_va_range) { +trace_i915_va_alloc (vma->vm, vma->node.start, vma->node.size, +VM_TO_TRACE_NAME (vma->vm)); ret = vma->vm->allocate_va_range (vma->vm, vma->node.start, vma->node.size);
drivers/gpu/drm/i915/i915_gem_gtt.c +117 -11
230 lines not on this page · raw message
drivers/gpu/drm/i915/i915_trace.h +115 -0
121 lines not on this page · raw message
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
Re: [PATCH v2 14/24] drm/i915: Finish gen6/7 dynamic page table allocation
Quoted 39 lines · Michel Thierry
From: Ben Widawsky <[email protected]>
This patch continues on the idea from the previous patch. From here on, in the steady state, PDEs are all pointing to the scratch page table (as recommended in the spec). When an object is allocated in the VA range, the code will determine if we need to allocate a page for the page table. Similarly when the object is destroyed, we will remove, and free the page table pointing the PDE back to the scratch page.
Following patches will work to unify the code a bit as we bring in GEN8 support. GEN6 and GEN8 are different enough that I had a hard time to get to this point with as much common code as I do.
The aliasing PPGTT must pre-allocate all of the page tables. There are a few reasons for this. Two trivial ones: aliasing ppgtt goes through the ggtt paths, so it's hard to maintain, we currently do not restore the default context (assuming the previous force reload is indeed necessary). Most importantly though, the only way (it seems from empirical evidence) to invalidate the CS TLBs on non-render ring is to either use ring sync (which requires actually stopping the rings in order to synchronize when the sync completes vs. where you are in execution), or to reload DCLV. Since without full PPGTT we do not ever reload the DCLV register, there is no good way to achieve this. The simplest solution is just to not support dynamic page table creation/destruction in the aliasing PPGTT.
We could always reload DCLV, but this seems like quite a bit of excess overhead only to save at most 2MB-4k of memory for the aliasing PPGTT page tables.
v2: Make the page table bitmap declared inside the function (Chris) Simplify the way scratching address space works.
Move the alloc/teardown tracepoints up a level in the call stack so that both all implementations get the trace.v3: Updated trace event to spit out a name
v4: Aliasing ppgtt is now initialized differently (in setup global gtt)
Quoted 7 lines
v5: Rebase to latest code. Also removed unnecessary aliasing ppgtt check for trace, as it is no longer possible after the PPGTT cleanup patch series of a couple of months ago (Daniel).
Cc Daniel Vetter <[email protected]>Signed-off-by Ben Widawsky <[email protected]>Signed-off-by Michel Thierry <[email protected]> (v4+)
The tracepoints should be split into a separate patch. Although the teardown stuff will likely disappear I guess ...
Two more comments below.
-Daniel
Quoted 58 lines
--- drivers/gpu/drm/i915/i915_debugfs.c | 3 +- drivers/gpu/drm/i915/i915_gem.c | 2 + drivers/gpu/drm/i915/i915_gem_gtt.c | 128 ++++++++++++++++++++++++++++++++---- drivers/gpu/drm/i915/i915_trace.h | 115 ++++++++++++++++++++++++++++++++ 4 files changed, 236 insertions(+), 12 deletions(-)drivers/gpu/drm/i915/i915_debugfs.c +2
-1@@ -2149,6 +2149,8 @@ static void gen6_ppgtt_info(struct seq_file *m, struct drm_device *dev)seq_printf (m,"PP_DIR_BASE_READ: 0x%08x\n" ,I915_READ (RING_PP_DIR_BASE_READ (ring)));seq_printf (m,"PP_DIR_DCLV: 0x%08x\n" ,I915_READ (RING_PP_DIR_DCLV (ring))); } +seq_printf (m,"ECOCHK: 0x%08x\n\n" ,I915_READ (GAM_ECOCHK )); +if (dev_priv->mm.aliasing_ppgtt) {struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;@@ -2165,7 +2167,6 @@ static void gen6_ppgtt_info(struct seq_file *m, struct drm_device *dev) get_pid_task(file->pid, PIDTYPE_PID)->comm);idr_for_each (&file_priv->context_idr, per_file_ctx, m); } -seq_printf (m,"ECOCHK: 0x%08x\n" ,I915_READ (GAM_ECOCHK )); }static int i915_ppgtt_info (struct seq_file *m,void *data)drivers/gpu/drm/i915/i915_gem.c +2
-0@@ -3599,6 +3599,8 @@ search_free:/* allocate before insert / bind */ if (vma->vm->allocate_va_range) { +trace_i915_va_alloc (vma->vm, vma->node.start, vma->node.size, +VM_TO_TRACE_NAME (vma->vm)); ret = vma->vm->allocate_va_range (vma->vm, vma->node.start, vma->node.size);drivers/gpu/drm/i915/i915_gem_gtt.c +5
-0@@ -1138,10 +1138,47 @@ static void gen6_ppgtt_unmap_pages(struct i915_hw_ppgtt *ppgtt)static int gen6_alloc_va_range (struct i915_address_space *vm,uint64_t start,uint64_t length) { +DECLARE_BITMAP (new_page_tables,GEN6_PPGTT_PD_ENTRIES ); +struct drm_device *dev = vm->dev; +struct drm_i915_private *dev_priv = dev->dev_private;struct i915_hw_ppgtt *ppgtt = container_of(vm, struct i915_hw_ppgtt, base);struct i915_pagetab *pt; +const uint32_t start_save = start, length_save = length;uint32_t pde, temp; +int ret;
drivers/gpu/drm/i915/i915_gem_gtt.c static void gen6_ppgtt_unmap_pages(struct i915_hw_ppgtt *ppgtt) · line 1159
Quoted 10 lines
drivers/gpu/drm/i915/i915_gem_gtt.c +10
-0+ +BUG_ON (upper_32_bits (start)); + +bitmap_zero (new_page_tables,GEN6_PPGTT_PD_ENTRIES ); + +/* The allocation is done in two stages so that we can bail out with +* minimal amount of pain. The first stage finds new page tables that +* need allocation. The second stage marks use ptes within the page +* tables. +*/
If we drop the bitmask tracking we could massively simplify this - checking just the various pt pointers should be enough?
Quoted 107 lines
drivers/gpu/drm/i915/i915_gem_gtt.c +71
-3+gen6_for_each_pde (pt, &ppgtt->pd, start, length, temp, pde) { +if (pt != ppgtt->scratch_pt) { +WARN_ON (bitmap_empty (pt->used_ptes,I915_PPGTT_PT_ENTRIES )); +continue ; + } + +/* We've already allocated a page table */ +WARN_ON (!bitmap_empty (pt->used_ptes,I915_PPGTT_PT_ENTRIES )); + + pt =alloc_pt_single (dev); +if (IS_ERR (pt)) { + ret =PTR_ERR (pt); + goto unwind_out; + } + + ppgtt->pd.page_tables[pde] = pt; +set_bit (pde, new_page_tables); +trace_i915_pagetable_alloc (vm, pde, start,GEN6_PDE_SHIFT ); + } + + start = start_save; + length = length_save;gen6_for_each_pde (pt, &ppgtt->pd, start, length, temp, pde) {int j;@@ -1159,12 +1196,35 @@ static int gen6_alloc_va_range(struct i915_address_space *vm, } } -bitmap_or (pt->used_ptes, pt->used_ptes, tmp_bitmap, +if (test_and_clear_bit (pde, new_page_tables)) +gen6_write_pdes (&ppgtt->pd, pde, pt); + +trace_i915_pagetable_map (vm, pde, pt, +gen6_pte_index (start), +gen6_pte_count (start, length), +I915_PPGTT_PT_ENTRIES ); +bitmap_or (pt->used_ptes, tmp_bitmap, pt->used_ptes,I915_PPGTT_PT_ENTRIES ); } +WARN_ON (!bitmap_empty (new_page_tables,GEN6_PPGTT_PD_ENTRIES )); + +/* Make sure write is complete before other code can use this page +* table. Also require for WC mapped PTEs */ +readl (dev_priv->gtt.gsm); +ppgtt_invalidate_tlbs (vm);return 0 ; + +unwind_out: +for_each_set_bit (pde, new_page_tables,GEN6_PPGTT_PD_ENTRIES ) { +struct i915_pagetab *pt = ppgtt->pd.page_tables[pde]; + ppgtt->pd.page_tables[pde] =NULL ; +free_pt_single (pt, vm->dev); + } + +ppgtt_invalidate_tlbs (vm); +return ret; } static void gen6_teardown_va_range(struct i915_address_space *vm,@@ -1176,8 +1236,27 @@ static void gen6_teardown_va_range(struct i915_address_space *vm,uint32_t pde, temp;gen6_for_each_pde (pt, &ppgtt->pd, start, length, temp, pde) { + +if (WARN (pt == ppgtt->scratch_pt, +"Tried to teardown scratch page vm %p. pde %u: %llx-%llx\n" , + vm, pde, start, start + length)) +continue ; + +trace_i915_pagetable_unmap (vm, pde, pt, +gen6_pte_index (start), +gen6_pte_count (start, length), +I915_PPGTT_PT_ENTRIES ); +bitmap_clear (pt->used_ptes,gen6_pte_index (start),gen6_pte_count (start, length)); + +if (bitmap_empty (pt->used_ptes,I915_PPGTT_PT_ENTRIES )) { +trace_i915_pagetable_destroy (vm, pde, + start &GENMASK_ULL (63 ,GEN6_PDE_SHIFT ), +GEN6_PDE_SHIFT ); +gen6_write_pdes (&ppgtt->pd, pde, ppgtt->scratch_pt); + ppgtt->pd.page_tables[pde] = ppgtt->scratch_pt; + } }ppgtt_invalidate_tlbs (vm);@@ -1187,9 +1266,13 @@ static void gen6_ppgtt_free(struct i915_hw_ppgtt *ppgtt) {int i; -for (i =0 ; i < ppgtt->num_pd_entries; i++) -free_pt_single (ppgtt->pd.page_tables[i], ppgtt->base.dev); +for (i =0 ; i < ppgtt->num_pd_entries; i++) { +struct i915_pagetab *pt = ppgtt->pd.page_tables[i]; +if (pt != ppgtt->scratch_pt) +free_pt_single (ppgtt->pd.page_tables[i], ppgtt->base.dev); + } +/* Consider putting this as part of pd free. */ free_pt_scratch (ppgtt->scratch_pt, ppgtt->base.dev);free_pd_single (&ppgtt->pd); }
drivers/gpu/drm/i915/i915_gem_gtt.c err_out: · line 1340
Quoted 5 lines
drivers/gpu/drm/i915/i915_gem_gtt.c +1
-1@@ -1254,7 +1337,7 @@ err_out:return ret; } -static int gen6_ppgtt_alloc (struct i915_hw_ppgtt *ppgtt) +static int gen6_ppgtt_alloc (struct i915_hw_ppgtt *ppgtt,bool preallocate_pt)
Imo it would be clearer to move the pt preallocation for alising ppgtt into the ppgtt_init function. Makes for a bit a bigger diff, but will result in less convoluted control flow since we should end up in a nice
if (alising)
/* create all pts */ else /* allocate&use scratch_pt */
Aside: Should we only allocate the scratch_pt for !aliasing?
Quoted 242 lines
drivers/gpu/drm/i915/i915_gem_gtt.c +30
-7{int ret;@@ -1262,10 +1345,14 @@ static int gen6_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt)if (ret)return ret; +if (!preallocate_pt) +return 0 ; + ret =alloc_pt_range (&ppgtt->pd,0 , ppgtt->num_pd_entries, ppgtt->base.dev);if (ret) { +free_pt_scratch (ppgtt->scratch_pt, ppgtt->base.dev);drm_mm_remove_node (&ppgtt->node);return ret; }@@ -1273,7 +1360,17 @@ static int gen6_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt)return 0 ; } -static int gen6_ppgtt_init (struct i915_hw_ppgtt *ppgtt) +static void gen6_scratch_va_range (struct i915_hw_ppgtt *ppgtt, +uint64_t start,uint64_t length) +{ +struct i915_pagetab *unused; +uint32_t pde, temp; + +gen6_for_each_pde (unused, &ppgtt->pd, start, length, temp, pde) + ppgtt->pd.page_tables[pde] = ppgtt->scratch_pt; +} + +static int gen6_ppgtt_init (struct i915_hw_ppgtt *ppgtt,bool aliasing) {struct drm_device *dev = ppgtt->base.dev;struct drm_i915_private *dev_priv = dev->dev_private;@@ -1289,7 +1386,7 @@ static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt) }else BUG (); - ret =gen6_ppgtt_alloc (ppgtt); + ret =gen6_ppgtt_alloc (ppgtt, aliasing);if (ret)return ret;@@ -1308,6 +1405,9 @@ static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt) ppgtt->pd_addr = (gen6_gtt_pte_t __iomem *)dev_priv->gtt.gsm + ppgtt->pd.pd_offset /sizeof (gen6_gtt_pte_t); +if (!aliasing) +gen6_scratch_va_range (ppgtt,0 , ppgtt->base.total); +gen6_write_page_range (dev_priv, &ppgtt->pd,0 , ppgtt->base.total); DRM_DEBUG_DRIVER("Allocated pde space (%ldM) at GTT entry: %lx\n",@@ -1320,7 +1420,8 @@ static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt)return 0 ; } -static int __hw_ppgtt_init (struct drm_device *dev,struct i915_hw_ppgtt *ppgtt) +static int __hw_ppgtt_init (struct drm_device *dev,struct i915_hw_ppgtt *ppgtt, +bool aliasing) {struct drm_i915_private *dev_priv = dev->dev_private;@@ -1328,7 +1429,7 @@ static int __hw_ppgtt_init(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt) ppgtt->base.scratch = dev_priv->gtt.base.scratch;if (INTEL_INFO (dev)->gen <8 ) -return gen6_ppgtt_init (ppgtt); +return gen6_ppgtt_init (ppgtt, aliasing);else return gen8_ppgtt_init (ppgtt, dev_priv->gtt.base.total); }@@ -1337,7 +1438,7 @@ int i915_ppgtt_init(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt)struct drm_i915_private *dev_priv = dev->dev_private;int ret =0 ; - ret =__hw_ppgtt_init (dev, ppgtt); + ret =__hw_ppgtt_init (dev, ppgtt, false);if (ret ==0 ) {kref_init (&ppgtt->ref); drm_mm_init(&ppgtt->base.mm, ppgtt->base.start,@@ -1445,9 +1546,14 @@ static void ppgtt_unbind_vma(struct i915_vma *vma) vma->node.start, vma->obj->base.size, true); -if (vma->vm->teardown_va_range) +if (vma->vm->teardown_va_range) { +trace_i915_va_teardown (vma->vm, + vma->node.start, vma->node.size, +VM_TO_TRACE_NAME (vma->vm)); + vma->vm->teardown_va_range (vma->vm, vma->node.start, vma->node.size); + } }extern int intel_iommu_gfx_mapped;@@ -1963,7 +2069,7 @@ static int i915_gem_setup_global_gtt(struct drm_device *dev,if (!ppgtt)return -ENOMEM ; - ret =__hw_ppgtt_init (dev, ppgtt); + ret =__hw_ppgtt_init (dev, ppgtt, true);if (ret !=0 )return ret;drivers/gpu/drm/i915/i915_trace.h +115
-0@@ -156,6 +156,121 @@ TRACE_EVENT(i915_vma_unbind, __entry->obj, __entry->offset, __entry->size, __entry->vm) ); +#define VM_TO_TRACE_NAME (vm) \ + (i915_is_ggtt(vm) ? "GGTT" : \ + "Private VM") + +DECLARE_EVENT_CLASS (i915_va, + TP_PROTO(struct i915_address_space *vm, u64 start, u64 length, const char *name), +TP_ARGS (vm, start, length, name), + +TP_STRUCT__entry ( + __field(struct i915_address_space *, vm) + __field(u64, start) + __field(u64, end) +__string (name, name) + ), + +TP_fast_assign ( + __entry->vm = vm; + __entry->start = start; + __entry->end = start + length; + __assign_str(name, name); + ), + +TP_printk ("vm=%p (%s), 0x%llx-0x%llx" , + __entry->vm,__get_str (name), __entry->start, __entry->end) +); + +DEFINE_EVENT (i915_va, i915_va_alloc, + TP_PROTO(struct i915_address_space *vm, u64 start, u64 length, const char *name), +TP_ARGS (vm, start, length, name) +); + +DEFINE_EVENT (i915_va, i915_va_teardown, + TP_PROTO(struct i915_address_space *vm, u64 start, u64 length, const char *name), +TP_ARGS (vm, start, length, name) +); + +DECLARE_EVENT_CLASS (i915_pagetable, + TP_PROTO(struct i915_address_space *vm, u32 pde, u64 start, u64 pde_shift), +TP_ARGS (vm, pde, start, pde_shift), + +TP_STRUCT__entry ( + __field(struct i915_address_space *, vm) + __field(u32, pde) + __field(u64, start) +__field (u64, end) + ), + +TP_fast_assign ( + __entry->vm = vm; + __entry->pde = pde; + __entry->start = start; + __entry->end = (start + (1ULL << pde_shift)) & ~((1ULL << pde_shift)-1); + ), + +TP_printk ("vm=%p, pde=%d (0x%llx-0x%llx)" , + __entry->vm, __entry->pde, __entry->start, __entry->end) +); + +DEFINE_EVENT (i915_pagetable, i915_pagetable_alloc, + TP_PROTO(struct i915_address_space *vm, u32 pde, u64 start, u64 pde_shift), +TP_ARGS (vm, pde, start, pde_shift) +); + +DEFINE_EVENT (i915_pagetable, i915_pagetable_destroy, + TP_PROTO(struct i915_address_space *vm, u32 pde, u64 start, u64 pde_shift), +TP_ARGS (vm, pde, start, pde_shift) +); + +/* Avoid extra math because we only support two sizes. The format is defined by +* bitmap_scnprintf. Each 32 bits is 8 HEX digits followed by comma */ +#define TRACE_PT_SIZE (bits) \ + ((((bits) == 1024) ? 288 : 144) + 1) + +DECLARE_EVENT_CLASS (i915_pagetable_update, + TP_PROTO(struct i915_address_space *vm, u32 pde, + struct i915_pagetab *pt, u32 first, u32 len, size_t bits), +TP_ARGS (vm, pde, pt, first, len, bits), + +TP_STRUCT__entry ( + __field(struct i915_address_space *, vm) + __field(u32, pde) + __field(u32, first) + __field(u32, last) +__dynamic_array (char, cur_ptes,TRACE_PT_SIZE (bits)) + ), + +TP_fast_assign ( + __entry->vm = vm; + __entry->pde = pde; + __entry->first = first; + __entry->last = first + len; + +bitmap_scnprintf (__get_str (cur_ptes), +TRACE_PT_SIZE (bits), + pt->used_ptes, + bits); + ), + +TP_printk ("vm=%p, pde=%d, updating %u:%u\t%s" , + __entry->vm, __entry->pde, __entry->last, __entry->first, +__get_str (cur_ptes)) +); + +DEFINE_EVENT (i915_pagetable_update, i915_pagetable_map, + TP_PROTO(struct i915_address_space *vm, u32 pde, + struct i915_pagetab *pt, u32 first, u32 len, size_t bits), +TP_ARGS (vm, pde, pt, first, len, bits) +); + +DEFINE_EVENT (i915_pagetable_update, i915_pagetable_unmap, + TP_PROTO(struct i915_address_space *vm, u32 pde, + struct i915_pagetab *pt, u32 first, u32 len, size_t bits), +TP_ARGS (vm, pde, pt, first, len, bits) +); + TRACE_EVENT(i915_gem_object_change_domain, TP_PROTO(struct drm_i915_gem_object *obj, u32 old_read, u32 old_write), TP_ARGS(obj, old_read, old_write),Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
Signature · 7 lines
Daniel Vetter Software Engineer, Intel Corporation +41 (0) 79 365 57 48 - http://blog.ffwll.ch _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
Re: [PATCH v2 14/24] drm/i915: Finish gen6/7 dynamic page table allocation
Quoted 240 lines · Daniel Vetter
Quoted 39 lines · Michel Thierry
From: Ben Widawsky <[email protected]>
This patch continues on the idea from the previous patch. From here on, in the steady state, PDEs are all pointing to the scratch page table (as recommended in the spec). When an object is allocated in the VA range, the code will determine if we need to allocate a page for the page table. Similarly when the object is destroyed, we will remove, and free the page table pointing the PDE back to the scratch page.
Following patches will work to unify the code a bit as we bring in GEN8 support. GEN6 and GEN8 are different enough that I had a hard time to get to this point with as much common code as I do.
The aliasing PPGTT must pre-allocate all of the page tables. There are a few reasons for this. Two trivial ones: aliasing ppgtt goes through the ggtt paths, so it's hard to maintain, we currently do not restore the default context (assuming the previous force reload is indeed necessary). Most importantly though, the only way (it seems from empirical evidence) to invalidate the CS TLBs on non-render ring is to either use ring sync (which requires actually stopping the rings in order to synchronize when the sync completes vs. where you are in execution), or to reload DCLV. Since without full PPGTT we do not ever reload the DCLV register, there is no good way to achieve this. The simplest solution is just to not support dynamic page table creation/destruction in the aliasing PPGTT.
We could always reload DCLV, but this seems like quite a bit of excess overhead only to save at most 2MB-4k of memory for the aliasing PPGTT page tables.
v2: Make the page table bitmap declared inside the function (Chris) Simplify the way scratching address space works.
Move the alloc/teardown tracepoints up a level in the call stack so that both all implementations get the trace.v3: Updated trace event to spit out a name
v4: Aliasing ppgtt is now initialized differently (in setup global gtt)
Quoted 7 lines
v5: Rebase to latest code. Also removed unnecessary aliasing ppgtt check for trace, as it is no longer possible after the PPGTT cleanup patch series of a couple of months ago (Daniel).
Cc Daniel Vetter <[email protected]>Signed-off-by Ben Widawsky <[email protected]>Signed-off-by Michel Thierry <[email protected]> (v4+)The tracepoints should be split into a separate patch. Although the teardown stuff will likely disappear I guess ...
Two more comments below.
-DanielQuoted 58 lines
--- drivers/gpu/drm/i915/i915_debugfs.c | 3 +- drivers/gpu/drm/i915/i915_gem.c | 2 + drivers/gpu/drm/i915/i915_gem_gtt.c | 128 ++++++++++++++++++++++++++++++++---- drivers/gpu/drm/i915/i915_trace.h | 115 ++++++++++++++++++++++++++++++++ 4 files changed, 236 insertions(+), 12 deletions(-)drivers/gpu/drm/i915/i915_debugfs.c +2
-1@@ -2149,6 +2149,8 @@ static void gen6_ppgtt_info(struct seq_file *m, struct drm_device *dev)seq_printf (m,"PP_DIR_BASE_READ: 0x%08x\n" ,I915_READ (RING_PP_DIR_BASE_READ (ring)));seq_printf (m,"PP_DIR_DCLV: 0x%08x\n" ,I915_READ (RING_PP_DIR_DCLV (ring))); } +seq_printf (m,"ECOCHK: 0x%08x\n\n" ,I915_READ (GAM_ECOCHK )); +if (dev_priv->mm.aliasing_ppgtt) {struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;@@ -2165,7 +2167,6 @@ static void gen6_ppgtt_info(struct seq_file *m, struct drm_device *dev) get_pid_task(file->pid, PIDTYPE_PID)->comm);idr_for_each (&file_priv->context_idr, per_file_ctx, m); } -seq_printf (m,"ECOCHK: 0x%08x\n" ,I915_READ (GAM_ECOCHK )); }static int i915_ppgtt_info (struct seq_file *m,void *data)drivers/gpu/drm/i915/i915_gem.c +2
-0@@ -3599,6 +3599,8 @@ search_free:/* allocate before insert / bind */ if (vma->vm->allocate_va_range) { +trace_i915_va_alloc (vma->vm, vma->node.start, vma->node.size, +VM_TO_TRACE_NAME (vma->vm)); ret = vma->vm->allocate_va_range (vma->vm, vma->node.start, vma->node.size);drivers/gpu/drm/i915/i915_gem_gtt.c +5
-0@@ -1138,10 +1138,47 @@ static void gen6_ppgtt_unmap_pages(struct i915_hw_ppgtt *ppgtt)static int gen6_alloc_va_range (struct i915_address_space *vm,uint64_t start,uint64_t length) { +DECLARE_BITMAP (new_page_tables,GEN6_PPGTT_PD_ENTRIES ); +struct drm_device *dev = vm->dev; +struct drm_i915_private *dev_priv = dev->dev_private;struct i915_hw_ppgtt *ppgtt = container_of(vm, struct i915_hw_ppgtt, base);struct i915_pagetab *pt; +const uint32_t start_save = start, length_save = length;uint32_t pde, temp; +int ret;drivers/gpu/drm/i915/i915_gem_gtt.c static void gen6_ppgtt_unmap_pages(struct i915_hw_ppgtt *ppgtt) · line 1159
Quoted 10 lines
drivers/gpu/drm/i915/i915_gem_gtt.c +10
-0+ +BUG_ON (upper_32_bits (start)); + +bitmap_zero (new_page_tables,GEN6_PPGTT_PD_ENTRIES ); + +/* The allocation is done in two stages so that we can bail out with +* minimal amount of pain. The first stage finds new page tables that +* need allocation. The second stage marks use ptes within the page +* tables. +*/ If we drop the bitmask tracking we could massively simplify this - checking just the various pt pointers should be enough?
Quoted 107 lines
drivers/gpu/drm/i915/i915_gem_gtt.c +71
-3+gen6_for_each_pde (pt, &ppgtt->pd, start, length, temp, pde) { +if (pt != ppgtt->scratch_pt) { +WARN_ON (bitmap_empty (pt->used_ptes,I915_PPGTT_PT_ENTRIES )); +continue ; + } + +/* We've already allocated a page table */ +WARN_ON (!bitmap_empty (pt->used_ptes,I915_PPGTT_PT_ENTRIES )); + + pt =alloc_pt_single (dev); +if (IS_ERR (pt)) { + ret =PTR_ERR (pt); + goto unwind_out; + } + + ppgtt->pd.page_tables[pde] = pt; +set_bit (pde, new_page_tables); +trace_i915_pagetable_alloc (vm, pde, start,GEN6_PDE_SHIFT ); + } + + start = start_save; + length = length_save;gen6_for_each_pde (pt, &ppgtt->pd, start, length, temp, pde) {int j;@@ -1159,12 +1196,35 @@ static int gen6_alloc_va_range(struct i915_address_space *vm, } } -bitmap_or (pt->used_ptes, pt->used_ptes, tmp_bitmap, +if (test_and_clear_bit (pde, new_page_tables)) +gen6_write_pdes (&ppgtt->pd, pde, pt); + +trace_i915_pagetable_map (vm, pde, pt, +gen6_pte_index (start), +gen6_pte_count (start, length), +I915_PPGTT_PT_ENTRIES ); +bitmap_or (pt->used_ptes, tmp_bitmap, pt->used_ptes,I915_PPGTT_PT_ENTRIES ); } +WARN_ON (!bitmap_empty (new_page_tables,GEN6_PPGTT_PD_ENTRIES )); + +/* Make sure write is complete before other code can use this page +* table. Also require for WC mapped PTEs */ +readl (dev_priv->gtt.gsm); +ppgtt_invalidate_tlbs (vm);return 0 ; + +unwind_out: +for_each_set_bit (pde, new_page_tables,GEN6_PPGTT_PD_ENTRIES ) { +struct i915_pagetab *pt = ppgtt->pd.page_tables[pde]; + ppgtt->pd.page_tables[pde] =NULL ; +free_pt_single (pt, vm->dev); + } + +ppgtt_invalidate_tlbs (vm); +return ret; } static void gen6_teardown_va_range(struct i915_address_space *vm,@@ -1176,8 +1236,27 @@ static void gen6_teardown_va_range(struct i915_address_space *vm,uint32_t pde, temp;gen6_for_each_pde (pt, &ppgtt->pd, start, length, temp, pde) { + +if (WARN (pt == ppgtt->scratch_pt, +"Tried to teardown scratch page vm %p. pde %u: %llx-%llx\n" , + vm, pde, start, start + length)) +continue ; + +trace_i915_pagetable_unmap (vm, pde, pt, +gen6_pte_index (start), +gen6_pte_count (start, length), +I915_PPGTT_PT_ENTRIES ); +bitmap_clear (pt->used_ptes,gen6_pte_index (start),gen6_pte_count (start, length)); + +if (bitmap_empty (pt->used_ptes,I915_PPGTT_PT_ENTRIES )) { +trace_i915_pagetable_destroy (vm, pde, + start &GENMASK_ULL (63 ,GEN6_PDE_SHIFT ), +GEN6_PDE_SHIFT ); +gen6_write_pdes (&ppgtt->pd, pde, ppgtt->scratch_pt); + ppgtt->pd.page_tables[pde] = ppgtt->scratch_pt; + } }ppgtt_invalidate_tlbs (vm);@@ -1187,9 +1266,13 @@ static void gen6_ppgtt_free(struct i915_hw_ppgtt *ppgtt) {int i; -for (i =0 ; i < ppgtt->num_pd_entries; i++) -free_pt_single (ppgtt->pd.page_tables[i], ppgtt->base.dev); +for (i =0 ; i < ppgtt->num_pd_entries; i++) { +struct i915_pagetab *pt = ppgtt->pd.page_tables[i]; +if (pt != ppgtt->scratch_pt) +free_pt_single (ppgtt->pd.page_tables[i], ppgtt->base.dev); + } +/* Consider putting this as part of pd free. */ free_pt_scratch (ppgtt->scratch_pt, ppgtt->base.dev);free_pd_single (&ppgtt->pd); }drivers/gpu/drm/i915/i915_gem_gtt.c err_out: · line 1340
Quoted 5 lines
drivers/gpu/drm/i915/i915_gem_gtt.c +1
-1@@ -1254,7 +1337,7 @@ err_out:return ret; } -static int gen6_ppgtt_alloc (struct i915_hw_ppgtt *ppgtt) +static int gen6_ppgtt_alloc (struct i915_hw_ppgtt *ppgtt,bool preallocate_pt)Imo it would be clearer to move the pt preallocation for alising ppgtt into the
ppgtt_initfunction. Makes for a bit a bigger diff, but will result in less convoluted control flow since we should end up in a nice
Quoted 6 lines
if (alising)
/* create all pts */ else /* allocate&use scratch_pt */Aside: Should we only allocate the
scratch_ptfor !aliasing?
The next patch version will have the changes.
About the scratch_pt, I'm not sure if it's a requirement in gen6/7 (to point unused page tables to the scratch, e.g. if there's less than 2GB). We know in gen8 that's the case, and systems with less than 4GB must have the remaining PDPs set to scratch page.
Quoted 239 lines
Quoted 239 lines
drivers/gpu/drm/i915/i915_gem_gtt.c +30
-7{int ret;@@ -1262,10 +1345,14 @@ static int gen6_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt)if (ret)return ret; +if (!preallocate_pt) +return 0 ; + ret =alloc_pt_range (&ppgtt->pd,0 , ppgtt->num_pd_entries, ppgtt->base.dev);if (ret) { +free_pt_scratch (ppgtt->scratch_pt, ppgtt->base.dev);drm_mm_remove_node (&ppgtt->node);return ret; }@@ -1273,7 +1360,17 @@ static int gen6_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt)return 0 ; } -static int gen6_ppgtt_init (struct i915_hw_ppgtt *ppgtt) +static void gen6_scratch_va_range (struct i915_hw_ppgtt *ppgtt, +uint64_t start,uint64_t length) +{ +struct i915_pagetab *unused; +uint32_t pde, temp; + +gen6_for_each_pde (unused, &ppgtt->pd, start, length, temp, pde) + ppgtt->pd.page_tables[pde] = ppgtt->scratch_pt; +} + +static int gen6_ppgtt_init (struct i915_hw_ppgtt *ppgtt,bool aliasing) {struct drm_device *dev = ppgtt->base.dev;struct drm_i915_private *dev_priv = dev->dev_private;@@ -1289,7 +1386,7 @@ static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt) }else BUG (); - ret =gen6_ppgtt_alloc (ppgtt); + ret =gen6_ppgtt_alloc (ppgtt, aliasing);if (ret)return ret;@@ -1308,6 +1405,9 @@ static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt) ppgtt->pd_addr = (gen6_gtt_pte_t __iomem *)dev_priv->gtt.gsm + ppgtt->pd.pd_offset /sizeof (gen6_gtt_pte_t); +if (!aliasing) +gen6_scratch_va_range (ppgtt,0 , ppgtt->base.total); +gen6_write_page_range (dev_priv, &ppgtt->pd,0 , ppgtt->base.total); DRM_DEBUG_DRIVER("Allocated pde space (%ldM) at GTT entry: %lx\n",@@ -1320,7 +1420,8 @@ static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt)return 0 ; } -static int __hw_ppgtt_init (struct drm_device *dev,struct i915_hw_ppgtt *ppgtt) +static int __hw_ppgtt_init (struct drm_device *dev,struct i915_hw_ppgtt *ppgtt, +bool aliasing) {struct drm_i915_private *dev_priv = dev->dev_private;@@ -1328,7 +1429,7 @@ static int __hw_ppgtt_init(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt) ppgtt->base.scratch = dev_priv->gtt.base.scratch;if (INTEL_INFO (dev)->gen <8 ) -return gen6_ppgtt_init (ppgtt); +return gen6_ppgtt_init (ppgtt, aliasing);else return gen8_ppgtt_init (ppgtt, dev_priv->gtt.base.total); }@@ -1337,7 +1438,7 @@ int i915_ppgtt_init(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt)struct drm_i915_private *dev_priv = dev->dev_private;int ret =0 ; - ret =__hw_ppgtt_init (dev, ppgtt); + ret =__hw_ppgtt_init (dev, ppgtt, false);if (ret ==0 ) {kref_init (&ppgtt->ref); drm_mm_init(&ppgtt->base.mm, ppgtt->base.start,@@ -1445,9 +1546,14 @@ static void ppgtt_unbind_vma(struct i915_vma *vma) vma->node.start, vma->obj->base.size, true); -if (vma->vm->teardown_va_range) +if (vma->vm->teardown_va_range) { +trace_i915_va_teardown (vma->vm, + vma->node.start, vma->node.size, +VM_TO_TRACE_NAME (vma->vm)); + vma->vm->teardown_va_range (vma->vm, vma->node.start, vma->node.size); + } }extern int intel_iommu_gfx_mapped;@@ -1963,7 +2069,7 @@ static int i915_gem_setup_global_gtt(struct drm_device *dev,if (!ppgtt)return -ENOMEM ; - ret =__hw_ppgtt_init (dev, ppgtt); + ret =__hw_ppgtt_init (dev, ppgtt, true);if (ret !=0 )return ret;drivers/gpu/drm/i915/i915_trace.h +115
-0@@ -156,6 +156,121 @@ TRACE_EVENT(i915_vma_unbind, __entry->obj, __entry->offset, __entry->size, __entry->vm) ); +#define VM_TO_TRACE_NAME (vm) \ + (i915_is_ggtt(vm) ? "GGTT" : \ + "Private VM") + +DECLARE_EVENT_CLASS (i915_va, + TP_PROTO(struct i915_address_space *vm, u64 start, u64 length, const char *name), +TP_ARGS (vm, start, length, name), + +TP_STRUCT__entry ( + __field(struct i915_address_space *, vm) + __field(u64, start) + __field(u64, end) +__string (name, name) + ), + +TP_fast_assign ( + __entry->vm = vm; + __entry->start = start; + __entry->end = start + length; + __assign_str(name, name); + ), + +TP_printk ("vm=%p (%s), 0x%llx-0x%llx" , + __entry->vm,__get_str (name), __entry->start, __entry->end) +); + +DEFINE_EVENT (i915_va, i915_va_alloc, + TP_PROTO(struct i915_address_space *vm, u64 start, u64 length, const char *name), +TP_ARGS (vm, start, length, name) +); + +DEFINE_EVENT (i915_va, i915_va_teardown, + TP_PROTO(struct i915_address_space *vm, u64 start, u64 length, const char *name), +TP_ARGS (vm, start, length, name) +); + +DECLARE_EVENT_CLASS (i915_pagetable, + TP_PROTO(struct i915_address_space *vm, u32 pde, u64 start, u64 pde_shift), +TP_ARGS (vm, pde, start, pde_shift), + +TP_STRUCT__entry ( + __field(struct i915_address_space *, vm) + __field(u32, pde) + __field(u64, start) +__field (u64, end) + ), + +TP_fast_assign ( + __entry->vm = vm; + __entry->pde = pde; + __entry->start = start; + __entry->end = (start + (1ULL << pde_shift)) & ~((1ULL << pde_shift)-1); + ), + +TP_printk ("vm=%p, pde=%d (0x%llx-0x%llx)" , + __entry->vm, __entry->pde, __entry->start, __entry->end) +); + +DEFINE_EVENT (i915_pagetable, i915_pagetable_alloc, + TP_PROTO(struct i915_address_space *vm, u32 pde, u64 start, u64 pde_shift), +TP_ARGS (vm, pde, start, pde_shift) +); + +DEFINE_EVENT (i915_pagetable, i915_pagetable_destroy, + TP_PROTO(struct i915_address_space *vm, u32 pde, u64 start, u64 pde_shift), +TP_ARGS (vm, pde, start, pde_shift) +); + +/* Avoid extra math because we only support two sizes. The format is defined by +* bitmap_scnprintf. Each 32 bits is 8 HEX digits followed by comma */ +#define TRACE_PT_SIZE (bits) \ + ((((bits) == 1024) ? 288 : 144) + 1) + +DECLARE_EVENT_CLASS (i915_pagetable_update, + TP_PROTO(struct i915_address_space *vm, u32 pde, + struct i915_pagetab *pt, u32 first, u32 len, size_t bits), +TP_ARGS (vm, pde, pt, first, len, bits), + +TP_STRUCT__entry ( + __field(struct i915_address_space *, vm) + __field(u32, pde) + __field(u32, first) + __field(u32, last) +__dynamic_array (char, cur_ptes,TRACE_PT_SIZE (bits)) + ), + +TP_fast_assign ( + __entry->vm = vm; + __entry->pde = pde; + __entry->first = first; + __entry->last = first + len; + +bitmap_scnprintf (__get_str (cur_ptes), +TRACE_PT_SIZE (bits), + pt->used_ptes, + bits); + ), + +TP_printk ("vm=%p, pde=%d, updating %u:%u\t%s" , + __entry->vm, __entry->pde, __entry->last, __entry->first, +__get_str (cur_ptes)) +); + +DEFINE_EVENT (i915_pagetable_update, i915_pagetable_map, + TP_PROTO(struct i915_address_space *vm, u32 pde, + struct i915_pagetab *pt, u32 first, u32 len, size_t bits), +TP_ARGS (vm, pde, pt, first, len, bits) +); + +DEFINE_EVENT (i915_pagetable_update, i915_pagetable_unmap, + TP_PROTO(struct i915_address_space *vm, u32 pde, + struct i915_pagetab *pt, u32 first, u32 len, size_t bits), +TP_ARGS (vm, pde, pt, first, len, bits) +); + TRACE_EVENT(i915_gem_object_change_domain, TP_PROTO(struct drm_i915_gem_object *obj, u32 old_read, u32 old_write), TP_ARGS(obj, old_read, old_write),Signature · 1 line
2.1.1
Quoted 4 lines
Quoted 4 lines
_______________________________________________
Intel-gfx mailing list
[email protected]
http://lists.freedesktop.org/mailman/listinfo/intel-gfx
_______________________________________________
Intel-gfx mailing list
[email protected]
http://lists.freedesktop.org/mailman/listinfo/intel-gfx
Re: [PATCH v2 14/24] drm/i915: Finish gen6/7 dynamic page table allocation
Quoted 7 lines · Michel Thierry
Quoted 1 line · Daniel Vetter
Aside: Should we only allocate the
scratch_ptfor !aliasing?The next patch version will have the changes.
About thescratch_pt, I'm not sure if it's a requirement in gen6/7 (to point unused page tables to the scratch, e.g. if there's less than 2GB). We know in gen8 that's the case, and systems with less than 4GB must have the remaining PDPs set to scratch page.
Assuming I understand things correctly we need the scratch_pt for replacing the intermediate levels to collapse the tree a bit. Which is useful both on gen7 and gen8+.
My question was to move the allocation into the full-ppgtt code (i.e. i915.enable_ppgtt=2, not =1) since we don't really need this for aliasing ppgtt mode. That should help a bit with readability since it makes it clearer which things are used in which modes of the driver. -Daniel
Signature · 7 lines
Daniel Vetter Software Engineer, Intel Corporation +41 (0) 79 365 57 48 - http://blog.ffwll.ch _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH v2 15/24 drm/i915/bdw: Use dynamic allocation idioms on free
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
The page directory freer is left here for now as it's still useful given that GEN8 still preallocates. Once the allocation functions are broken up into more discrete chunks, we'll follow suit and destroy this leftover piece.
v2: Match trace_i915_va_teardown params
v3: Multiple rebases.
--- drivers/gpu/drm/i915/i915_gem_gtt.c | 54 +++++++++++++++++++++++-------------- drivers/gpu/drm/i915/i915_gem_gtt.h | 46 +++++++++++++++++++++++++++++++ 2 files changed, 80 insertions(+), 20 deletions(-)
drivers/gpu/drm/i915/i915_gem_gtt.c +34 -20
@@ -579,27 +579,32 @@ static void gen8_ppgtt_insert_entries(struct i915_address_space *vm, } } -static void gen8_free_page_tables (struct i915_pagedir *pd,struct drm_device *dev) +static void gen8_teardown_va_range (struct i915_address_space *vm, +uint64_t start,uint64_t length) { -int i; - -if (!pd->page) -return ; - -for (i =0 ; i <GEN8_PDES_PER_PAGE ; i++) { -free_pt_single (pd->page_tables[i], dev); - pd->page_tables[i] =NULL ; +struct i915_hw_ppgtt *ppgtt = + container_of(vm, struct i915_hw_ppgtt, base); +struct i915_pagedir *pd; +struct i915_pagetab *pt; +uint64_t temp; +uint32_t pdpe, pde; + +gen8_for_each_pdpe (pd, &ppgtt->pdp, start, length, temp, pdpe) { +uint64_t pd_len =gen8_clamp_pd (start, length); +uint64_t pd_start = start; +gen8_for_each_pde (pt, pd, pd_start, pd_len, temp, pde) { +free_pt_single (pt, vm->dev); + } +free_pd_single (pd); } } -static void gen8_ppgtt_free (struct i915_hw_ppgtt *ppgtt) +/* This function will die soon */ +static void gen8_free_full_pagedir (struct i915_hw_ppgtt *ppgtt,int i) { -int i; - -for (i =0 ; i < ppgtt->num_pd_pages; i++) { -gen8_free_page_tables (ppgtt->pdp.pagedir[i], ppgtt->base.dev); -free_pd_single (ppgtt->pdp.pagedir[i]); - } +gen8_teardown_va_range (&ppgtt->base, + i <<GEN8_PDPE_SHIFT , + (1 <<GEN8_PDPE_SHIFT )); }static void gen8_ppgtt_unmap_pages (struct i915_hw_ppgtt *ppgtt)
@@ -614,19 +619,28 @@ static void gen8_ppgtt_unmap_pages(struct i915_hw_ppgtt *ppgtt)continue ;pci_unmap_page (hwdev, ppgtt->pdp.pagedir[i]->daddr,PAGE_SIZE , -PCI_DMA_BIDIRECTIONAL ); +PCI_DMA_BIDIRECTIONAL );for (j =0 ; j <GEN8_PDES_PER_PAGE ; j++) {struct i915_pagedir *pd = ppgtt->pdp.pagedir[i]; -struct i915_pagetab *pt = pd->page_tables[j]; +struct i915_pagetab *pt = pd->page_tables[j];dma_addr_t addr = pt->daddr;if (addr)pci_unmap_page (hwdev, addr,PAGE_SIZE , -PCI_DMA_BIDIRECTIONAL ); +PCI_DMA_BIDIRECTIONAL ); } } } +static void gen8_ppgtt_free (struct i915_hw_ppgtt *ppgtt) +{ +trace_i915_va_teardown (&ppgtt->base, + ppgtt->base.start, ppgtt->base.total, +VM_TO_TRACE_NAME (&ppgtt->base)); +gen8_teardown_va_range (&ppgtt->base, + ppgtt->base.start, ppgtt->base.total); +} +static void gen8_ppgtt_cleanup (struct i915_address_space *vm) { struct i915_hw_ppgtt *ppgtt =
@@ -651,7 +665,7 @@ static int gen8_ppgtt_allocate_page_tables(struct i915_hw_ppgtt *ppgtt) unwind_out:while (i--) -gen8_free_page_tables (ppgtt->pdp.pagedir[i], ppgtt->base.dev); +gen8_free_full_pagedir (ppgtt, i);return -ENOMEM ; }
drivers/gpu/drm/i915/i915_gem_gtt.h +46 -0
52 lines not on this page · raw message
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH v2 16/24 drm/i915/bdw: pagedirs rework allocation
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
Start using gen8_for_each_pdpe macro to allocate the page directories.
--- drivers/gpu/drm/i915/i915_gem_gtt.c | 43 ++++++++++++++++++++++++++----------- 1 file changed, 31 insertions(+), 12 deletions(-)
drivers/gpu/drm/i915/i915_gem_gtt.c +31 -12
@@ -594,8 +594,10 @@ static void gen8_teardown_va_range(struct i915_address_space *vm,uint64_t pd_start = start;gen8_for_each_pde (pt, pd, pd_start, pd_len, temp, pde) {free_pt_single (pt, vm->dev); + pd->page_tables[pde] =NULL ; }free_pd_single (pd); + ppgtt->pdp.pagedir[pdpe] =NULL ; } }
@@ -670,25 +672,39 @@ unwind_out:return -ENOMEM ; } -static int gen8_ppgtt_allocate_page_directories (struct i915_hw_ppgtt *ppgtt, -const int max_pdp) +static int gen8_ppgtt_alloc_pagedirs (struct i915_pagedirpo *pdp, +uint64_t start, +uint64_t length) { -int i; +struct i915_hw_ppgtt *ppgtt = + container_of(pdp, struct i915_hw_ppgtt, pdp); +struct i915_pagedir *unused; +uint64_t temp; +uint32_t pdpe; -for (i =0 ; i < max_pdp; i++) { - ppgtt->pdp.pagedir[i] =alloc_pd_single (); -if (IS_ERR (ppgtt->pdp.pagedir[i])) +/* FIXME: PPGTT container_of won't work for 64b */ +BUG_ON ((start + length) >0x800000000ULL ); + +gen8_for_each_pdpe (unused, pdp, start, length, temp, pdpe) { +BUG_ON (unused); + pdp->pagedir[pdpe] =alloc_pd_single (); +if (IS_ERR (ppgtt->pdp.pagedir[pdpe])) goto unwind_out; + + ppgtt->num_pd_pages++; } - ppgtt->num_pd_pages = max_pdp;BUG_ON (ppgtt->num_pd_pages >GEN8_LEGACY_PDPES );return 0 ; unwind_out: -while (i--) -free_pd_single (ppgtt->pdp.pagedir[i]); +while (pdpe--) { +free_pd_single (ppgtt->pdp.pagedir[pdpe]); + ppgtt->num_pd_pages--; + } + +WARN_ON (ppgtt->num_pd_pages);return -ENOMEM ; }
@@ -698,7 +714,8 @@ static int gen8_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt, {int ret; - ret =gen8_ppgtt_allocate_page_directories (ppgtt, max_pdp); + ret =gen8_ppgtt_alloc_pagedirs (&ppgtt->pdp, ppgtt->base.start, + ppgtt->base.total);if (ret)return ret;
@@ -775,6 +792,10 @@ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size)if (size % (1 <<30 ))DRM_INFO ("Pages will be wasted unless GTT size (%llu) is divisible by 1GB\n" , size); + ppgtt->base.start =0 ; + ppgtt->base.total = size; +BUG_ON (ppgtt->base.total ==0 ); +/* 1. Do all our allocations for page directories and page tables. */ ret =gen8_ppgtt_alloc (ppgtt, max_pdp);if (ret)
@@ -822,8 +843,6 @@ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size) ppgtt->base.clear_range = gen8_ppgtt_clear_range; ppgtt->base.insert_entries = gen8_ppgtt_insert_entries; ppgtt->base.cleanup = gen8_ppgtt_cleanup; - ppgtt->base.start =0 ; - ppgtt->base.total = ppgtt->num_pd_entries *GEN8_PTES_PER_PAGE *PAGE_SIZE ;DRM_DEBUG_DRIVER ("Allocated %d pages for page directories (%d wasted)\n" , ppgtt->num_pd_pages, ppgtt->num_pd_pages - max_pdp);
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH v2 17/24 drm/i915/bdw: pagetable allocation rework
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
Start using gen8_for_each_pde macro to allocate page tables.
--- drivers/gpu/drm/i915/i915_gem_gtt.c | 54 ++++++++++++++++++++----------------- drivers/gpu/drm/i915/i915_gem_gtt.h | 10 +++++++ 2 files changed, 39 insertions(+), 25 deletions(-)
drivers/gpu/drm/i915/i915_gem_gtt.c +29 -25
@@ -601,14 +601,6 @@ static void gen8_teardown_va_range(struct i915_address_space *vm, } } -/* This function will die soon */ -static void gen8_free_full_pagedir (struct i915_hw_ppgtt *ppgtt,int i) -{ -gen8_teardown_va_range (&ppgtt->base, - i <<GEN8_PDPE_SHIFT , - (1 <<GEN8_PDPE_SHIFT )); -} -static void gen8_ppgtt_unmap_pages (struct i915_hw_ppgtt *ppgtt) {struct pci_dev *hwdev = ppgtt->base.dev->pdev;
@@ -652,22 +644,27 @@ static void gen8_ppgtt_cleanup(struct i915_address_space *vm)gen8_ppgtt_free (ppgtt); } -static int gen8_ppgtt_allocate_page_tables (struct i915_hw_ppgtt *ppgtt) +static int gen8_ppgtt_alloc_pagetabs (struct i915_pagedir *pd, +uint64_t start, +uint64_t length, +struct drm_device *dev) { -int i, ret; +struct i915_pagetab *unused; +uint64_t temp; +uint32_t pde; -for (i =0 ; i < ppgtt->num_pd_pages; i++) { - ret =alloc_pt_range (ppgtt->pdp.pagedir[i], -0 ,GEN8_PDES_PER_PAGE , ppgtt->base.dev); -if (ret) +gen8_for_each_pde (unused, pd, start, length, temp, pde) { +BUG_ON (unused); + pd->page_tables[pde] =alloc_pt_single (dev); +if (IS_ERR (pd->page_tables[pde])) goto unwind_out; }return 0 ; unwind_out: -while (i--) -gen8_free_full_pagedir (ppgtt, i); +while (pde--) +free_pt_single (pd->page_tables[pde], dev);return -ENOMEM ; }
@@ -710,20 +707,28 @@ unwind_out: }static int gen8_ppgtt_alloc (struct i915_hw_ppgtt *ppgtt, -const int max_pdp) +uint64_t start, +uint64_t length) { +struct i915_pagedir *pd; +uint64_t temp; +uint32_t pdpe;int ret; - ret =gen8_ppgtt_alloc_pagedirs (&ppgtt->pdp, ppgtt->base.start, - ppgtt->base.total); + ret =gen8_ppgtt_alloc_pagedirs (&ppgtt->pdp, start, length);if (ret)return ret; - ret =gen8_ppgtt_allocate_page_tables (ppgtt); -if (ret) - goto err_out; +gen8_for_each_pdpe (pd, &ppgtt->pdp, start, length, temp, pdpe) { + ret =gen8_ppgtt_alloc_pagetabs (pd, start, length, + ppgtt->base.dev); +if (ret) + goto err_out; + + ppgtt->num_pd_entries +=GEN8_PDES_PER_PAGE ; + } - ppgtt->num_pd_entries = max_pdp *GEN8_PDES_PER_PAGE ; +BUG_ON (pdpe > ppgtt->num_pd_pages);return 0 ;
@@ -794,10 +799,9 @@ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size) ppgtt->base.start =0 ; ppgtt->base.total = size; -BUG_ON (ppgtt->base.total ==0 );/* 1. Do all our allocations for page directories and page tables. */ - ret =gen8_ppgtt_alloc (ppgtt, max_pdp); + ret =gen8_ppgtt_alloc (ppgtt, ppgtt->base.start, ppgtt->base.total);if (ret)return ret;
drivers/gpu/drm/i915/i915_gem_gtt.h +10 -0
@@ -403,6 +403,16 @@ static inline uint32_t gen6_pde_index(uint32_t addr) temp = min(temp, length), \ start += temp, length -= temp) +/* Clamp length to the next pagetab boundary */ +static inline uint64_t gen8_clamp_pt (uint64_t start,uint64_t length) +{ +uint64_t next_pt =ALIGN (start +1 ,1 <<GEN8_PDE_SHIFT ); +if (next_pt > (start + length)) +return length; + +return next_pt - start; +} +/* Clamp length to the next pagedir boundary */ static inline uint64_t gen8_clamp_pd (uint64_t start,uint64_t length) {
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH v2 18/24 drm/i915/bdw: Update pdp switch and point unused PDPs to scratch page
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
One important part of this patch is we now write a scratch page directory into any unused PDP descriptors. This matters for 2 reasons, first, we're not allowed to just use 0, or an invalid pointer, and second, we must wipe out any previous contents from the last context.
The latter point only matters with full PPGTT. The former point only effect platforms with less than 4GB memory.
v2: Updated commit message to point that we must set unused PDPs to the scratch page.
--- drivers/gpu/drm/i915/i915_gem_gtt.c | 29 ++++++++++++++++++----------- drivers/gpu/drm/i915/i915_gem_gtt.h | 5 ++++- 2 files changed, 22 insertions(+), 12 deletions(-)
drivers/gpu/drm/i915/i915_gem_gtt.c +18 -11
@@ -445,8 +445,9 @@ static struct i915_pagedir *alloc_pd_single(void) }/* Broadwell Page Directory Pointer Descriptors */ -static int gen8_write_pdp (struct intel_engine_cs *ring,unsigned entry, -uint64_t val) +static int gen8_write_pdp (struct intel_engine_cs *ring, +unsigned entry, +dma_addr_t addr) {int ret;
@@ -458,10 +459,10 @@ static int gen8_write_pdp(struct intel_engine_cs *ring, unsigned entry,intel_ring_emit (ring,MI_LOAD_REGISTER_IMM (1 ));intel_ring_emit (ring,GEN8_RING_PDP_UDW (ring, entry)); -intel_ring_emit (ring, (u32)(val >>32 )); +intel_ring_emit (ring,upper_32_bits (addr));intel_ring_emit (ring,MI_LOAD_REGISTER_IMM (1 ));intel_ring_emit (ring,GEN8_RING_PDP_LDW (ring, entry)); -intel_ring_emit (ring, (u32)(val)); +intel_ring_emit (ring,lower_32_bits (addr));intel_ring_advance (ring);return 0 ;
@@ -472,12 +473,12 @@ static int gen8_mm_switch(struct i915_hw_ppgtt *ppgtt, {int i, ret; -/* bit of a hack to find the actual last used pd */ -int used_pd = ppgtt->num_pd_entries /GEN8_PDES_PER_PAGE ; - -for (i = used_pd -1 ; i >=0 ; i--) { -dma_addr_t addr = ppgtt->pdp.pagedir[i]->daddr; - ret =gen8_write_pdp (ring, i, addr); +for (i =GEN8_LEGACY_PDPES -1 ; i >=0 ; i--) { +struct i915_pagedir *pd = ppgtt->pdp.pagedir[i]; +dma_addr_t pd_daddr = pd ? pd->daddr : ppgtt->scratch_pd->daddr; +/* The page directory might be NULL, but we need to clear out +* whatever the previous context might have used. */ + ret =gen8_write_pdp (ring, i, pd_daddr);if (ret)return ret; }
@@ -800,10 +801,16 @@ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size) ppgtt->base.start =0 ; ppgtt->base.total = size; + ppgtt->scratch_pd =alloc_pt_scratch (ppgtt->base.dev); +if (IS_ERR (ppgtt->scratch_pd)) +return PTR_ERR (ppgtt->scratch_pd); +/* 1. Do all our allocations for page directories and page tables. */ ret =gen8_ppgtt_alloc (ppgtt, ppgtt->base.start, ppgtt->base.total); -if (ret) +if (ret) { +free_pt_scratch (ppgtt->scratch_pd, ppgtt->base.dev);return ret; + }/* * 2. Create DMA mappings for the page directories and page tables.
drivers/gpu/drm/i915/i915_gem_gtt.h +4 -1
@@ -283,7 +283,10 @@ struct i915_hw_ppgtt {struct i915_pagedir pd; }; -struct i915_pagetab *scratch_pt; +union { +struct i915_pagetab *scratch_pt; +struct i915_pagetab *scratch_pd;/* Just need the daddr */ + };struct drm_i915_file_private *file_priv;
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH v2 19/24 drm/i915: num_pd_pages/num_pd_entries isn't useful
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
These values are never quite useful for dynamic allocations of the page tables. Getting rid of them will help prevent later confusion.
--- drivers/gpu/drm/i915/i915_debugfs.c | 2 -- drivers/gpu/drm/i915/i915_gem_gtt.c | 68 ++++++++++++------------------------- drivers/gpu/drm/i915/i915_gem_gtt.h | 7 ++-- 3 files changed, 27 insertions(+), 50 deletions(-)
drivers/gpu/drm/i915/i915_debugfs.c +0 -2
@@ -2117,8 +2117,6 @@ static void gen8_ppgtt_info(struct seq_file *m, struct drm_device *dev)if (!ppgtt)return ; -seq_printf (m,"Page directories: %d\n" , ppgtt->num_pd_pages); -seq_printf (m,"Page tables: %d\n" , ppgtt->num_pd_entries);for_each_ring (ring, dev_priv, unused) {seq_printf (m,"%s\n" , ring->name);for (i =0 ; i <4 ; i++) {
drivers/gpu/drm/i915/i915_gem_gtt.c +22 -46
160 lines not on this page · raw message
drivers/gpu/drm/i915/i915_gem_gtt.h +5 -2
@@ -276,8 +276,6 @@ struct i915_hw_ppgtt {struct i915_address_space base;struct kref ref;struct drm_mm_node node; -unsigned num_pd_entries; -unsigned num_pd_pages;/* gen8+ */ union {struct i915_pagedirpo pdp;struct i915_pagedir pd;
@@ -343,6 +341,11 @@ struct i915_gtt { temp = min(temp, (unsigned)length), \ start += temp, length -= temp) +#define gen6_for_all_pdes (pt, ppgtt, iter) \ + for (iter = 0, pt = ppgtt->pd.page_tables[iter]; \ + iter < gen6_pde_index(ppgtt->base.total); \ + pt = ppgtt->pd.page_tables[++iter]) +static inline uint32_t i915_pte_index (uint64_t address,uint32_t pde_shift) {const uint32_t mask =NUM_PTE (pde_shift) -1 ;
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH v2 20/24 drm/i915: Extract PPGTT param from pagedir alloc
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
Now that we don't need to trace num_pd_pages, we may as well kill all need for the PPGTT structure in the alloc_pagedirs. This is very useful for when we move to 48b addressing, and the PDP isn't the root of the page table structure.
The param is replaced with drm_device, which is an unavoidable wart throughout the series. (in other words, not extra flagrant).
--- drivers/gpu/drm/i915/i915_gem_gtt.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-)
drivers/gpu/drm/i915/i915_gem_gtt.c +2 -4
@@ -674,8 +674,6 @@ static int gen8_ppgtt_alloc_pagedirs(struct i915_pagedirpo *pdp, uint64_t start, uint64_t length) { -struct i915_hw_ppgtt *ppgtt = - container_of(pdp, struct i915_hw_ppgtt, pdp);struct i915_pagedir *unused;uint64_t temp;uint32_t pdpe;
@@ -686,7 +684,7 @@ static int gen8_ppgtt_alloc_pagedirs(struct i915_pagedirpo *pdp,gen8_for_each_pdpe (unused, pdp, start, length, temp, pdpe) {BUG_ON (unused); pdp->pagedir[pdpe] =alloc_pd_single (); -if (IS_ERR (ppgtt->pdp.pagedir[pdpe])) +if (IS_ERR (pdp->pagedir[pdpe])) goto unwind_out; }
@@ -694,7 +692,7 @@ static int gen8_ppgtt_alloc_pagedirs(struct i915_pagedirpo *pdp, unwind_out:while (pdpe--) -free_pd_single (ppgtt->pdp.pagedir[pdpe]); +free_pd_single (pdp->pagedir[pdpe]);return -ENOMEM ; }
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH v2 21/24 drm/i915/bdw: Split out mappings
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
When we do dynamic page table allocations for gen8, we'll need to have more control over how and when we map page tables, similar to gen6. In particular, DMA mappings for page directories/tables occur at allocation time.
This patch adds the functionality and calls it at init, which should have no functional change.
The PDPEs are still a special case for now. We'll need a function for that in the future as well.
--- drivers/gpu/drm/i915/i915_gem_gtt.c | 186 ++++++++++++++---------------------- 1 file changed, 72 insertions(+), 114 deletions(-)
drivers/gpu/drm/i915/i915_gem_gtt.c +72 -114
@@ -415,21 +415,23 @@ err_out:return ret; } -static void __free_pd_single (struct i915_pagedir *pd) +static void __free_pd_single (struct i915_pagedir *pd,struct drm_device *dev) { +i915_dma_unmap_single (pd, dev);__free_page (pd->page);kfree (pd); } -#define free_pd_single (pd) do { \ +#define free_pd_single (pd, dev) do { \ if ((pd)->page) { \ - __free_pd_single(pd); \ + __free_pd_single(pd, dev); \ } \ } while (0) -static struct i915_pagedir *alloc_pd_single (void ) +static struct i915_pagedir *alloc_pd_single (struct drm_device *dev) {struct i915_pagedir *pd; +int ret; pd =kzalloc (sizeof (*pd),GFP_KERNEL );if (!pd)
@@ -441,6 +443,13 @@ static struct i915_pagedir *alloc_pd_single(void)return ERR_PTR (-ENOMEM ); } + ret =i915_dma_map_px_single (pd, dev); +if (ret) { +__free_page (pd->page); +kfree (pd); +return ERR_PTR (ret); + } +return pd; }
@@ -580,6 +589,36 @@ static void gen8_ppgtt_insert_entries(struct i915_address_space *vm, } } +static void __gen8_do_map_pt (gen8_ppgtt_pde_t *pde, +struct i915_pagetab *pt, +struct drm_device *dev) +{ +gen8_ppgtt_pde_t entry = +gen8_pde_encode (dev, pt->daddr,I915_CACHE_LLC ); + *pde = entry; +} + +/* It's likely we'll map more than one pagetable at a time. This function will +* save us unnecessary kmap calls, but do no more functionally than multiple +* calls to map_pt. */ +static void gen8_map_pagetable_range (struct i915_pagedir *pd, +uint64_t start, +uint64_t length, +struct drm_device *dev) +{ +gen8_ppgtt_pde_t *pagedir =kmap_atomic (pd->page); +struct i915_pagetab *pt; +uint64_t temp, pde; + +gen8_for_each_pde (pt, pd, start, length, temp, pde) +__gen8_do_map_pt (pagedir + pde, pt, dev); + +if (!HAS_LLC (dev)) +drm_clflush_virt_range (pagedir,PAGE_SIZE ); + +kunmap_atomic (pagedir); +} +static void gen8_teardown_va_range (struct i915_address_space *vm,uint64_t start,uint64_t length) {
@@ -597,7 +636,7 @@ static void gen8_teardown_va_range(struct i915_address_space *vm,free_pt_single (pt, vm->dev); pd->page_tables[pde] =NULL ; } -free_pd_single (pd); +free_pd_single (pd, vm->dev); ppgtt->pdp.pagedir[pdpe] =NULL ; } }
@@ -629,9 +668,6 @@ static void gen8_ppgtt_unmap_pages(struct i915_hw_ppgtt *ppgtt)static void gen8_ppgtt_free (struct i915_hw_ppgtt *ppgtt) { -trace_i915_va_teardown (&ppgtt->base, - ppgtt->base.start, ppgtt->base.total, -VM_TO_TRACE_NAME (&ppgtt->base));gen8_teardown_va_range (&ppgtt->base, ppgtt->base.start, ppgtt->base.total); }
@@ -672,7 +708,8 @@ unwind_out:static int gen8_ppgtt_alloc_pagedirs (struct i915_pagedirpo *pdp,uint64_t start, -uint64_t length) +uint64_t length, +struct drm_device *dev) {struct i915_pagedir *unused;uint64_t temp;
@@ -683,7 +720,7 @@ static int gen8_ppgtt_alloc_pagedirs(struct i915_pagedirpo *pdp,gen8_for_each_pdpe (unused, pdp, start, length, temp, pdpe) {BUG_ON (unused); - pdp->pagedir[pdpe] =alloc_pd_single (); + pdp->pagedir[pdpe] =alloc_pd_single (dev);if (IS_ERR (pdp->pagedir[pdpe])) goto unwind_out; }
@@ -692,21 +729,25 @@ static int gen8_ppgtt_alloc_pagedirs(struct i915_pagedirpo *pdp, unwind_out:while (pdpe--) -free_pd_single (pdp->pagedir[pdpe]); +free_pd_single (pdp->pagedir[pdpe], dev);return -ENOMEM ; } -static int gen8_ppgtt_alloc (struct i915_hw_ppgtt *ppgtt, -uint64_t start, -uint64_t length) +static int gen8_alloc_va_range (struct i915_address_space *vm, +uint64_t start, +uint64_t length) { +struct i915_hw_ppgtt *ppgtt = + container_of(vm, struct i915_hw_ppgtt, base);struct i915_pagedir *pd; +const uint64_t orig_start = start;uint64_t temp;uint32_t pdpe;int ret; - ret =gen8_ppgtt_alloc_pagedirs (&ppgtt->pdp, start, length); + ret =gen8_ppgtt_alloc_pagedirs (&ppgtt->pdp, start, length, + ppgtt->base.dev);if (ret)return ret;
@@ -719,133 +760,50 @@ static int gen8_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt,return 0 ; -/* TODO: Check this for all cases */ err_out: -gen8_ppgtt_free (ppgtt); +gen8_teardown_va_range (vm, orig_start, start);return ret; } -static int gen8_ppgtt_setup_page_directories (struct i915_hw_ppgtt *ppgtt, -const int pd) -{ -dma_addr_t pd_addr; -int ret; - - pd_addr =pci_map_page (ppgtt->base.dev->pdev, - ppgtt->pdp.pagedir[pd]->page,0 , -PAGE_SIZE ,PCI_DMA_BIDIRECTIONAL ); - - ret =pci_dma_mapping_error (ppgtt->base.dev->pdev, pd_addr); -if (ret) -return ret; - - ppgtt->pdp.pagedir[pd]->daddr = pd_addr; - -return 0 ; -} - -static int gen8_ppgtt_setup_page_tables (struct i915_hw_ppgtt *ppgtt, -const int pd, -const int pt) -{ -dma_addr_t pt_addr; -struct i915_pagedir *pdir = ppgtt->pdp.pagedir[pd]; -struct i915_pagetab *ptab = pdir->page_tables[pt]; -struct page *p = ptab->page; -int ret; - - pt_addr =pci_map_page (ppgtt->base.dev->pdev, - p,0 ,PAGE_SIZE ,PCI_DMA_BIDIRECTIONAL ); - ret =pci_dma_mapping_error (ppgtt->base.dev->pdev, pt_addr); -if (ret) -return ret; - - ptab->daddr = pt_addr; - -return 0 ; -} -/** * GEN8 legacy ppgtt programming is accomplished through a max 4 PDP registers * with a net effect resembling a 2-level page table in normal x86 terms. Each * PDP represents 1GB of memory 4 * 512 * 512 * 4096 = 4GB legacy 32b address * space. * -* FIXME: split allocation into smaller pieces. For now we only ever do this -* once, but with full PPGTT, the multiple contiguous allocations will be bad. -* TODO: Do something with the size parameter */ static int gen8_ppgtt_init (struct i915_hw_ppgtt *ppgtt,uint64_t size) { -const int max_pdp =DIV_ROUND_UP (size,1 <<30 ); -int i, j, ret; - -if (size % (1 <<30 )) -DRM_INFO ("Pages will be wasted unless GTT size (%llu) is divisible by 1GB\n" , size); +struct i915_pagedir *pd; +uint64_t temp, start =0 ; +const uint64_t orig_length = size; +uint32_t pdpe; +int ret; ppgtt->base.start =0 ; ppgtt->base.total = size; + ppgtt->base.clear_range = gen8_ppgtt_clear_range; + ppgtt->base.insert_entries = gen8_ppgtt_insert_entries; + ppgtt->base.cleanup = gen8_ppgtt_cleanup; + ppgtt->switch_mm = gen8_mm_switch; ppgtt->scratch_pd =alloc_pt_scratch (ppgtt->base.dev);if (IS_ERR (ppgtt->scratch_pd))return PTR_ERR (ppgtt->scratch_pd); -/* 1. Do all our allocations for page directories and page tables. */ - ret =gen8_ppgtt_alloc (ppgtt, ppgtt->base.start, ppgtt->base.total); + ret =gen8_alloc_va_range (&ppgtt->base, start, size);if (ret) {free_pt_scratch (ppgtt->scratch_pd, ppgtt->base.dev);return ret; } -/* -* 2. Create DMA mappings for the page directories and page tables. -*/ -for (i =0 ; i < max_pdp; i++) { - ret =gen8_ppgtt_setup_page_directories (ppgtt, i); -if (ret) - goto bail; - -for (j =0 ; j <GEN8_PDES_PER_PAGE ; j++) { - ret =gen8_ppgtt_setup_page_tables (ppgtt, i, j); -if (ret) - goto bail; - } - } - -/* -* 3. Map all the page directory entires to point to the page tables -* we've allocated. -* -* For now, the PPGTT helper functions all require that the PDEs are -* plugged in correctly. So we do that now/here. For aliasing PPGTT, we -* will never need to touch the PDEs again. -*/ -for (i =0 ; i < max_pdp; i++) { -struct i915_pagedir *pd = ppgtt->pdp.pagedir[i]; -gen8_ppgtt_pde_t *pd_vaddr; - pd_vaddr =kmap_atomic (ppgtt->pdp.pagedir[i]->page); -for (j =0 ; j <GEN8_PDES_PER_PAGE ; j++) { -struct i915_pagetab *pt = pd->page_tables[j]; -dma_addr_t addr = pt->daddr; - pd_vaddr[j] =gen8_pde_encode (ppgtt->base.dev, addr, -I915_CACHE_LLC ); - } -if (!HAS_LLC (ppgtt->base.dev)) -drm_clflush_virt_range (pd_vaddr,PAGE_SIZE ); -kunmap_atomic (pd_vaddr); - } + start =0 ; + size = orig_length; - ppgtt->switch_mm = gen8_mm_switch; - ppgtt->base.clear_range = gen8_ppgtt_clear_range; - ppgtt->base.insert_entries = gen8_ppgtt_insert_entries; - ppgtt->base.cleanup = gen8_ppgtt_cleanup; +gen8_for_each_pdpe (pd, &ppgtt->pdp, start, size, temp, pdpe) +gen8_map_pagetable_range (pd, start, size, ppgtt->base.dev);return 0 ; - -bail: -gen8_ppgtt_unmap_pages (ppgtt); -gen8_ppgtt_free (ppgtt); -return ret; }static void gen6_dump_ppgtt (struct i915_hw_ppgtt *ppgtt,struct seq_file *m)
@@ -1295,7 +1253,7 @@ static void gen6_ppgtt_free(struct i915_hw_ppgtt *ppgtt)/* Consider putting this as part of pd free. */ free_pt_scratch (ppgtt->scratch_pt, ppgtt->base.dev); -free_pd_single (&ppgtt->pd); +free_pd_single (&ppgtt->pd, ppgtt->base.dev); }static void gen6_ppgtt_cleanup (struct i915_address_space *vm)
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH v2 22/24 drm/i915/bdw: begin bitmap tracking
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
Like with gen6/7, we can enable bitmap tracking with all the preallocations to make sure things actually don't blow up.
--- drivers/gpu/drm/i915/i915_gem_gtt.c | 121 +++++++++++++++++++++++++----------- drivers/gpu/drm/i915/i915_gem_gtt.h | 24 +++++++ 2 files changed, 108 insertions(+), 37 deletions(-)
drivers/gpu/drm/i915/i915_gem_gtt.c +84 -37
@@ -417,8 +417,12 @@ err_out:static void __free_pd_single (struct i915_pagedir *pd,struct drm_device *dev) { +WARN (!bitmap_empty (pd->used_pdes,GEN8_PDES_PER_PAGE ), +"Free page directory with %d used pages\n" , +bitmap_weight (pd->used_pdes,GEN8_PDES_PER_PAGE ));i915_dma_unmap_single (pd, dev);__free_page (pd->page); +kfree (pd->used_pdes);kfree (pd); }
@@ -431,26 +435,35 @@ static void __free_pd_single(struct i915_pagedir *pd, struct drm_device *dev)static struct i915_pagedir *alloc_pd_single (struct drm_device *dev) {struct i915_pagedir *pd; -int ret; +int ret = -ENOMEM ; pd =kzalloc (sizeof (*pd),GFP_KERNEL );if (!pd)return ERR_PTR (-ENOMEM ); + pd->used_pdes =kcalloc (BITS_TO_LONGS (GEN8_PDES_PER_PAGE ), +sizeof (*pd->used_pdes),GFP_KERNEL ); +if (!pd->used_pdes) + goto free_pd; + pd->page =alloc_page (GFP_KERNEL | __GFP_ZERO); -if (!pd->page) { -kfree (pd); -return ERR_PTR (-ENOMEM ); - } +if (!pd->page) + goto free_bitmap; ret =i915_dma_map_px_single (pd, dev); -if (ret) { -__free_page (pd->page); -kfree (pd); -return ERR_PTR (ret); - } +if (ret) + goto free_page;return pd; + +free_page: +__free_page (pd->page); +free_bitmap: +kfree (pd->used_pdes); +free_pd: +kfree (pd); + +return ERR_PTR (ret); }/* Broadwell Page Directory Pointer Descriptors */
@@ -632,36 +645,47 @@ static void gen8_teardown_va_range(struct i915_address_space *vm,gen8_for_each_pdpe (pd, &ppgtt->pdp, start, length, temp, pdpe) {uint64_t pd_len =gen8_clamp_pd (start, length);uint64_t pd_start = start; -gen8_for_each_pde (pt, pd, pd_start, pd_len, temp, pde) { -free_pt_single (pt, vm->dev); - pd->page_tables[pde] =NULL ; - } -free_pd_single (pd, vm->dev); - ppgtt->pdp.pagedir[pdpe] =NULL ; - } -} -static void gen8_ppgtt_unmap_pages (struct i915_hw_ppgtt *ppgtt) -{ -struct pci_dev *hwdev = ppgtt->base.dev->pdev; -int i, j; - -for (i =0 ; i <GEN8_PDES_PER_PAGE ; i++) { -/* TODO: In the future we'll support sparse mappings, so this -* will have to change. */ -if (!ppgtt->pdp.pagedir[i]->daddr) +/* Page directories might not be present since the macro rounds +* down, and up. +*/ +if (!pd) { +WARN (test_bit (pdpe, ppgtt->pdp.used_pdpes), +"PDPE %d is not allocated, but is reserved (%p)\n" , + pdpe, vm);continue ; + }else { +WARN (!test_bit (pdpe, ppgtt->pdp.used_pdpes), +"PDPE %d not reserved, but is allocated (%p)" , + pdpe, vm); + } -pci_unmap_page (hwdev, ppgtt->pdp.pagedir[i]->daddr,PAGE_SIZE , -PCI_DMA_BIDIRECTIONAL ); +gen8_for_each_pde (pt, pd, pd_start, pd_len, temp, pde) { +if (!pt) { +WARN (test_bit (pde, pd->used_pdes), +"PDE %d is not allocated, but is reserved (%p)\n" , + pde, vm); +continue ; + }else +WARN (!test_bit (pde, pd->used_pdes), +"PDE %d not reserved, but is allocated (%p)" , + pde, vm); + +bitmap_clear (pt->used_ptes, +gen8_pte_index (pd_start), +gen8_pte_count (pd_start, pd_len)); + +if (bitmap_empty (pt->used_ptes,GEN8_PTES_PER_PAGE )) { +free_pt_single (pt, vm->dev); + pd->page_tables[pde] =NULL ; +WARN_ON (!test_and_clear_bit (pde, pd->used_pdes)); + } + } -for (j =0 ; j <GEN8_PDES_PER_PAGE ; j++) { -struct i915_pagedir *pd = ppgtt->pdp.pagedir[i]; -struct i915_pagetab *pt = pd->page_tables[j]; -dma_addr_t addr = pt->daddr; -if (addr) -pci_unmap_page (hwdev, addr,PAGE_SIZE , -PCI_DMA_BIDIRECTIONAL ); +if (bitmap_empty (pd->used_pdes,GEN8_PDES_PER_PAGE )) { +free_pd_single (pd, vm->dev); + ppgtt->pdp.pagedir[pdpe] =NULL ; +WARN_ON (!test_and_clear_bit (pdpe, ppgtt->pdp.used_pdpes)); } } }
@@ -677,7 +701,6 @@ static void gen8_ppgtt_cleanup(struct i915_address_space *vm)struct i915_hw_ppgtt *ppgtt = container_of(vm, struct i915_hw_ppgtt, base); -gen8_ppgtt_unmap_pages (ppgtt);gen8_ppgtt_free (ppgtt); }
@@ -706,6 +729,7 @@ unwind_out:return -ENOMEM ; } +/* bitmap of new pagedirs */ static int gen8_ppgtt_alloc_pagedirs(struct i915_pagedirpo *pdp, uint64_t start, uint64_t length,
@@ -721,6 +745,7 @@ static int gen8_ppgtt_alloc_pagedirs(struct i915_pagedirpo *pdp,gen8_for_each_pdpe (unused, pdp, start, length, temp, pdpe) {BUG_ON (unused); pdp->pagedir[pdpe] =alloc_pd_single (dev); +if (IS_ERR (pdp->pagedir[pdpe])) goto unwind_out; }
@@ -742,10 +767,12 @@ static int gen8_alloc_va_range(struct i915_address_space *vm, container_of(vm, struct i915_hw_ppgtt, base);struct i915_pagedir *pd;const uint64_t orig_start = start; +const uint64_t orig_length = length;uint64_t temp;uint32_t pdpe;int ret; +/* Do the allocations first so we can easily bail out */ ret =gen8_ppgtt_alloc_pagedirs (&ppgtt->pdp, start, length, ppgtt->base.dev);if (ret)
@@ -758,6 +785,26 @@ static int gen8_alloc_va_range(struct i915_address_space *vm, goto err_out; } +/* Now mark everything we've touched as used. This doesn't allow for +* robust error checking, but it makes the code a hell of a lot simpler. +*/ + start = orig_start; + length = orig_length; + +gen8_for_each_pdpe (pd, &ppgtt->pdp, start, length, temp, pdpe) { +struct i915_pagetab *pt; +uint64_t pd_len =gen8_clamp_pd (start, length); +uint64_t pd_start = start; +uint32_t pde; +gen8_for_each_pde (pt, &ppgtt->pd, pd_start, pd_len, temp, pde) { +bitmap_set (pd->page_tables[pde]->used_ptes, +gen8_pte_index (start), +gen8_pte_count (start, length)); +set_bit (pde, pd->used_pdes); + } +set_bit (pdpe, ppgtt->pdp.used_pdpes); + } +return 0 ; err_out:
drivers/gpu/drm/i915/i915_gem_gtt.h +24 -0
@@ -206,11 +206,13 @@ struct i915_pagedir {dma_addr_t daddr; }; +unsigned long *used_pdes;struct i915_pagetab *page_tables[GEN6_PPGTT_PD_ENTRIES ]; };struct i915_pagedirpo {/* struct page *page; */ + DECLARE_BITMAP(used_pdpes, GEN8_LEGACY_PDPES);struct i915_pagedir *pagedir[GEN8_LEGACY_PDPES ]; };
@@ -449,6 +451,28 @@ static inline uint32_t gen8_pml4e_index(uint64_t address)BUG ();/* For 64B */ } +static inline size_t gen8_pte_count (uint64_t addr,uint64_t length) +{ +return i915_pte_count (addr, length,GEN8_PDE_SHIFT ); +} + +static inline size_t gen8_pde_count (uint64_t addr,uint64_t length) +{ +const uint32_t pdp_shift =GEN8_PDE_SHIFT +9 ; +const uint64_t mask = ~((1 << pdp_shift) -1 ); +uint64_t end; + +BUG_ON (length ==0 ); +BUG_ON (offset_in_page (addr|length)); + + end = addr + length; + +if ((addr & mask) != (end & mask)) +return GEN8_PDES_PER_PAGE -i915_pde_index (addr,GEN8_PDE_SHIFT ); + +return i915_pde_index (end,GEN8_PDE_SHIFT ) -i915_pde_index (addr,GEN8_PDE_SHIFT ); +} +int i915_gem_gtt_init (struct drm_device *dev);void i915_gem_init_global_gtt (struct drm_device *dev);void i915_global_gtt_cleanup (struct drm_device *dev);
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH v2 23/24 drm/i915/bdw: Dynamic page table allocations
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
This finishes off the dynamic page tables allocations, in the legacy 3 level style that already exists. Most everything has already been setup to this point, the patch finishes off the enabling by setting the appropriate function pointers.
Zombie tracking:
This could be a separate patch, but I found it helpful for debugging. Since we write page tables asynchronously with respect to the GPU using them, we can't actually free the page tables until we know the GPU won't use them. With this patch, that is always when the context dies. It would be possible to write a reaper to go through zombies and clean them up when under memory pressure. That exercise is left for the reader.
Scratch unused pages:
The object pages can get freed even if a page table still points to them. Like the zombie fix, we need to make sure we don't let our GPU access arbitrary memory when we've unmapped things.
v2: Update aliasing/true ppgtt allocate/teardown/clear functions for gen 6 & 7.
v3: Rebase.
--- drivers/gpu/drm/i915/i915_gem_gtt.c | 377 +++++++++++++++++++++++++++++------- drivers/gpu/drm/i915/i915_gem_gtt.h | 16 +- 2 files changed, 326 insertions(+), 67 deletions(-)
drivers/gpu/drm/i915/i915_gem_gtt.c +311 -66
@@ -602,7 +602,7 @@ static void gen8_ppgtt_insert_entries(struct i915_address_space *vm, } } -static void __gen8_do_map_pt (gen8_ppgtt_pde_t *pde, +static void __gen8_do_map_pt (gen8_ppgtt_pde_t *const pde,struct i915_pagetab *pt,struct drm_device *dev) {
@@ -619,7 +619,7 @@ static void gen8_map_pagetable_range(struct i915_pagedir *pd,uint64_t length, struct drm_device *dev) { - gen8_ppgtt_pde_t *pagedir = kmap_atomic(pd->page); + gen8_ppgtt_pde_t * const pagedir = kmap_atomic(pd->page);struct i915_pagetab *pt;uint64_t temp, pde;
@@ -632,8 +632,9 @@ static void gen8_map_pagetable_range(struct i915_pagedir *pd,kunmap_atomic (pagedir); } -static void gen8_teardown_va_range (struct i915_address_space *vm, -uint64_t start,uint64_t length) +static void __gen8_teardown_va_range (struct i915_address_space *vm, +uint64_t start,uint64_t length, +bool dead) {struct i915_hw_ppgtt *ppgtt = container_of(vm, struct i915_hw_ppgtt, base);
@@ -655,6 +656,13 @@ static void gen8_teardown_va_range(struct i915_address_space *vm, pdpe, vm);continue ; }else { +if (dead && pd->zombie) { +WARN_ON (test_bit (pdpe, ppgtt->pdp.used_pdpes)); +free_pd_single (pd, vm->dev); + ppgtt->pdp.pagedir[pdpe] =NULL ; +continue ; + } +WARN (!test_bit (pdpe, ppgtt->pdp.used_pdpes),"PDPE %d not reserved, but is allocated (%p)" , pdpe, vm);
@@ -666,34 +674,64 @@ static void gen8_teardown_va_range(struct i915_address_space *vm,"PDE %d is not allocated, but is reserved (%p)\n" , pde, vm);continue ; - }else + }else { +if (dead && pt->zombie) { +WARN_ON (test_bit (pde, pd->used_pdes)); +free_pt_single (pt, vm->dev); + pd->page_tables[pde] =NULL ; +continue ; + }WARN (!test_bit (pde, pd->used_pdes),"PDE %d not reserved, but is allocated (%p)" , pde, vm); + }bitmap_clear (pt->used_ptes,gen8_pte_index (pd_start),gen8_pte_count (pd_start, pd_len));if (bitmap_empty (pt->used_ptes,GEN8_PTES_PER_PAGE )) { +WARN_ON (!test_and_clear_bit (pde, pd->used_pdes)); +if (!dead) { + pt->zombie =1 ; +continue ; + }free_pt_single (pt, vm->dev); pd->page_tables[pde] =NULL ; -WARN_ON (!test_and_clear_bit (pde, pd->used_pdes)); + } } +gen8_ppgtt_clear_range (vm, pd_start, pd_len, true); +if (bitmap_empty (pd->used_pdes,GEN8_PDES_PER_PAGE )) { +WARN_ON (!test_and_clear_bit (pdpe, ppgtt->pdp.used_pdpes)); +if (!dead) { +/* We've unmapped a possibly live context. Make +* note of it so we can clean it up later. */ + pd->zombie =1 ; +continue ; + }free_pd_single (pd, vm->dev); ppgtt->pdp.pagedir[pdpe] =NULL ; -WARN_ON (!test_and_clear_bit (pdpe, ppgtt->pdp.used_pdpes)); } } } +static void gen8_teardown_va_range (struct i915_address_space *vm, +uint64_t start,uint64_t length) +{ +__gen8_teardown_va_range (vm, start, length, false); +} +static void gen8_ppgtt_free (struct i915_hw_ppgtt *ppgtt) { -gen8_teardown_va_range (&ppgtt->base, - ppgtt->base.start, ppgtt->base.total); +trace_i915_va_teardown (&ppgtt->base, + ppgtt->base.start, ppgtt->base.total, +VM_TO_TRACE_NAME (&ppgtt->base)); +__gen8_teardown_va_range (&ppgtt->base, + ppgtt->base.start, ppgtt->base.total, + true); }static void gen8_ppgtt_cleanup (struct i915_address_space *vm)
@@ -704,67 +742,177 @@ static void gen8_ppgtt_cleanup(struct i915_address_space *vm)gen8_ppgtt_free (ppgtt); } -static int gen8_ppgtt_alloc_pagetabs (struct i915_pagedir *pd, +/** +* gen8_ppgtt_alloc_pagetabs() - Allocate page tables for VA range. +* @ppgtt: Master ppgtt structure. +* @pd: Page directory for this address range. +* @start: Starting virtual address to begin allocations. +* @length Size of the allocations. +* @new_pts: Bitmap set by function with new allocations. Likely used by the +* caller to free on error. +* +* Allocate the required number of page tables. Extremely similar to +* gen8_ppgtt_alloc_pagedirs(). The main difference is here we are limited by +* the page directory boundary (instead of the page directory pointer). That +* boundary is 1GB virtual. Therefore, unlike gen8_ppgtt_alloc_pagedirs(), it is +* possible, and likely that the caller will need to use multiple calls of this +* function to achieve the appropriate allocation. +* +* Return: 0 if success; negative error code otherwise. +*/ +static int gen8_ppgtt_alloc_pagetabs (struct i915_hw_ppgtt *ppgtt, +struct i915_pagedir *pd,uint64_t start,uint64_t length, -struct drm_device *dev) +unsigned long *new_pts) { -struct i915_pagetab *unused; +struct i915_pagetab *pt;uint64_t temp;uint32_t pde; -gen8_for_each_pde (unused, pd, start, length, temp, pde) { -BUG_ON (unused); - pd->page_tables[pde] =alloc_pt_single (dev); -if (IS_ERR (pd->page_tables[pde])) +gen8_for_each_pde (pt, pd, start, length, temp, pde) { +/* Don't reallocate page tables */ +if (pt) { +/* Scratch is never allocated this way */ +WARN_ON (pt->scratch); +/* If there is a zombie, we can reuse it and save time +* on the allocation. If we clear the zombie status and +* the caller somehow fails, we'll probably hit some +* assertions, so it's up to them to fix up the bitmaps. +*/ +continue ; + } + + pt =alloc_pt_single (ppgtt->base.dev); +if (IS_ERR (pt)) goto unwind_out; + + pd->page_tables[pde] = pt; +set_bit (pde, new_pts); }return 0 ; unwind_out: -while (pde--) -free_pt_single (pd->page_tables[pde], dev); +for_each_set_bit (pde, new_pts,GEN8_PDES_PER_PAGE ) +free_pt_single (pd->page_tables[pde], ppgtt->base.dev);return -ENOMEM ; } -/* bitmap of new pagedirs */ -static int gen8_ppgtt_alloc_pagedirs (struct i915_pagedirpo *pdp, +/** +* gen8_ppgtt_alloc_pagedirs() - Allocate page directories for VA range. +* @ppgtt: Master ppgtt structure. +* @pdp: Page directory pointer for this address range. +* @start: Starting virtual address to begin allocations. +* @length Size of the allocations. +* @new_pds Bitmap set by function with new allocations. Likely used by the +* caller to free on error. +* +* Allocate the required number of page directories starting at the pde index of +* @start, and ending at the pde index @start + @length. This function will skip +* over already allocated page directories within the range, and only allocate +* new ones, setting the appropriate pointer within the pdp as well as the +* correct position in the bitmap @new_pds. +* +* The function will only allocate the pages within the range for a give page +* directory pointer. In other words, if @start + @length straddles a virtually +* addressed PDP boundary (512GB for 4k pages), there will be more allocations +* required by the caller, This is not currently possible, and the BUG in the +* code will prevent it. +* +* Return: 0 if success; negative error code otherwise. +*/ +static int gen8_ppgtt_alloc_pagedirs (struct i915_hw_ppgtt *ppgtt, +struct i915_pagedirpo *pdp,uint64_t start,uint64_t length, -struct drm_device *dev) +unsigned long *new_pds) { -struct i915_pagedir *unused; +struct i915_pagedir *pd;uint64_t temp;uint32_t pdpe; +BUG_ON (!bitmap_empty (new_pds,GEN8_LEGACY_PDPES )); +/* FIXME: PPGTT container_of won't work for 64b */ BUG_ON ((start + length) >0x800000000ULL ); -gen8_for_each_pdpe (unused, pdp, start, length, temp, pdpe) { -BUG_ON (unused); - pdp->pagedir[pdpe] =alloc_pd_single (dev); +gen8_for_each_pdpe (pd, pdp, start, length, temp, pdpe) { +if (pd) +continue ; -if (IS_ERR (pdp->pagedir[pdpe])) + pd =alloc_pd_single (ppgtt->base.dev); +if (IS_ERR (pd)) goto unwind_out; + + pdp->pagedir[pdpe] = pd; +set_bit (pdpe, new_pds); }return 0 ; unwind_out: -while (pdpe--) -free_pd_single (pdp->pagedir[pdpe], dev); +for_each_set_bit (pdpe, new_pds,GEN8_LEGACY_PDPES ) +free_pd_single (pdp->pagedir[pdpe], ppgtt->base.dev);return -ENOMEM ; } +static inline void +free_gen8_temp_bitmaps (unsigned long *new_pds,unsigned long **new_pts) +{ +int i; +for (i =0 ; i <GEN8_LEGACY_PDPES ; i++) +kfree (new_pts[i]); +kfree (new_pts); +kfree (new_pds); +} + +/* Fills in the page directory bitmap, ant the array of page tables bitmap. Both +* of these are based on the number of PDPEs in the system. +*/ +int __must_check alloc_gen8_temp_bitmaps (unsigned long **new_pds, +unsigned long ***new_pts) +{ +int i; +unsigned long *pds; +unsigned long **pts; + + pds =kcalloc (BITS_TO_LONGS (GEN8_LEGACY_PDPES ),sizeof (unsigned long ),GFP_KERNEL ); +if (!pds) +return -ENOMEM ; + + pts =kcalloc (GEN8_PDES_PER_PAGE ,sizeof (unsigned long *),GFP_KERNEL ); +if (!pts) { +kfree (pds); +return -ENOMEM ; + } + +for (i =0 ; i <GEN8_LEGACY_PDPES ; i++) { + pts[i] =kcalloc (BITS_TO_LONGS (GEN8_PDES_PER_PAGE ), +sizeof (unsigned long ),GFP_KERNEL ); +if (!pts[i]) + goto err_out; + } + + *new_pds = pds; + *new_pts = (unsigned long **)pts; + +return 0 ; + +err_out: +free_gen8_temp_bitmaps (pds, pts); +return -ENOMEM ; +} +static int gen8_alloc_va_range (struct i915_address_space *vm,uint64_t start,uint64_t length) {struct i915_hw_ppgtt *ppgtt = container_of(vm, struct i915_hw_ppgtt, base); +unsigned long *new_page_dirs, **new_page_tables;struct i915_pagedir *pd;const uint64_t orig_start = start;const uint64_t orig_length = length;
@@ -772,43 +920,103 @@ static int gen8_alloc_va_range(struct i915_address_space *vm,uint32_t pdpe;int ret; -/* Do the allocations first so we can easily bail out */ - ret =gen8_ppgtt_alloc_pagedirs (&ppgtt->pdp, start, length, - ppgtt->base.dev); +#ifndef CONFIG_64BIT +/* Disallow 64b address on 32b platforms. Nothing is wrong with doing +* this in hardware, but a lot of the drm code is not prepared to handle +* 64b offset on 32b platforms. */ +if (start + length >0x100000000ULL ) +return -E2BIG ; +#endif + +/* Wrap is never okay since we can only represent 48b, and we don't +* actually use the other side of the canonical address space. +*/ +if (WARN_ON (start + length < start)) +return -ERANGE ; + + ret =alloc_gen8_temp_bitmaps (&new_page_dirs, &new_page_tables);if (ret)return ret; +/* Do the allocations first so we can easily bail out */ + ret =gen8_ppgtt_alloc_pagedirs (ppgtt, &ppgtt->pdp, start, length, + new_page_dirs); +if (ret) { +free_gen8_temp_bitmaps (new_page_dirs, new_page_tables); +return ret; + } + +/* For every page directory referenced, allocate page tables */ gen8_for_each_pdpe (pd, &ppgtt->pdp, start, length, temp, pdpe) { - ret =gen8_ppgtt_alloc_pagetabs (pd, start, length, - ppgtt->base.dev); +bitmap_zero (new_page_tables[pdpe],GEN8_PDES_PER_PAGE ); + ret =gen8_ppgtt_alloc_pagetabs (ppgtt, pd, start, length, + new_page_tables[pdpe]);if (ret) goto err_out; } -/* Now mark everything we've touched as used. This doesn't allow for -* robust error checking, but it makes the code a hell of a lot simpler. -*/ start = orig_start; length = orig_length; +/* Allocations have completed successfully, so set the bitmaps, and do +* the mappings. */ gen8_for_each_pdpe (pd, &ppgtt->pdp, start, length, temp, pdpe) { +gen8_ppgtt_pde_t *const pagedir =kmap_atomic (pd->page);struct i915_pagetab *pt;uint64_t pd_len =gen8_clamp_pd (start, length);uint64_t pd_start = start;uint32_t pde; -gen8_for_each_pde (pt, &ppgtt->pd, pd_start, pd_len, temp, pde) { -bitmap_set (pd->page_tables[pde]->used_ptes, -gen8_pte_index (start), -gen8_pte_count (start, length)); + +/* Every pd should be allocated, we just did that above. */ +BUG_ON (!pd); + +gen8_for_each_pde (pt, pd, pd_start, pd_len, temp, pde) { +/* Same reasoning as pd */ +BUG_ON (!pt); +BUG_ON (!pd_len); +BUG_ON (!gen8_pte_count (pd_start, pd_len)); + +/* Set our used ptes within the page table */ +bitmap_set (pt->used_ptes, +gen8_pte_index (pd_start), +gen8_pte_count (pd_start, pd_len)); + +/* Our pde is now pointing to the pagetable, pt */ set_bit (pde, pd->used_pdes); + +/* Map the PDE to the page table */ +__gen8_do_map_pt (pagedir + pde, pt, vm->dev); + +/* NB: We haven't yet mapped ptes to pages. At this +* point we're still relying on insert_entries() */ + +/* No longer possible this page table is a zombie */ + pt->zombie =0 ; } + +if (!HAS_LLC (vm->dev)) +drm_clflush_virt_range (pagedir,PAGE_SIZE ); + +kunmap_atomic (pagedir); +set_bit (pdpe, ppgtt->pdp.used_pdpes); +/* This pd is officially not a zombie either */ + ppgtt->pdp.pagedir[pdpe]->zombie =0 ; } +free_gen8_temp_bitmaps (new_page_dirs, new_page_tables);return 0 ; err_out: -gen8_teardown_va_range (vm, orig_start, start); +while (pdpe--) { +for_each_set_bit (temp, new_page_tables[pdpe],GEN8_PDES_PER_PAGE ) +free_pt_single (pd->page_tables[temp], vm->dev); + } + +for_each_set_bit (pdpe, new_page_dirs,GEN8_LEGACY_PDPES ) +free_pd_single (ppgtt->pdp.pagedir[pdpe], vm->dev); + +free_gen8_temp_bitmaps (new_page_dirs, new_page_tables);return ret; }
@@ -819,37 +1027,68 @@ err_out:* space. * */ -static int gen8_ppgtt_init (struct i915_hw_ppgtt *ppgtt,uint64_t size) +static int gen8_ppgtt_init_common (struct i915_hw_ppgtt *ppgtt,uint64_t size) { -struct i915_pagedir *pd; -uint64_t temp, start =0 ; -const uint64_t orig_length = size; -uint32_t pdpe; -int ret; + ppgtt->scratch_pd =alloc_pt_scratch (ppgtt->base.dev); +if (IS_ERR (ppgtt->scratch_pd)) +return PTR_ERR (ppgtt->scratch_pd); ppgtt->base.start =0 ; ppgtt->base.total = size; - ppgtt->base.clear_range = gen8_ppgtt_clear_range; - ppgtt->base.insert_entries = gen8_ppgtt_insert_entries; ppgtt->base.cleanup = gen8_ppgtt_cleanup; + ppgtt->base.insert_entries = gen8_ppgtt_insert_entries; + ppgtt->switch_mm = gen8_mm_switch; - ppgtt->scratch_pd =alloc_pt_scratch (ppgtt->base.dev); -if (IS_ERR (ppgtt->scratch_pd)) -return PTR_ERR (ppgtt->scratch_pd); +return 0 ; +} + +static int gen8_aliasing_ppgtt_init (struct i915_hw_ppgtt *ppgtt) +{ +struct drm_device *dev = ppgtt->base.dev; +struct drm_i915_private *dev_priv = dev->dev_private; +struct i915_pagedir *pd; +uint64_t temp, start =0 , size = dev_priv->gtt.base.total; +uint32_t pdpe; +int ret; + ret =gen8_ppgtt_init_common (ppgtt, dev_priv->gtt.base.total); +if (ret) +return ret; + +/* Aliasing PPGTT has to always work and be mapped because of the way we +* use RESTORE_INHIBIT in the context switch. This will be fixed +* eventually. */ ret =gen8_alloc_va_range (&ppgtt->base, start, size);if (ret) {free_pt_scratch (ppgtt->scratch_pd, ppgtt->base.dev);return ret; } - start =0 ; - size = orig_length; -gen8_for_each_pdpe (pd, &ppgtt->pdp, start, size, temp, pdpe)gen8_map_pagetable_range (pd, start, size, ppgtt->base.dev); + ppgtt->base.allocate_va_range =NULL ; + ppgtt->base.teardown_va_range =NULL ; + ppgtt->base.clear_range = gen8_ppgtt_clear_range; + +return 0 ; +} + +static int gen8_ppgtt_init (struct i915_hw_ppgtt *ppgtt) +{ +struct drm_device *dev = ppgtt->base.dev; +struct drm_i915_private *dev_priv = dev->dev_private; +int ret; + + ret =gen8_ppgtt_init_common (ppgtt, dev_priv->gtt.base.total); +if (ret) +return ret; + + ppgtt->base.allocate_va_range = gen8_alloc_va_range; + ppgtt->base.teardown_va_range = gen8_teardown_va_range; + ppgtt->base.clear_range =NULL ; +return 0 ; }
@@ -1413,9 +1652,9 @@ static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt, bool aliasing)if (ret)return ret; - ppgtt->base.allocate_va_range = gen6_alloc_va_range; - ppgtt->base.teardown_va_range = gen6_teardown_va_range; - ppgtt->base.clear_range = gen6_ppgtt_clear_range; + ppgtt->base.allocate_va_range = aliasing ?NULL : gen6_alloc_va_range; + ppgtt->base.teardown_va_range = aliasing ?NULL : gen6_teardown_va_range; + ppgtt->base.clear_range = aliasing ? gen6_ppgtt_clear_range :NULL ; ppgtt->base.insert_entries = gen6_ppgtt_insert_entries; ppgtt->base.cleanup = gen6_ppgtt_cleanup; ppgtt->base.start =0 ;
@@ -1453,8 +1692,10 @@ static int __hw_ppgtt_init(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt,if (INTEL_INFO (dev)->gen <8 )return gen6_ppgtt_init (ppgtt, aliasing); +else if (aliasing) +return gen8_aliasing_ppgtt_init (ppgtt);else -return gen8_ppgtt_init (ppgtt, dev_priv->gtt.base.total); +return gen8_ppgtt_init (ppgtt); }int i915_ppgtt_init (struct drm_device *dev,struct i915_hw_ppgtt *ppgtt) {
@@ -1466,8 +1707,9 @@ int i915_ppgtt_init(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt)kref_init (&ppgtt->ref);drm_mm_init (&ppgtt->base.mm, ppgtt->base.start, ppgtt->base.total); - ppgtt->base.clear_range (&ppgtt->base,0 , - ppgtt->base.total, true); +if (ppgtt->base.clear_range) + ppgtt->base.clear_range (&ppgtt->base,0 , + ppgtt->base.total, true);i915_init_vm (dev_priv, &ppgtt->base); }
@@ -1565,10 +1807,7 @@ ppgtt_bind_vma(struct i915_vma *vma,static void ppgtt_unbind_vma (struct i915_vma *vma) { - vma->vm->clear_range (vma->vm, - vma->node.start, - vma->obj->base.size, - true); +WARN_ON (vma->vm->teardown_va_range && vma->vm->clear_range);if (vma->vm->teardown_va_range) { trace_i915_va_teardown(vma->vm, vma->node.start, vma->node.size,
@@ -1576,7 +1815,13 @@ static void ppgtt_unbind_vma(struct i915_vma *vma) vma->vm->teardown_va_range (vma->vm, vma->node.start, vma->node.size); - } + }else if (vma->vm->clear_range) { + vma->vm->clear_range (vma->vm, + vma->node.start, + vma->obj->base.size, + true); + }else +BUG (); }extern int intel_iommu_gfx_mapped;
drivers/gpu/drm/i915/i915_gem_gtt.h +15 -1
@@ -190,13 +190,26 @@ struct i915_vma { u32 flags); }; - +/* Zombies. We write page tables with the CPU, and hardware switches them with +* the GPU. As such, the only time we can safely remove a page table is when we +* know the context is idle. Since we have no good way to do this, we use the +* zombie. +* +* Under memory pressure, if the system is idle, zombies may be reaped. +* +* There are 3 states a page table can be in (not including scratch) +* bitmap = 0, zombie = 0: unallocated +* bitmap = 1, zombie = 0: allocated +* bitmap = 0, zombie = 1: zombie +* bitmap = 1, zombie = 1: invalid +*/ struct i915_pagetab {struct page *page;dma_addr_t daddr;unsigned long *used_ptes;unsigned int scratch:1 ; +unsigned zombie:1 ; };struct i915_pagedir {
@@ -208,6 +221,7 @@ struct i915_pagedir {unsigned long *used_pdes;struct i915_pagetab *page_tables[GEN6_PPGTT_PD_ENTRIES ]; + unsigned zombie:1; };struct i915_pagedirpo {
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
Re: [PATCH v2 23/24] drm/i915/bdw: Dynamic page table allocations
Quoted 7 lines · Michel Thierry
From: Ben Widawsky <[email protected]>
This finishes off the dynamic page tables allocations, in the legacy 3 level style that already exists. Most everything has already been setup to this point, the patch finishes off the enabling by setting the appropriate function pointers.
Quoted 7 lines
Zombie tracking:
This could be a separate patch, but I found it helpful for debugging. Since we write page tables asynchronously with respect to the GPU using them, we can't actually free the page tables until we know the GPU won't use them. With this patch, that is always when the context dies. It would be possible to write a reaper to go through zombies and clean them up when under memory pressure. That exercise is left for the reader.
As mention in some previous reply, freeing pagetables is a separate issue entirely. Imo we can just idle the gpu and then rip out all pagetables for empty vms.
Quoted 4 lines
Scratch unused pages:
The object pages can get freed even if a page table still points to them. Like the zombie fix, we need to make sure we don't let our GPU access arbitrary memory when we've unmapped things.
Hm, either I don't follow or this would mean that our active tracking is broken and we release backing storage before the gpu stopped using it.
In any case I vote to simplify things a lot for now and just never teardown pagetables at all. Implementing a last-ditch attempt to free memory can be done with a lot less complexity imo than trying to be super careful without stalling the gpu in normal operations.
One more comment below.
-Daniel
Quoted 579 lines
v2: Update aliasing/true ppgtt allocate/teardown/clear functions for gen 6 & 7.
v3: Rebase.
Signed-off-by Ben Widawsky <[email protected]>Signed-off-by Michel Thierry <[email protected]> (v2+)--- drivers/gpu/drm/i915/i915_gem_gtt.c | 377 +++++++++++++++++++++++++++++------- drivers/gpu/drm/i915/i915_gem_gtt.h | 16 +- 2 files changed, 326 insertions(+), 67 deletions(-)drivers/gpu/drm/i915/i915_gem_gtt.c +304
-65@@ -602,7 +602,7 @@ static void gen8_ppgtt_insert_entries(struct i915_address_space *vm, } } -static void __gen8_do_map_pt (gen8_ppgtt_pde_t *pde, +static void __gen8_do_map_pt (gen8_ppgtt_pde_t *const pde,struct i915_pagetab *pt,struct drm_device *dev) {@@ -619,7 +619,7 @@ static void gen8_map_pagetable_range(struct i915_pagedir *pd,uint64_t length, struct drm_device *dev) { - gen8_ppgtt_pde_t *pagedir = kmap_atomic(pd->page); + gen8_ppgtt_pde_t * const pagedir = kmap_atomic(pd->page);struct i915_pagetab *pt;uint64_t temp, pde;@@ -632,8 +632,9 @@ static void gen8_map_pagetable_range(struct i915_pagedir *pd,kunmap_atomic (pagedir); } -static void gen8_teardown_va_range (struct i915_address_space *vm, -uint64_t start,uint64_t length) +static void __gen8_teardown_va_range (struct i915_address_space *vm, +uint64_t start,uint64_t length, +bool dead) {struct i915_hw_ppgtt *ppgtt = container_of(vm, struct i915_hw_ppgtt, base);@@ -655,6 +656,13 @@ static void gen8_teardown_va_range(struct i915_address_space *vm, pdpe, vm);continue ; }else { +if (dead && pd->zombie) { +WARN_ON (test_bit (pdpe, ppgtt->pdp.used_pdpes)); +free_pd_single (pd, vm->dev); + ppgtt->pdp.pagedir[pdpe] =NULL ; +continue ; + } +WARN (!test_bit (pdpe, ppgtt->pdp.used_pdpes),"PDPE %d not reserved, but is allocated (%p)" , pdpe, vm);@@ -666,34 +674,64 @@ static void gen8_teardown_va_range(struct i915_address_space *vm,"PDE %d is not allocated, but is reserved (%p)\n" , pde, vm);continue ; - }else + }else { +if (dead && pt->zombie) { +WARN_ON (test_bit (pde, pd->used_pdes)); +free_pt_single (pt, vm->dev); + pd->page_tables[pde] =NULL ; +continue ; + }WARN (!test_bit (pde, pd->used_pdes),"PDE %d not reserved, but is allocated (%p)" , pde, vm); + }bitmap_clear (pt->used_ptes,gen8_pte_index (pd_start),gen8_pte_count (pd_start, pd_len));if (bitmap_empty (pt->used_ptes,GEN8_PTES_PER_PAGE )) { +WARN_ON (!test_and_clear_bit (pde, pd->used_pdes)); +if (!dead) { + pt->zombie =1 ; +continue ; + }free_pt_single (pt, vm->dev); pd->page_tables[pde] =NULL ; -WARN_ON (!test_and_clear_bit (pde, pd->used_pdes)); + } } +gen8_ppgtt_clear_range (vm, pd_start, pd_len, true); +if (bitmap_empty (pd->used_pdes,GEN8_PDES_PER_PAGE )) { +WARN_ON (!test_and_clear_bit (pdpe, ppgtt->pdp.used_pdpes)); +if (!dead) { +/* We've unmapped a possibly live context. Make +* note of it so we can clean it up later. */ + pd->zombie =1 ; +continue ; + }free_pd_single (pd, vm->dev); ppgtt->pdp.pagedir[pdpe] =NULL ; -WARN_ON (!test_and_clear_bit (pdpe, ppgtt->pdp.used_pdpes)); } } } +static void gen8_teardown_va_range (struct i915_address_space *vm, +uint64_t start,uint64_t length) +{ +__gen8_teardown_va_range (vm, start, length, false); +} +static void gen8_ppgtt_free (struct i915_hw_ppgtt *ppgtt) { -gen8_teardown_va_range (&ppgtt->base, - ppgtt->base.start, ppgtt->base.total); +trace_i915_va_teardown (&ppgtt->base, + ppgtt->base.start, ppgtt->base.total, +VM_TO_TRACE_NAME (&ppgtt->base)); +__gen8_teardown_va_range (&ppgtt->base, + ppgtt->base.start, ppgtt->base.total, + true); }static void gen8_ppgtt_cleanup (struct i915_address_space *vm)@@ -704,67 +742,177 @@ static void gen8_ppgtt_cleanup(struct i915_address_space *vm)gen8_ppgtt_free (ppgtt); } -static int gen8_ppgtt_alloc_pagetabs (struct i915_pagedir *pd, +/** +* gen8_ppgtt_alloc_pagetabs() - Allocate page tables for VA range. +* @ppgtt: Master ppgtt structure. +* @pd: Page directory for this address range. +* @start: Starting virtual address to begin allocations. +* @length Size of the allocations. +* @new_pts: Bitmap set by function with new allocations. Likely used by the +* caller to free on error. +* +* Allocate the required number of page tables. Extremely similar to +* gen8_ppgtt_alloc_pagedirs(). The main difference is here we are limited by +* the page directory boundary (instead of the page directory pointer). That +* boundary is 1GB virtual. Therefore, unlike gen8_ppgtt_alloc_pagedirs(), it is +* possible, and likely that the caller will need to use multiple calls of this +* function to achieve the appropriate allocation. +* +* Return: 0 if success; negative error code otherwise. +*/ +static int gen8_ppgtt_alloc_pagetabs (struct i915_hw_ppgtt *ppgtt, +struct i915_pagedir *pd,uint64_t start,uint64_t length, -struct drm_device *dev) +unsigned long *new_pts) { -struct i915_pagetab *unused; +struct i915_pagetab *pt;uint64_t temp;uint32_t pde; -gen8_for_each_pde (unused, pd, start, length, temp, pde) { -BUG_ON (unused); - pd->page_tables[pde] =alloc_pt_single (dev); -if (IS_ERR (pd->page_tables[pde])) +gen8_for_each_pde (pt, pd, start, length, temp, pde) { +/* Don't reallocate page tables */ +if (pt) { +/* Scratch is never allocated this way */ +WARN_ON (pt->scratch); +/* If there is a zombie, we can reuse it and save time +* on the allocation. If we clear the zombie status and +* the caller somehow fails, we'll probably hit some +* assertions, so it's up to them to fix up the bitmaps. +*/ +continue ; + } + + pt =alloc_pt_single (ppgtt->base.dev); +if (IS_ERR (pt)) goto unwind_out; + + pd->page_tables[pde] = pt; +set_bit (pde, new_pts); }return 0 ; unwind_out: -while (pde--) -free_pt_single (pd->page_tables[pde], dev); +for_each_set_bit (pde, new_pts,GEN8_PDES_PER_PAGE ) +free_pt_single (pd->page_tables[pde], ppgtt->base.dev);return -ENOMEM ; } -/* bitmap of new pagedirs */ -static int gen8_ppgtt_alloc_pagedirs (struct i915_pagedirpo *pdp, +/** +* gen8_ppgtt_alloc_pagedirs() - Allocate page directories for VA range. +* @ppgtt: Master ppgtt structure. +* @pdp: Page directory pointer for this address range. +* @start: Starting virtual address to begin allocations. +* @length Size of the allocations. +* @new_pds Bitmap set by function with new allocations. Likely used by the +* caller to free on error. +* +* Allocate the required number of page directories starting at the pde index of +* @start, and ending at the pde index @start + @length. This function will skip +* over already allocated page directories within the range, and only allocate +* new ones, setting the appropriate pointer within the pdp as well as the +* correct position in the bitmap @new_pds. +* +* The function will only allocate the pages within the range for a give page +* directory pointer. In other words, if @start + @length straddles a virtually +* addressed PDP boundary (512GB for 4k pages), there will be more allocations +* required by the caller, This is not currently possible, and the BUG in the +* code will prevent it. +* +* Return: 0 if success; negative error code otherwise. +*/ +static int gen8_ppgtt_alloc_pagedirs (struct i915_hw_ppgtt *ppgtt, +struct i915_pagedirpo *pdp,uint64_t start,uint64_t length, -struct drm_device *dev) +unsigned long *new_pds) { -struct i915_pagedir *unused; +struct i915_pagedir *pd;uint64_t temp;uint32_t pdpe; +BUG_ON (!bitmap_empty (new_pds,GEN8_LEGACY_PDPES )); +/* FIXME: PPGTT container_of won't work for 64b */ BUG_ON ((start + length) >0x800000000ULL ); -gen8_for_each_pdpe (unused, pdp, start, length, temp, pdpe) { -BUG_ON (unused); - pdp->pagedir[pdpe] =alloc_pd_single (dev); +gen8_for_each_pdpe (pd, pdp, start, length, temp, pdpe) { +if (pd) +continue ; -if (IS_ERR (pdp->pagedir[pdpe])) + pd =alloc_pd_single (ppgtt->base.dev); +if (IS_ERR (pd)) goto unwind_out; + + pdp->pagedir[pdpe] = pd; +set_bit (pdpe, new_pds); }return 0 ; unwind_out: -while (pdpe--) -free_pd_single (pdp->pagedir[pdpe], dev); +for_each_set_bit (pdpe, new_pds,GEN8_LEGACY_PDPES ) +free_pd_single (pdp->pagedir[pdpe], ppgtt->base.dev);return -ENOMEM ; } +static inline void +free_gen8_temp_bitmaps (unsigned long *new_pds,unsigned long **new_pts) +{ +int i; +for (i =0 ; i <GEN8_LEGACY_PDPES ; i++) +kfree (new_pts[i]); +kfree (new_pts); +kfree (new_pds); +} + +/* Fills in the page directory bitmap, ant the array of page tables bitmap. Both +* of these are based on the number of PDPEs in the system. +*/ +int __must_check alloc_gen8_temp_bitmaps (unsigned long **new_pds, +unsigned long ***new_pts) +{ +int i; +unsigned long *pds; +unsigned long **pts; + + pds =kcalloc (BITS_TO_LONGS (GEN8_LEGACY_PDPES ),sizeof (unsigned long ),GFP_KERNEL ); +if (!pds) +return -ENOMEM ; + + pts =kcalloc (GEN8_PDES_PER_PAGE ,sizeof (unsigned long *),GFP_KERNEL ); +if (!pts) { +kfree (pds); +return -ENOMEM ; + } + +for (i =0 ; i <GEN8_LEGACY_PDPES ; i++) { + pts[i] =kcalloc (BITS_TO_LONGS (GEN8_PDES_PER_PAGE ), +sizeof (unsigned long ),GFP_KERNEL ); +if (!pts[i]) + goto err_out; + } + + *new_pds = pds; + *new_pts = (unsigned long **)pts; + +return 0 ; + +err_out: +free_gen8_temp_bitmaps (pds, pts); +return -ENOMEM ; +} +static int gen8_alloc_va_range (struct i915_address_space *vm,uint64_t start,uint64_t length) {struct i915_hw_ppgtt *ppgtt = container_of(vm, struct i915_hw_ppgtt, base); +unsigned long *new_page_dirs, **new_page_tables;struct i915_pagedir *pd;const uint64_t orig_start = start;const uint64_t orig_length = length;@@ -772,43 +920,103 @@ static int gen8_alloc_va_range(struct i915_address_space *vm,uint32_t pdpe;int ret; -/* Do the allocations first so we can easily bail out */ - ret =gen8_ppgtt_alloc_pagedirs (&ppgtt->pdp, start, length, - ppgtt->base.dev); +#ifndef CONFIG_64BIT +/* Disallow 64b address on 32b platforms. Nothing is wrong with doing +* this in hardware, but a lot of the drm code is not prepared to handle +* 64b offset on 32b platforms. */ +if (start + length >0x100000000ULL ) +return -E2BIG ; +#endif + +/* Wrap is never okay since we can only represent 48b, and we don't +* actually use the other side of the canonical address space. +*/ +if (WARN_ON (start + length < start)) +return -ERANGE ; + + ret =alloc_gen8_temp_bitmaps (&new_page_dirs, &new_page_tables);if (ret)return ret; +/* Do the allocations first so we can easily bail out */ + ret =gen8_ppgtt_alloc_pagedirs (ppgtt, &ppgtt->pdp, start, length, + new_page_dirs); +if (ret) { +free_gen8_temp_bitmaps (new_page_dirs, new_page_tables); +return ret; + } + +/* For every page directory referenced, allocate page tables */ gen8_for_each_pdpe (pd, &ppgtt->pdp, start, length, temp, pdpe) { - ret =gen8_ppgtt_alloc_pagetabs (pd, start, length, - ppgtt->base.dev); +bitmap_zero (new_page_tables[pdpe],GEN8_PDES_PER_PAGE ); + ret =gen8_ppgtt_alloc_pagetabs (ppgtt, pd, start, length, + new_page_tables[pdpe]);if (ret) goto err_out; } -/* Now mark everything we've touched as used. This doesn't allow for -* robust error checking, but it makes the code a hell of a lot simpler. -*/ start = orig_start; length = orig_length; +/* Allocations have completed successfully, so set the bitmaps, and do +* the mappings. */ gen8_for_each_pdpe (pd, &ppgtt->pdp, start, length, temp, pdpe) { +gen8_ppgtt_pde_t *const pagedir =kmap_atomic (pd->page);struct i915_pagetab *pt;uint64_t pd_len =gen8_clamp_pd (start, length);uint64_t pd_start = start;uint32_t pde; -gen8_for_each_pde (pt, &ppgtt->pd, pd_start, pd_len, temp, pde) { -bitmap_set (pd->page_tables[pde]->used_ptes, -gen8_pte_index (start), -gen8_pte_count (start, length)); + +/* Every pd should be allocated, we just did that above. */ +BUG_ON (!pd); + +gen8_for_each_pde (pt, pd, pd_start, pd_len, temp, pde) { +/* Same reasoning as pd */ +BUG_ON (!pt); +BUG_ON (!pd_len); +BUG_ON (!gen8_pte_count (pd_start, pd_len)); + +/* Set our used ptes within the page table */ +bitmap_set (pt->used_ptes, +gen8_pte_index (pd_start), +gen8_pte_count (pd_start, pd_len)); + +/* Our pde is now pointing to the pagetable, pt */ set_bit (pde, pd->used_pdes); + +/* Map the PDE to the page table */ +__gen8_do_map_pt (pagedir + pde, pt, vm->dev); + +/* NB: We haven't yet mapped ptes to pages. At this +* point we're still relying on insert_entries() */ + +/* No longer possible this page table is a zombie */ + pt->zombie =0 ; } + +if (!HAS_LLC (vm->dev)) +drm_clflush_virt_range (pagedir,PAGE_SIZE ); + +kunmap_atomic (pagedir); +set_bit (pdpe, ppgtt->pdp.used_pdpes); +/* This pd is officially not a zombie either */ + ppgtt->pdp.pagedir[pdpe]->zombie =0 ; } +free_gen8_temp_bitmaps (new_page_dirs, new_page_tables);return 0 ; err_out: -gen8_teardown_va_range (vm, orig_start, start); +while (pdpe--) { +for_each_set_bit (temp, new_page_tables[pdpe],GEN8_PDES_PER_PAGE ) +free_pt_single (pd->page_tables[temp], vm->dev); + } + +for_each_set_bit (pdpe, new_page_dirs,GEN8_LEGACY_PDPES ) +free_pd_single (ppgtt->pdp.pagedir[pdpe], vm->dev); + +free_gen8_temp_bitmaps (new_page_dirs, new_page_tables);return ret; }@@ -819,37 +1027,68 @@ err_out:* space. * */ -static int gen8_ppgtt_init (struct i915_hw_ppgtt *ppgtt,uint64_t size) +static int gen8_ppgtt_init_common (struct i915_hw_ppgtt *ppgtt,uint64_t size) { -struct i915_pagedir *pd; -uint64_t temp, start =0 ; -const uint64_t orig_length = size; -uint32_t pdpe; -int ret; + ppgtt->scratch_pd =alloc_pt_scratch (ppgtt->base.dev); +if (IS_ERR (ppgtt->scratch_pd)) +return PTR_ERR (ppgtt->scratch_pd); ppgtt->base.start =0 ; ppgtt->base.total = size; - ppgtt->base.clear_range = gen8_ppgtt_clear_range; - ppgtt->base.insert_entries = gen8_ppgtt_insert_entries; ppgtt->base.cleanup = gen8_ppgtt_cleanup; + ppgtt->base.insert_entries = gen8_ppgtt_insert_entries; + ppgtt->switch_mm = gen8_mm_switch; - ppgtt->scratch_pd =alloc_pt_scratch (ppgtt->base.dev); -if (IS_ERR (ppgtt->scratch_pd)) -return PTR_ERR (ppgtt->scratch_pd); +return 0 ; +} + +static int gen8_aliasing_ppgtt_init (struct i915_hw_ppgtt *ppgtt) +{ +struct drm_device *dev = ppgtt->base.dev; +struct drm_i915_private *dev_priv = dev->dev_private; +struct i915_pagedir *pd; +uint64_t temp, start =0 , size = dev_priv->gtt.base.total; +uint32_t pdpe; +int ret; + ret =gen8_ppgtt_init_common (ppgtt, dev_priv->gtt.base.total); +if (ret) +return ret; + +/* Aliasing PPGTT has to always work and be mapped because of the way we +* use RESTORE_INHIBIT in the context switch. This will be fixed +* eventually. */ ret =gen8_alloc_va_range (&ppgtt->base, start, size);if (ret) {free_pt_scratch (ppgtt->scratch_pd, ppgtt->base.dev);return ret; } - start =0 ; - size = orig_length; -gen8_for_each_pdpe (pd, &ppgtt->pdp, start, size, temp, pdpe)gen8_map_pagetable_range (pd, start, size, ppgtt->base.dev); + ppgtt->base.allocate_va_range =NULL ; + ppgtt->base.teardown_va_range =NULL ; + ppgtt->base.clear_range = gen8_ppgtt_clear_range; + +return 0 ; +} + +static int gen8_ppgtt_init (struct i915_hw_ppgtt *ppgtt) +{ +struct drm_device *dev = ppgtt->base.dev; +struct drm_i915_private *dev_priv = dev->dev_private; +int ret; + + ret =gen8_ppgtt_init_common (ppgtt, dev_priv->gtt.base.total); +if (ret) +return ret; + + ppgtt->base.allocate_va_range = gen8_alloc_va_range; + ppgtt->base.teardown_va_range = gen8_teardown_va_range; + ppgtt->base.clear_range =NULL ; +return 0 ; }@@ -1413,9 +1652,9 @@ static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt, bool aliasing)if (ret)return ret; - ppgtt->base.allocate_va_range = gen6_alloc_va_range; - ppgtt->base.teardown_va_range = gen6_teardown_va_range; - ppgtt->base.clear_range = gen6_ppgtt_clear_range; + ppgtt->base.allocate_va_range = aliasing ?NULL : gen6_alloc_va_range; + ppgtt->base.teardown_va_range = aliasing ?NULL : gen6_teardown_va_range; + ppgtt->base.clear_range = aliasing ? gen6_ppgtt_clear_range :NULL ; ppgtt->base.insert_entries = gen6_ppgtt_insert_entries; ppgtt->base.cleanup = gen6_ppgtt_cleanup; ppgtt->base.start =0 ;@@ -1453,8 +1692,10 @@ static int __hw_ppgtt_init(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt,if (INTEL_INFO (dev)->gen <8 )return gen6_ppgtt_init (ppgtt, aliasing); +else if (aliasing) +return gen8_aliasing_ppgtt_init (ppgtt);else -return gen8_ppgtt_init (ppgtt, dev_priv->gtt.base.total); +return gen8_ppgtt_init (ppgtt); }int i915_ppgtt_init (struct drm_device *dev,struct i915_hw_ppgtt *ppgtt) {@@ -1466,8 +1707,9 @@ int i915_ppgtt_init(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt)kref_init (&ppgtt->ref);drm_mm_init (&ppgtt->base.mm, ppgtt->base.start, ppgtt->base.total); - ppgtt->base.clear_range (&ppgtt->base,0 , - ppgtt->base.total, true); +if (ppgtt->base.clear_range) + ppgtt->base.clear_range (&ppgtt->base,0 , + ppgtt->base.total, true);i915_init_vm (dev_priv, &ppgtt->base); }@@ -1565,10 +1807,7 @@ ppgtt_bind_vma(struct i915_vma *vma,static void ppgtt_unbind_vma (struct i915_vma *vma) { - vma->vm->clear_range (vma->vm, - vma->node.start, - vma->obj->base.size, - true); +WARN_ON (vma->vm->teardown_va_range && vma->vm->clear_range);if (vma->vm->teardown_va_range) { trace_i915_va_teardown(vma->vm, vma->node.start, vma->node.size,@@ -1576,7 +1815,13 @@ static void ppgtt_unbind_vma(struct i915_vma *vma)
drivers/gpu/drm/i915/i915_gem_gtt.c static void ppgtt_unbind_vma(struct i915_vma *vma) · line 1824
Quoted 10 lines
drivers/gpu/drm/i915/i915_gem_gtt.c +7
-1vma->vm->teardown_va_range (vma->vm, vma->node.start, vma->node.size); - } + }else if (vma->vm->clear_range) { + vma->vm->clear_range (vma->vm, + vma->node.start, + vma->obj->base.size, + true); + }else +BUG ();
No gratitious additions of BUG please.
Quoted 50 lines
drivers/gpu/drm/i915/i915_gem_gtt.c +0
-0}extern int intel_iommu_gfx_mapped;drivers/gpu/drm/i915/i915_gem_gtt.h +15
-1@@ -190,13 +190,26 @@ struct i915_vma { u32 flags); }; - +/* Zombies. We write page tables with the CPU, and hardware switches them with +* the GPU. As such, the only time we can safely remove a page table is when we +* know the context is idle. Since we have no good way to do this, we use the +* zombie. +* +* Under memory pressure, if the system is idle, zombies may be reaped. +* +* There are 3 states a page table can be in (not including scratch) +* bitmap = 0, zombie = 0: unallocated +* bitmap = 1, zombie = 0: allocated +* bitmap = 0, zombie = 1: zombie +* bitmap = 1, zombie = 1: invalid +*/ struct i915_pagetab {struct page *page;dma_addr_t daddr;unsigned long *used_ptes;unsigned int scratch:1 ; +unsigned zombie:1 ; };struct i915_pagedir {@@ -208,6 +221,7 @@ struct i915_pagedir {unsigned long *used_pdes;struct i915_pagetab *page_tables[GEN6_PPGTT_PD_ENTRIES ]; + unsigned zombie:1; };struct i915_pagedirpo {Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
Signature · 7 lines
Daniel Vetter Software Engineer, Intel Corporation +41 (0) 79 365 57 48 - http://blog.ffwll.ch _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH v2 24/24 drm/i915/bdw: Dynamic page table allocations in lrc mode
In reply to Michel ThierryLogic ring contexts need to know the PDPs when they are populated. With dynamic page table allocations, these PDPs may not exist yet.
Check if PDPs have been allocated and use the scratch page if they do not exist yet.
Before submission, update the PDPs in the logic ring context as PDPs have been allocated.
--- drivers/gpu/drm/i915/intel_lrc.c | 80 +++++++++++++++++++++++++++++++++++----- 1 file changed, 70 insertions(+), 10 deletions(-)
drivers/gpu/drm/i915/intel_lrc.c +70 -10
@@ -358,6 +358,7 @@ static void execlists_elsp_write(struct intel_engine_cs *ring,static int execlists_update_context (struct drm_i915_gem_object *ctx_obj,struct drm_i915_gem_object *ring_obj, +struct i915_hw_ppgtt *ppgtt,u32 tail) {struct page *page;
@@ -369,6 +370,40 @@ static int execlists_update_context(struct drm_i915_gem_object *ctx_obj, reg_state[CTX_RING_TAIL +1 ] = tail; reg_state[CTX_RING_BUFFER_START +1 ] =i915_gem_obj_ggtt_offset (ring_obj); +/* True PPGTT with dynamic page allocation: update PDP registers and +* point the unallocated PDPs to the scratch page +*/ +if (ppgtt) { +if (test_bit (3 , ppgtt->pdp.used_pdpes)) { + reg_state[CTX_PDP3_UDW +1 ] =upper_32_bits (ppgtt->pdp.pagedir[3 ]->daddr); + reg_state[CTX_PDP3_LDW +1 ] =lower_32_bits (ppgtt->pdp.pagedir[3 ]->daddr); + }else { + reg_state[CTX_PDP3_UDW +1 ] =upper_32_bits (ppgtt->scratch_pd->daddr); + reg_state[CTX_PDP3_LDW +1 ] =lower_32_bits (ppgtt->scratch_pd->daddr); + } +if (test_bit (2 , ppgtt->pdp.used_pdpes)) { + reg_state[CTX_PDP2_UDW +1 ] =upper_32_bits (ppgtt->pdp.pagedir[2 ]->daddr); + reg_state[CTX_PDP2_LDW +1 ] =lower_32_bits (ppgtt->pdp.pagedir[2 ]->daddr); + }else { + reg_state[CTX_PDP2_UDW +1 ] =upper_32_bits (ppgtt->scratch_pd->daddr); + reg_state[CTX_PDP2_LDW +1 ] =lower_32_bits (ppgtt->scratch_pd->daddr); + } +if (test_bit (1 , ppgtt->pdp.used_pdpes)) { + reg_state[CTX_PDP1_UDW +1 ] =upper_32_bits (ppgtt->pdp.pagedir[1 ]->daddr); + reg_state[CTX_PDP1_LDW +1 ] =lower_32_bits (ppgtt->pdp.pagedir[1 ]->daddr); + }else { + reg_state[CTX_PDP1_UDW +1 ] =upper_32_bits (ppgtt->scratch_pd->daddr); + reg_state[CTX_PDP1_LDW +1 ] =lower_32_bits (ppgtt->scratch_pd->daddr); + } +if (test_bit (0 , ppgtt->pdp.used_pdpes)) { + reg_state[CTX_PDP0_UDW +1 ] =upper_32_bits (ppgtt->pdp.pagedir[0 ]->daddr); + reg_state[CTX_PDP0_LDW +1 ] =lower_32_bits (ppgtt->pdp.pagedir[0 ]->daddr); + }else { + reg_state[CTX_PDP0_UDW +1 ] =upper_32_bits (ppgtt->scratch_pd->daddr); + reg_state[CTX_PDP0_LDW +1 ] =lower_32_bits (ppgtt->scratch_pd->daddr); + } + } +kunmap_atomic (reg_state);return 0 ;
@@ -387,7 +422,7 @@ static void execlists_submit_contexts(struct intel_engine_cs *ring,WARN_ON (!i915_gem_obj_is_pinned (ctx_obj0));WARN_ON (!i915_gem_obj_is_pinned (ringbuf0->obj)); -execlists_update_context (ctx_obj0, ringbuf0->obj, tail0); +execlists_update_context (ctx_obj0, ringbuf0->obj, to0->ppgtt, tail0);if (to1) { ringbuf1 = to1->engine[ring->id].ringbuf;
@@ -396,7 +431,7 @@ static void execlists_submit_contexts(struct intel_engine_cs *ring,WARN_ON (!i915_gem_obj_is_pinned (ctx_obj1));WARN_ON (!i915_gem_obj_is_pinned (ringbuf1->obj)); -execlists_update_context (ctx_obj1, ringbuf1->obj, tail1); +execlists_update_context (ctx_obj1, ringbuf1->obj, to1->ppgtt, tail1); }execlists_elsp_write (ring, ctx_obj0, ctx_obj1);
@@ -1731,14 +1766,39 @@ populate_lr_context(struct intel_context *ctx, struct drm_i915_gem_object *ctx_o reg_state[CTX_PDP1_LDW ] =GEN8_RING_PDP_LDW (ring,1 ); reg_state[CTX_PDP0_UDW ] =GEN8_RING_PDP_UDW (ring,0 ); reg_state[CTX_PDP0_LDW ] =GEN8_RING_PDP_LDW (ring,0 ); - reg_state[CTX_PDP3_UDW +1 ] =upper_32_bits (ppgtt->pdp.pagedir[3 ]->daddr); - reg_state[CTX_PDP3_LDW +1 ] =lower_32_bits (ppgtt->pdp.pagedir[3 ]->daddr); - reg_state[CTX_PDP2_UDW +1 ] =upper_32_bits (ppgtt->pdp.pagedir[2 ]->daddr); - reg_state[CTX_PDP2_LDW +1 ] =lower_32_bits (ppgtt->pdp.pagedir[2 ]->daddr); - reg_state[CTX_PDP1_UDW +1 ] =upper_32_bits (ppgtt->pdp.pagedir[1 ]->daddr); - reg_state[CTX_PDP1_LDW +1 ] =lower_32_bits (ppgtt->pdp.pagedir[1 ]->daddr); - reg_state[CTX_PDP0_UDW +1 ] =upper_32_bits (ppgtt->pdp.pagedir[0 ]->daddr); - reg_state[CTX_PDP0_LDW +1 ] =lower_32_bits (ppgtt->pdp.pagedir[0 ]->daddr); + +/* With dynamic page allocation, PDPs may not be allocated at this point, +* Point the unallocated PDPs to the scratch page +*/ +if (test_bit (3 , ppgtt->pdp.used_pdpes)) { + reg_state[CTX_PDP3_UDW +1 ] =upper_32_bits (ppgtt->pdp.pagedir[3 ]->daddr); + reg_state[CTX_PDP3_LDW +1 ] =lower_32_bits (ppgtt->pdp.pagedir[3 ]->daddr); + }else { + reg_state[CTX_PDP3_UDW +1 ] =upper_32_bits (ppgtt->scratch_pd->daddr); + reg_state[CTX_PDP3_LDW +1 ] =lower_32_bits (ppgtt->scratch_pd->daddr); + } +if (test_bit (2 , ppgtt->pdp.used_pdpes)) { + reg_state[CTX_PDP2_UDW +1 ] =upper_32_bits (ppgtt->pdp.pagedir[2 ]->daddr); + reg_state[CTX_PDP2_LDW +1 ] =lower_32_bits (ppgtt->pdp.pagedir[2 ]->daddr); + }else { + reg_state[CTX_PDP2_UDW +1 ] =upper_32_bits (ppgtt->scratch_pd->daddr); + reg_state[CTX_PDP2_LDW +1 ] =lower_32_bits (ppgtt->scratch_pd->daddr); + } +if (test_bit (1 , ppgtt->pdp.used_pdpes)) { + reg_state[CTX_PDP1_UDW +1 ] =upper_32_bits (ppgtt->pdp.pagedir[1 ]->daddr); + reg_state[CTX_PDP1_LDW +1 ] =lower_32_bits (ppgtt->pdp.pagedir[1 ]->daddr); + }else { + reg_state[CTX_PDP1_UDW +1 ] =upper_32_bits (ppgtt->scratch_pd->daddr); + reg_state[CTX_PDP1_LDW +1 ] =lower_32_bits (ppgtt->scratch_pd->daddr); + } +if (test_bit (0 , ppgtt->pdp.used_pdpes)) { + reg_state[CTX_PDP0_UDW +1 ] =upper_32_bits (ppgtt->pdp.pagedir[0 ]->daddr); + reg_state[CTX_PDP0_LDW +1 ] =lower_32_bits (ppgtt->pdp.pagedir[0 ]->daddr); + }else { + reg_state[CTX_PDP0_UDW +1 ] =upper_32_bits (ppgtt->scratch_pd->daddr); + reg_state[CTX_PDP0_LDW +1 ] =lower_32_bits (ppgtt->scratch_pd->daddr); + } +if (ring->id ==RCS ) { reg_state[CTX_LRI_HEADER_2 ] =MI_LOAD_REGISTER_IMM (1 ); reg_state[CTX_R_PWR_CLK_STATE ] =0x20c8 ;
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
Re: [PATCH v2 24/24] drm/i915/bdw: Dynamic page table allocations in lrc mode
Quoted 10 lines · Michel Thierry
Logic ring contexts need to know the PDPs when they are populated. With dynamic page table allocations, these PDPs may not exist yet.
Check if PDPs have been allocated and use the scratch page if they do not exist yet.
Before submission, update the PDPs in the logic ring context as PDPs have been allocated.
Signed-off-by Michel Thierry <[email protected]>
Patch subject is imo a bit misleading. What about "support dynamic pdp updates in lrc mode"?
-Daniel
Quoted 130 lines
--- drivers/gpu/drm/i915/intel_lrc.c | 80 +++++++++++++++++++++++++++++++++++----- 1 file changed, 70 insertions(+), 10 deletions(-)drivers/gpu/drm/i915/intel_lrc.c +70
-10@@ -358,6 +358,7 @@ static void execlists_elsp_write(struct intel_engine_cs *ring,static int execlists_update_context (struct drm_i915_gem_object *ctx_obj,struct drm_i915_gem_object *ring_obj, +struct i915_hw_ppgtt *ppgtt,u32 tail) {struct page *page;@@ -369,6 +370,40 @@ static int execlists_update_context(struct drm_i915_gem_object *ctx_obj, reg_state[CTX_RING_TAIL +1 ] = tail; reg_state[CTX_RING_BUFFER_START +1 ] =i915_gem_obj_ggtt_offset (ring_obj); +/* True PPGTT with dynamic page allocation: update PDP registers and +* point the unallocated PDPs to the scratch page +*/ +if (ppgtt) { +if (test_bit (3 , ppgtt->pdp.used_pdpes)) { + reg_state[CTX_PDP3_UDW +1 ] =upper_32_bits (ppgtt->pdp.pagedir[3 ]->daddr); + reg_state[CTX_PDP3_LDW +1 ] =lower_32_bits (ppgtt->pdp.pagedir[3 ]->daddr); + }else { + reg_state[CTX_PDP3_UDW +1 ] =upper_32_bits (ppgtt->scratch_pd->daddr); + reg_state[CTX_PDP3_LDW +1 ] =lower_32_bits (ppgtt->scratch_pd->daddr); + } +if (test_bit (2 , ppgtt->pdp.used_pdpes)) { + reg_state[CTX_PDP2_UDW +1 ] =upper_32_bits (ppgtt->pdp.pagedir[2 ]->daddr); + reg_state[CTX_PDP2_LDW +1 ] =lower_32_bits (ppgtt->pdp.pagedir[2 ]->daddr); + }else { + reg_state[CTX_PDP2_UDW +1 ] =upper_32_bits (ppgtt->scratch_pd->daddr); + reg_state[CTX_PDP2_LDW +1 ] =lower_32_bits (ppgtt->scratch_pd->daddr); + } +if (test_bit (1 , ppgtt->pdp.used_pdpes)) { + reg_state[CTX_PDP1_UDW +1 ] =upper_32_bits (ppgtt->pdp.pagedir[1 ]->daddr); + reg_state[CTX_PDP1_LDW +1 ] =lower_32_bits (ppgtt->pdp.pagedir[1 ]->daddr); + }else { + reg_state[CTX_PDP1_UDW +1 ] =upper_32_bits (ppgtt->scratch_pd->daddr); + reg_state[CTX_PDP1_LDW +1 ] =lower_32_bits (ppgtt->scratch_pd->daddr); + } +if (test_bit (0 , ppgtt->pdp.used_pdpes)) { + reg_state[CTX_PDP0_UDW +1 ] =upper_32_bits (ppgtt->pdp.pagedir[0 ]->daddr); + reg_state[CTX_PDP0_LDW +1 ] =lower_32_bits (ppgtt->pdp.pagedir[0 ]->daddr); + }else { + reg_state[CTX_PDP0_UDW +1 ] =upper_32_bits (ppgtt->scratch_pd->daddr); + reg_state[CTX_PDP0_LDW +1 ] =lower_32_bits (ppgtt->scratch_pd->daddr); + } + } +kunmap_atomic (reg_state);return 0 ;@@ -387,7 +422,7 @@ static void execlists_submit_contexts(struct intel_engine_cs *ring,WARN_ON (!i915_gem_obj_is_pinned (ctx_obj0));WARN_ON (!i915_gem_obj_is_pinned (ringbuf0->obj)); -execlists_update_context (ctx_obj0, ringbuf0->obj, tail0); +execlists_update_context (ctx_obj0, ringbuf0->obj, to0->ppgtt, tail0);if (to1) { ringbuf1 = to1->engine[ring->id].ringbuf;@@ -396,7 +431,7 @@ static void execlists_submit_contexts(struct intel_engine_cs *ring,WARN_ON (!i915_gem_obj_is_pinned (ctx_obj1));WARN_ON (!i915_gem_obj_is_pinned (ringbuf1->obj)); -execlists_update_context (ctx_obj1, ringbuf1->obj, tail1); +execlists_update_context (ctx_obj1, ringbuf1->obj, to1->ppgtt, tail1); }execlists_elsp_write (ring, ctx_obj0, ctx_obj1);@@ -1731,14 +1766,39 @@ populate_lr_context(struct intel_context *ctx, struct drm_i915_gem_object *ctx_o reg_state[CTX_PDP1_LDW ] =GEN8_RING_PDP_LDW (ring,1 ); reg_state[CTX_PDP0_UDW ] =GEN8_RING_PDP_UDW (ring,0 ); reg_state[CTX_PDP0_LDW ] =GEN8_RING_PDP_LDW (ring,0 ); - reg_state[CTX_PDP3_UDW +1 ] =upper_32_bits (ppgtt->pdp.pagedir[3 ]->daddr); - reg_state[CTX_PDP3_LDW +1 ] =lower_32_bits (ppgtt->pdp.pagedir[3 ]->daddr); - reg_state[CTX_PDP2_UDW +1 ] =upper_32_bits (ppgtt->pdp.pagedir[2 ]->daddr); - reg_state[CTX_PDP2_LDW +1 ] =lower_32_bits (ppgtt->pdp.pagedir[2 ]->daddr); - reg_state[CTX_PDP1_UDW +1 ] =upper_32_bits (ppgtt->pdp.pagedir[1 ]->daddr); - reg_state[CTX_PDP1_LDW +1 ] =lower_32_bits (ppgtt->pdp.pagedir[1 ]->daddr); - reg_state[CTX_PDP0_UDW +1 ] =upper_32_bits (ppgtt->pdp.pagedir[0 ]->daddr); - reg_state[CTX_PDP0_LDW +1 ] =lower_32_bits (ppgtt->pdp.pagedir[0 ]->daddr); + +/* With dynamic page allocation, PDPs may not be allocated at this point, +* Point the unallocated PDPs to the scratch page +*/ +if (test_bit (3 , ppgtt->pdp.used_pdpes)) { + reg_state[CTX_PDP3_UDW +1 ] =upper_32_bits (ppgtt->pdp.pagedir[3 ]->daddr); + reg_state[CTX_PDP3_LDW +1 ] =lower_32_bits (ppgtt->pdp.pagedir[3 ]->daddr); + }else { + reg_state[CTX_PDP3_UDW +1 ] =upper_32_bits (ppgtt->scratch_pd->daddr); + reg_state[CTX_PDP3_LDW +1 ] =lower_32_bits (ppgtt->scratch_pd->daddr); + } +if (test_bit (2 , ppgtt->pdp.used_pdpes)) { + reg_state[CTX_PDP2_UDW +1 ] =upper_32_bits (ppgtt->pdp.pagedir[2 ]->daddr); + reg_state[CTX_PDP2_LDW +1 ] =lower_32_bits (ppgtt->pdp.pagedir[2 ]->daddr); + }else { + reg_state[CTX_PDP2_UDW +1 ] =upper_32_bits (ppgtt->scratch_pd->daddr); + reg_state[CTX_PDP2_LDW +1 ] =lower_32_bits (ppgtt->scratch_pd->daddr); + } +if (test_bit (1 , ppgtt->pdp.used_pdpes)) { + reg_state[CTX_PDP1_UDW +1 ] =upper_32_bits (ppgtt->pdp.pagedir[1 ]->daddr); + reg_state[CTX_PDP1_LDW +1 ] =lower_32_bits (ppgtt->pdp.pagedir[1 ]->daddr); + }else { + reg_state[CTX_PDP1_UDW +1 ] =upper_32_bits (ppgtt->scratch_pd->daddr); + reg_state[CTX_PDP1_LDW +1 ] =lower_32_bits (ppgtt->scratch_pd->daddr); + } +if (test_bit (0 , ppgtt->pdp.used_pdpes)) { + reg_state[CTX_PDP0_UDW +1 ] =upper_32_bits (ppgtt->pdp.pagedir[0 ]->daddr); + reg_state[CTX_PDP0_LDW +1 ] =lower_32_bits (ppgtt->pdp.pagedir[0 ]->daddr); + }else { + reg_state[CTX_PDP0_UDW +1 ] =upper_32_bits (ppgtt->scratch_pd->daddr); + reg_state[CTX_PDP0_LDW +1 ] =lower_32_bits (ppgtt->scratch_pd->daddr); + } +if (ring->id ==RCS ) { reg_state[CTX_LRI_HEADER_2 ] =MI_LOAD_REGISTER_IMM (1 ); reg_state[CTX_R_PWR_CLK_STATE ] =0x20c8 ;Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
Signature · 7 lines
Daniel Vetter Software Engineer, Intel Corporation +41 (0) 79 365 57 48 - http://blog.ffwll.ch _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
Quoted 7 lines · Michel Thierry
Addressing comments from v1.
For GEN8, it has also been extended to work in logical ring submission (lrc) mode, as it will be the preferred mode of operation.
I also tried to update the lrc code at the same time the ppgtt refactoring occurred, leaving only one patch that is exclusively for lrc.
Quoted 5 lines
This list can be seen in 3 parts:
[01-10] Include code rework for PPGTT (all GENs).
[11-14] Adds page table allocation for GEN6/GEN7
[15-24] Enables dynamic allocation in GEN8. It is enabled for both legacy and execlist submission modes.
Ok, I think I start to see the forrest for the individual trees here. Thanks for reworking the series.
More comments in replies to individual patches. I've mostly concentrated on the gen6/7 code and mostly ignored the details for bdw, but where applicapable please do similar adjustments.
Since there's a bunch of simpler prep patches please start with the detailed review even when some of the later patches are still under discussion. That way I can start with merging. Otherwise I think we can go to the detailed review phase for the entire series with my comments addressed (and starting with the review while doing some of these reworks probably helps the reviewer, too).
Thanks, Daniel
Quoted 47 lines
Ben Widawsky (23):
drm/i915: Add some extra guards inevict_vmdrm/i915/trace: Fix offsets for 64b
drm/i915: Rename toGEN8_LEGACY_PDPES
drm/i915: Setup less PPGTT on failed pagedir drm/i915/gen8: Un-hardcode number of page directories drm/i915: Range clearing is PPGTT agnostic
drm/i915: page table abstractions
drm/i915: Complete page table structures
drm/i915: Create page table allocators
drm/i915: Track GEN6 page table usage
drm/i915: Extract context switch skip and pd load logic drm/i915: Track page table reload need
drm/i915: Initialize all contexts
drm/i915: Finish gen6/7 dynamic page table allocation drm/i915/bdw: Use dynamic allocation idioms on free drm/i915/bdw: pagedirs rework allocation
drm/i915/bdw: pagetable allocation rework
drm/i915/bdw: Update pdp switch and point unused PDPs to scratch page drm/i915: num_pd_pages/num_pd_entries isn't useful drm/i915: Extract PPGTT param from pagedir alloc drm/i915/bdw: Split out mappings
drm/i915/bdw: begin bitmap tracking
drm/i915/bdw: Dynamic page table allocationsMichel Thierry (1):
drm/i915/bdw: Dynamic page table allocations in lrc modedrivers/gpu/drm/i915/i915_debugfs.c | 7 +- drivers/gpu/drm/i915/i915_gem.c | 11 + drivers/gpu/drm/i915/i915_gem_context.c | 62 +- drivers/gpu/drm/i915/i915_gem_evict.c | 3 + drivers/gpu/drm/i915/i915_gem_execbuffer.c | 11 + drivers/gpu/drm/i915/i915_gem_gtt.c | 1200 ++++++++++++++++++++-------- drivers/gpu/drm/i915/i915_gem_gtt.h | 250 +++++- drivers/gpu/drm/i915/i915_trace.h | 123 ++- drivers/gpu/drm/i915/intel_lrc.c | 80 +- 9 files changed, 1360 insertions(+), 387 deletions(-)Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
Signature · 7 lines
Daniel Vetter Software Engineer, Intel Corporation +41 (0) 79 365 57 48 - http://blog.ffwll.ch _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
v3 cover PPGTT dynamic page allocations
In reply to Michel ThierryThis new patchset addresses most of the comments from v2. It still teardowns pagetables (which I plan to ammend shortly), but I think there were already enough changes to justify it.
For GEN8, it has also been extended to work in logical ring submission (lrc) mode, as it will be the preferred mode of operation.
I also tried to update the lrc code at the same time the ppgtt refactoring occurred, leaving only one patch that is exclusively for lrc.
This list can be seen in 3 parts:
[01-09] Include code rework for PPGTT (all GENs). [10-15] Adds page table allocation for GEN6/GEN7 [16-25] Enables dynamic allocation in GEN8,for both legacy and execlist submission modes.
Ben Widawsky (22):
drm/i915/trace: Fix offsets for 64b
drm/i915: Rename to GEN8_LEGACY_PDPES
drm/i915: Setup less PPGTT on failed page_directory drm/i915/gen8: Un-hardcode number of page directories drm/i915: Range clearing is PPGTT agnostic
drm/i915: page table abstractions
drm/i915: Complete page table structures
drm/i915: Create page table allocators
drm/i915: Track GEN6 page table usage
drm/i915: Extract context switch skip and pd load logic drm/i915: Track page table reload need
drm/i915: Initialize all contexts
drm/i915: Finish gen6/7 dynamic page table allocation drm/i915/bdw: Use dynamic allocation idioms on free drm/i915/bdw: page directories rework allocation drm/i915/bdw: pagetable allocation rework
drm/i915/bdw: Update pdp switch and point unused PDPs to scratch page drm/i915: num_pd_pages/num_pd_entries isn't useful drm/i915: Extract PPGTT param from page_directory alloc drm/i915/bdw: Split out mappings
drm/i915/bdw: begin bitmap tracking
drm/i915/bdw: Dynamic page table allocations
Michel Thierry (3):
drm/i915: Plumb drm_device through page tables operations drm/i915: Add dynamic page trace events drm/i915/bdw: Support dynamic pdp updates in lrc mode
drivers/gpu/drm/i915/i915_debugfs.c | 7 +- drivers/gpu/drm/i915/i915_gem.c | 11 + drivers/gpu/drm/i915/i915_gem_context.c | 62 +- drivers/gpu/drm/i915/i915_gem_execbuffer.c | 11 + drivers/gpu/drm/i915/i915_gem_gtt.c | 1184 ++++++++++++++++++++-------- drivers/gpu/drm/i915/i915_gem_gtt.h | 217 ++++- drivers/gpu/drm/i915/i915_trace.h | 123 ++- drivers/gpu/drm/i915/intel_lrc.c | 80 +- 8 files changed, 1329 insertions(+), 366 deletions(-)
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH v3 1/25 drm/i915/trace: Fix offsets for 64b
From: Ben Widawsky <[email protected]>
--- drivers/gpu/drm/i915/i915_trace.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-)
drivers/gpu/drm/i915/i915_trace.h +4 -4
@@ -115,7 +115,7 @@ TRACE_EVENT(i915_vma_bind,TP_STRUCT__entry (__field (struct drm_i915_gem_object *,obj ) __field(struct i915_address_space *, vm) -__field (u32, offset) +__field (u64, offset)__field (u32, size)__field (unsigned, flags) ),
@@ -128,7 +128,7 @@ TRACE_EVENT(i915_vma_bind, __entry->flags = flags; ), -TP_printk ("obj=%p, offset=%08x size=%x%s vm=%p" , +TP_printk ("obj=%p, offset=%016llx size=%x%s vm=%p" , __entry->obj, __entry->offset, __entry->size, __entry->flags &PIN_MAPPABLE ?", mappable" :"" , __entry->vm)
@@ -141,7 +141,7 @@ TRACE_EVENT(i915_vma_unbind,TP_STRUCT__entry (__field (struct drm_i915_gem_object *,obj ) __field(struct i915_address_space *, vm) -__field (u32, offset) +__field (u64, offset)__field (u32, size) ),
@@ -152,7 +152,7 @@ TRACE_EVENT(i915_vma_unbind, __entry->size = vma->node.size; ), -TP_printk ("obj=%p, offset=%08x size=%x vm=%p" , +TP_printk ("obj=%p, offset=%016llx size=%x vm=%p" , __entry->obj, __entry->offset, __entry->size, __entry->vm) );
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH v3 2/25 drm/i915: Rename to GEN8_LEGACY_PDPES
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
In gen8, 32b PPGTT has always had one "pdp" (it doesn't actually have one, but it resembles having one). The #define was confusing as is, and using "PDPE" is a much better description.
sed -i 's/GEN8_LEGACY_PDPS/GEN8_LEGACY_PDPES/' drivers/gpu/drm/i915/*.[ch]
It also matches the x86 pagetable terminology:
PTE = Page Table Entry - pagetable level 1 page PDE = Page Directory Entry - pagetable level 2 page PDPE = Page Directory Pointer Entry - pagetable level 3 page
And in the near future (for 48b addressing):
PML4E = Page Map Level 4 Entry
v2: Expanded information about Page Directory/Table nomenclature.
--- drivers/gpu/drm/i915/i915_gem_gtt.c | 6 +++--- drivers/gpu/drm/i915/i915_gem_gtt.h | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-)
drivers/gpu/drm/i915/i915_gem_gtt.c +3 -3
@@ -375,7 +375,7 @@ static void gen8_ppgtt_insert_entries(struct i915_address_space *vm, pt_vaddr =NULL ;for_each_sg_page (pages->sgl, &sg_iter, pages->nents,0 ) { -if (WARN_ON (pdpe >=GEN8_LEGACY_PDPS )) +if (WARN_ON (pdpe >=GEN8_LEGACY_PDPES ))break ;if (pt_vaddr ==NULL )
@@ -486,7 +486,7 @@ bail:static int gen8_ppgtt_allocate_page_tables (struct i915_hw_ppgtt *ppgtt,const int max_pdp) { -struct page **pt_pages[GEN8_LEGACY_PDPS ]; +struct page **pt_pages[GEN8_LEGACY_PDPES ];int i, ret;for (i =0 ; i < max_pdp; i++) {
@@ -537,7 +537,7 @@ static int gen8_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt,return -ENOMEM ; ppgtt->num_pd_pages =1 <<get_order (max_pdp <<PAGE_SHIFT ); -BUG_ON (ppgtt->num_pd_pages >GEN8_LEGACY_PDPS ); +BUG_ON (ppgtt->num_pd_pages >GEN8_LEGACY_PDPES );return 0 ; }
drivers/gpu/drm/i915/i915_gem_gtt.h +3 -3
@@ -88,7 +88,7 @@ typedef gen8_gtt_pte_t gen8_ppgtt_pde_t;#define GEN8_PDE_MASK 0x1ff#define GEN8_PTE_SHIFT 12#define GEN8_PTE_MASK 0x1ff -#define GEN8_LEGACY_PDPS 4 +#define GEN8_LEGACY_PDPES 4#define GEN8_PTES_PER_PAGE (PAGE_SIZE / sizeof(gen8_gtt_pte_t))#define GEN8_PDES_PER_PAGE (PAGE_SIZE / sizeof(gen8_ppgtt_pde_t))
@@ -273,12 +273,12 @@ struct i915_hw_ppgtt {unsigned num_pd_pages;/* gen8+ */ union {struct page **pt_pages; -struct page **gen8_pt_pages[GEN8_LEGACY_PDPS ]; +struct page **gen8_pt_pages[GEN8_LEGACY_PDPES ]; };struct page *pd_pages;union {uint32_t pd_offset; -dma_addr_t pd_dma_addr[GEN8_LEGACY_PDPS ]; +dma_addr_t pd_dma_addr[GEN8_LEGACY_PDPES ]; };union {dma_addr_t *pt_dma_addr;
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH v3 3/25 drm/i915: Setup less PPGTT on failed page_directory
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
The current code will both potentially print a WARN, and setup part of the PPGTT structure. Neither of these harm the current code, it is simply for clarity, and to perhaps prevent later bugs, or weird debug messages.
--- drivers/gpu/drm/i915/i915_gem_gtt.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-)
drivers/gpu/drm/i915/i915_gem_gtt.c +4 -1
@@ -1032,11 +1032,14 @@ alloc: goto alloc; } +if (ret) +return ret; +if (ppgtt->node.start < dev_priv->gtt.mappable_end)DRM_DEBUG ("Forced to use aperture for PDEs\n" ); ppgtt->num_pd_entries =GEN6_PPGTT_PD_ENTRIES ; -return ret; +return 0 ; }static int gen6_ppgtt_allocate_page_tables (struct i915_hw_ppgtt *ppgtt)
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH v3 4/25 drm/i915/gen8: Un-hardcode number of page directories
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
--- drivers/gpu/drm/i915/i915_gem_gtt.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
drivers/gpu/drm/i915/i915_gem_gtt.h +1 -1
@@ -282,7 +282,7 @@ struct i915_hw_ppgtt { };union {dma_addr_t *pt_dma_addr; -dma_addr_t *gen8_pt_dma_addr[4 ]; +dma_addr_t *gen8_pt_dma_addr[GEN8_LEGACY_PDPES ]; };struct drm_i915_file_private *file_priv;
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH v3 5/25 drm/i915: Range clearing is PPGTT agnostic
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
Therefore we can do it from our general init function. Eventually, I hope to have a lot more commonality like this. It won't arrive yet, but this was a nice easy one.
--- drivers/gpu/drm/i915/i915_gem_gtt.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-)
drivers/gpu/drm/i915/i915_gem_gtt.c +2 -4
@@ -672,8 +672,6 @@ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size) ppgtt->base.start =0 ; ppgtt->base.total = ppgtt->num_pd_entries *GEN8_PTES_PER_PAGE *PAGE_SIZE ; - ppgtt->base.clear_range (&ppgtt->base,0 , ppgtt->base.total, true); -DRM_DEBUG_DRIVER ("Allocated %d pages for page directories (%d wasted)\n" , ppgtt->num_pd_pages, ppgtt->num_pd_pages - max_pdp); DRM_DEBUG_DRIVER("Allocated %d pages for page tables (%lld wasted)\n",
@@ -1146,8 +1144,6 @@ static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt) ppgtt->pd_offset = ppgtt->node.start /PAGE_SIZE *sizeof (gen6_gtt_pte_t); - ppgtt->base.clear_range (&ppgtt->base,0 , ppgtt->base.total, true); -DRM_DEBUG_DRIVER ("Allocated pde space (%ldM) at GTT entry: %lx\n" , ppgtt->node.size >>20 , ppgtt->node.start /PAGE_SIZE );
@@ -1181,6 +1177,8 @@ int i915_ppgtt_init(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt)kref_init (&ppgtt->ref);drm_mm_init (&ppgtt->base.mm, ppgtt->base.start, ppgtt->base.total); + ppgtt->base.clear_range (&ppgtt->base,0 , + ppgtt->base.total, true);i915_init_vm (dev_priv, &ppgtt->base); }
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH v3 6/25 drm/i915: page table abstractions
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
When we move to dynamic page allocation, keeping page_directory and pagetabs as separate structures will help to break actions into simpler tasks.
To help transition the code nicely there is some wasted space in gen6/7. This will be ameliorated shortly.
Following the x86 pagetable terminology:
PDPE = struct i915_page_directory_pointer_entry. PDE = struct i915_page_directory_entry [page_directory]. PTE = struct i915_page_table_entry [page_tables].
v2: fixed mismatches after clean-up/rebase.
v3: Clarify the names of the multiple levels of page tables (Daniel)
--- drivers/gpu/drm/i915/i915_gem_gtt.c | 177 ++++++++++++++++++------------------ drivers/gpu/drm/i915/i915_gem_gtt.h | 23 ++++- 2 files changed, 107 insertions(+), 93 deletions(-)
drivers/gpu/drm/i915/i915_gem_gtt.c +89 -88
@@ -334,7 +334,8 @@ static void gen8_ppgtt_clear_range(struct i915_address_space *vm, I915_CACHE_LLC, use_scratch);while (num_entries) { -struct page *page_table = ppgtt->gen8_pt_pages[pdpe][pde]; +struct i915_page_directory_entry *pd = &ppgtt->pdp.page_directory[pdpe]; +struct page *page_table = pd->page_tables[pde].page; last_pte = pte + num_entries;if (last_pte >GEN8_PTES_PER_PAGE )
@@ -378,8 +379,12 @@ static void gen8_ppgtt_insert_entries(struct i915_address_space *vm,if (WARN_ON (pdpe >=GEN8_LEGACY_PDPES ))break ; -if (pt_vaddr ==NULL ) - pt_vaddr =kmap_atomic (ppgtt->gen8_pt_pages[pdpe][pde]); +if (pt_vaddr ==NULL ) { +struct i915_page_directory_entry *pd = &ppgtt->pdp.page_directory[pdpe]; +struct page *page_table = pd->page_tables[pde].page; + + pt_vaddr =kmap_atomic (page_table); + } pt_vaddr[pte] = gen8_pte_encode(sg_page_iter_dma_address(&sg_iter),
@@ -403,29 +408,33 @@ static void gen8_ppgtt_insert_entries(struct i915_address_space *vm, } } -static void gen8_free_page_tables (struct page **pt_pages) +static void gen8_free_page_tables (struct i915_page_directory_entry *pd) {int i; -if (pt_pages ==NULL ) +if (pd->page_tables ==NULL )return ;for (i =0 ; i <GEN8_PDES_PER_PAGE ; i++) -if (pt_pages[i]) -__free_pages (pt_pages[i],0 ); +if (pd->page_tables[i].page) +__free_page (pd->page_tables[i].page); } -static void gen8_ppgtt_free (const struct i915_hw_ppgtt *ppgtt) +static void gen8_free_page_directories (struct i915_page_directory_entry *pd) +{ +kfree (pd->page_tables); +__free_page (pd->page); +} + +static void gen8_ppgtt_free (struct i915_hw_ppgtt *ppgtt) {int i;for (i =0 ; i < ppgtt->num_pd_pages; i++) { -gen8_free_page_tables (ppgtt->gen8_pt_pages[i]); -kfree (ppgtt->gen8_pt_pages[i]); +gen8_free_page_tables (&ppgtt->pdp.page_directory[i]); +gen8_free_page_directories (&ppgtt->pdp.page_directory[i]);kfree (ppgtt->gen8_pt_dma_addr[i]); } - -__free_pages (ppgtt->pd_pages,get_order (ppgtt->num_pd_pages <<PAGE_SHIFT )); }static void gen8_ppgtt_unmap_pages (struct i915_hw_ppgtt *ppgtt)
@@ -460,86 +469,75 @@ static void gen8_ppgtt_cleanup(struct i915_address_space *vm)gen8_ppgtt_free (ppgtt); } -static struct page **__gen8_alloc_page_tables (void ) +static int gen8_ppgtt_allocate_dma (struct i915_hw_ppgtt *ppgtt) { -struct page **pt_pages;int i; - pt_pages =kcalloc (GEN8_PDES_PER_PAGE ,sizeof (struct page *),GFP_KERNEL ); -if (!pt_pages) -return ERR_PTR (-ENOMEM ); - -for (i =0 ; i <GEN8_PDES_PER_PAGE ; i++) { - pt_pages[i] =alloc_page (GFP_KERNEL ); -if (!pt_pages[i]) - goto bail; +for (i =0 ; i < ppgtt->num_pd_pages; i++) { + ppgtt->gen8_pt_dma_addr[i] =kcalloc (GEN8_PDES_PER_PAGE , +sizeof (dma_addr_t), +GFP_KERNEL ); +if (!ppgtt->gen8_pt_dma_addr[i]) +return -ENOMEM ; } -return pt_pages; - -bail: -gen8_free_page_tables (pt_pages); -kfree (pt_pages); -return ERR_PTR (-ENOMEM ); +return 0 ; } -static int gen8_ppgtt_allocate_page_tables (struct i915_hw_ppgtt *ppgtt, -const int max_pdp) +static int gen8_ppgtt_allocate_page_tables (struct i915_hw_ppgtt *ppgtt) { -struct page **pt_pages[GEN8_LEGACY_PDPES ]; -int i, ret; +int i, j; -for (i =0 ; i < max_pdp; i++) { - pt_pages[i] =__gen8_alloc_page_tables (); -if (IS_ERR (pt_pages[i])) { - ret =PTR_ERR (pt_pages[i]); - goto unwind_out; +for (i =0 ; i < ppgtt->num_pd_pages; i++) { +for (j =0 ; j <GEN8_PDES_PER_PAGE ; j++) { +struct i915_page_table_entry *pt = &ppgtt->pdp.page_directory[i].page_tables[j]; + + pt->page =alloc_page (GFP_KERNEL | __GFP_ZERO); +if (!pt->page) + goto unwind_out; } } -/* NB: Avoid touching gen8_pt_pages until last to keep the allocation, -* "atomic" - for cleanup purposes. -*/ -for (i =0 ; i < max_pdp; i++) - ppgtt->gen8_pt_pages[i] = pt_pages[i]; -return 0 ; unwind_out: -while (i--) { -gen8_free_page_tables (pt_pages[i]); -kfree (pt_pages[i]); - } +while (i--) +gen8_free_page_tables (&ppgtt->pdp.page_directory[i]); -return ret; +return -ENOMEM ; } -static int gen8_ppgtt_allocate_dma (struct i915_hw_ppgtt *ppgtt) +static int gen8_ppgtt_allocate_page_directories (struct i915_hw_ppgtt *ppgtt, +const int max_pdp) {int i; -for (i =0 ; i < ppgtt->num_pd_pages; i++) { - ppgtt->gen8_pt_dma_addr[i] =kcalloc (GEN8_PDES_PER_PAGE , -sizeof (dma_addr_t), -GFP_KERNEL ); -if (!ppgtt->gen8_pt_dma_addr[i]) -return -ENOMEM ; - } +for (i =0 ; i < max_pdp; i++) { +struct i915_page_table_entry *pt; -return 0 ; -} + pt =kcalloc (GEN8_PDES_PER_PAGE ,sizeof (*pt),GFP_KERNEL ); +if (!pt) + goto unwind_out; -static int gen8_ppgtt_allocate_page_directories (struct i915_hw_ppgtt *ppgtt, -const int max_pdp) -{ - ppgtt->pd_pages =alloc_pages (GFP_KERNEL ,get_order (max_pdp <<PAGE_SHIFT )); -if (!ppgtt->pd_pages) -return -ENOMEM ; + ppgtt->pdp.page_directory[i].page =alloc_page (GFP_KERNEL ); +if (!ppgtt->pdp.page_directory[i].page) + goto unwind_out; + + ppgtt->pdp.page_directory[i].page_tables = pt; + } - ppgtt->num_pd_pages =1 <<get_order (max_pdp <<PAGE_SHIFT ); + ppgtt->num_pd_pages = max_pdp;BUG_ON (ppgtt->num_pd_pages >GEN8_LEGACY_PDPES );return 0 ; + +unwind_out: +while (i--) { +kfree (ppgtt->pdp.page_directory[i].page_tables); +__free_page (ppgtt->pdp.page_directory[i].page); + } + +return -ENOMEM ; } static int gen8_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt,
@@ -551,18 +549,19 @@ static int gen8_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt,if (ret)return ret; - ret =gen8_ppgtt_allocate_page_tables (ppgtt, max_pdp); -if (ret) { -__free_pages (ppgtt->pd_pages,get_order (max_pdp <<PAGE_SHIFT )); -return ret; - } + ret =gen8_ppgtt_allocate_page_tables (ppgtt); +if (ret) + goto err_out; ppgtt->num_pd_entries = max_pdp *GEN8_PDES_PER_PAGE ; ret =gen8_ppgtt_allocate_dma (ppgtt); -if (ret) -gen8_ppgtt_free (ppgtt); +if (!ret) +return ret; +/* TODO: Check this for all cases */ +err_out: +gen8_ppgtt_free (ppgtt);return ret; }
@@ -573,7 +572,7 @@ static int gen8_ppgtt_setup_page_directories(struct i915_hw_ppgtt *ppgtt,int ret; pd_addr =pci_map_page (ppgtt->base.dev->pdev, - &ppgtt->pd_pages[pd],0 , + ppgtt->pdp.page_directory[pd].page,0 ,PAGE_SIZE ,PCI_DMA_BIDIRECTIONAL ); ret =pci_dma_mapping_error (ppgtt->base.dev->pdev, pd_addr);
@@ -593,7 +592,7 @@ static int gen8_ppgtt_setup_page_tables(struct i915_hw_ppgtt *ppgtt,struct page *p;int ret; - p = ppgtt->gen8_pt_pages[pd][pt]; + p = ppgtt->pdp.page_directory[pd].page_tables[pt].page; pt_addr =pci_map_page (ppgtt->base.dev->pdev, p,0 ,PAGE_SIZE ,PCI_DMA_BIDIRECTIONAL ); ret =pci_dma_mapping_error (ppgtt->base.dev->pdev, pt_addr);
@@ -654,7 +653,7 @@ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size)*/ for (i =0 ; i < max_pdp; i++) {gen8_ppgtt_pde_t *pd_vaddr; - pd_vaddr =kmap_atomic (&ppgtt->pd_pages[i]); + pd_vaddr =kmap_atomic (ppgtt->pdp.page_directory[i].page);for (j =0 ; j <GEN8_PDES_PER_PAGE ; j++) {dma_addr_t addr = ppgtt->gen8_pt_dma_addr[i][j]; pd_vaddr[j] = gen8_pde_encode(ppgtt->base.dev, addr,
@@ -715,7 +714,7 @@ static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m) expected);seq_printf (m,"\tPDE: %x\n" , pd_entry); - pt_vaddr =kmap_atomic (ppgtt->pt_pages[pde]); + pt_vaddr =kmap_atomic (ppgtt->pd.page_tables[pde].page);for (pte =0 ; pte <I915_PPGTT_PT_ENTRIES ; pte+=4 ) {unsigned long va = (pde * PAGE_SIZE * I915_PPGTT_PT_ENTRIES) +
@@ -920,7 +919,7 @@ static void gen6_ppgtt_clear_range(struct i915_address_space *vm,if (last_pte >I915_PPGTT_PT_ENTRIES ) last_pte =I915_PPGTT_PT_ENTRIES ; - pt_vaddr =kmap_atomic (ppgtt->pt_pages[act_pt]); + pt_vaddr =kmap_atomic (ppgtt->pd.page_tables[act_pt].page);for (i = first_pte; i < last_pte; i++) pt_vaddr[i] = scratch_pte;
@@ -949,7 +948,7 @@ static void gen6_ppgtt_insert_entries(struct i915_address_space *vm, pt_vaddr =NULL ;for_each_sg_page (pages->sgl, &sg_iter, pages->nents,0 ) {if (pt_vaddr ==NULL ) - pt_vaddr =kmap_atomic (ppgtt->pt_pages[act_pt]); + pt_vaddr =kmap_atomic (ppgtt->pd.page_tables[act_pt].page); pt_vaddr[act_pte] = vm->pte_encode(sg_page_iter_dma_address(&sg_iter),
@@ -984,8 +983,8 @@ static void gen6_ppgtt_free(struct i915_hw_ppgtt *ppgtt)kfree (ppgtt->pt_dma_addr);for (i =0 ; i < ppgtt->num_pd_entries; i++) -__free_page (ppgtt->pt_pages[i]); -kfree (ppgtt->pt_pages); +__free_page (ppgtt->pd.page_tables[i].page); +kfree (ppgtt->pd.page_tables); }static void gen6_ppgtt_cleanup (struct i915_address_space *vm)
@@ -1042,22 +1041,22 @@ alloc:static int gen6_ppgtt_allocate_page_tables (struct i915_hw_ppgtt *ppgtt) { +struct i915_page_table_entry *pt;int i; - ppgtt->pt_pages =kcalloc (ppgtt->num_pd_entries,sizeof (struct page *), -GFP_KERNEL ); - -if (!ppgtt->pt_pages) + pt =kcalloc (ppgtt->num_pd_entries,sizeof (*pt),GFP_KERNEL ); +if (!pt)return -ENOMEM ;for (i =0 ; i < ppgtt->num_pd_entries; i++) { - ppgtt->pt_pages[i] =alloc_page (GFP_KERNEL ); -if (!ppgtt->pt_pages[i]) { + pt[i].page =alloc_page (GFP_KERNEL ); +if (!pt->page) {gen6_ppgtt_free (ppgtt);return -ENOMEM ; } } + ppgtt->pd.page_tables = pt;return 0 ; }
@@ -1092,9 +1091,11 @@ static int gen6_ppgtt_setup_page_tables(struct i915_hw_ppgtt *ppgtt)int i;for (i =0 ; i < ppgtt->num_pd_entries; i++) { +struct page *page;dma_addr_t pt_addr; - pt_addr =pci_map_page (dev->pdev, ppgtt->pt_pages[i],0 ,4096 , + page = ppgtt->pd.page_tables[i].page; + pt_addr =pci_map_page (dev->pdev, page,0 ,4096 ,PCI_DMA_BIDIRECTIONAL );if (pci_dma_mapping_error (dev->pdev, pt_addr)) {
@@ -1138,7 +1139,7 @@ static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt) ppgtt->base.insert_entries = gen6_ppgtt_insert_entries; ppgtt->base.cleanup = gen6_ppgtt_cleanup; ppgtt->base.start =0 ; - ppgtt->base.total = ppgtt->num_pd_entries *I915_PPGTT_PT_ENTRIES *PAGE_SIZE ; + ppgtt->base.total = ppgtt->num_pd_entries *I915_PPGTT_PT_ENTRIES *PAGE_SIZE ; ppgtt->debug_dump = gen6_dump_ppgtt; ppgtt->pd_offset =
drivers/gpu/drm/i915/i915_gem_gtt.h +18 -5
@@ -187,6 +187,20 @@ struct i915_vma { u32 flags); }; +struct i915_page_table_entry { +struct page *page; +}; + +struct i915_page_directory_entry { +struct page *page;/* NULL for GEN6-GEN7 */ +struct i915_page_table_entry *page_tables; +}; + +struct i915_page_directory_pointer_entry { +/* struct page *page; */ +struct i915_page_directory_entry page_directory[GEN8_LEGACY_PDPES ]; +}; +struct i915_address_space {struct drm_mm mm;struct drm_device *dev;
@@ -272,11 +286,6 @@ struct i915_hw_ppgtt {unsigned num_pd_entries;unsigned num_pd_pages;/* gen8+ */ union { -struct page **pt_pages; -struct page **gen8_pt_pages[GEN8_LEGACY_PDPES ]; - }; -struct page *pd_pages; -union {uint32_t pd_offset;dma_addr_t pd_dma_addr[GEN8_LEGACY_PDPES ]; };
@@ -284,6 +293,10 @@ struct i915_hw_ppgtt {dma_addr_t *pt_dma_addr;dma_addr_t *gen8_pt_dma_addr[GEN8_LEGACY_PDPES ]; }; +union { +struct i915_page_directory_pointer_entry pdp; +struct i915_page_directory_entry pd; + };struct drm_i915_file_private *file_priv;
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH v3 7/25 drm/i915: Complete page table structures
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
Move the remaining members over to the new page table structures.
This can be squashed with the previous commit if desire. The reasoning is the same as that patch. I simply felt it is easier to review if split.
v2: In lrc: s/ppgtt->pd_dma_addr[i]/ppgtt->pdp.page_directory[i].daddr/
v3: Rebase.
--- drivers/gpu/drm/i915/i915_debugfs.c | 2 +- drivers/gpu/drm/i915/i915_gem_gtt.c | 85 +++++++++++++------------------------ drivers/gpu/drm/i915/i915_gem_gtt.h | 14 +++--- drivers/gpu/drm/i915/intel_lrc.c | 16 +++---- 4 files changed, 45 insertions(+), 72 deletions(-)
drivers/gpu/drm/i915/i915_debugfs.c +1 -1
@@ -2153,7 +2153,7 @@ static void gen6_ppgtt_info(struct seq_file *m, struct drm_device *dev)struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;seq_puts (m,"aliasing PPGTT:\n" ); -seq_printf (m,"pd gtt offset: 0x%08x\n" , ppgtt->pd_offset); +seq_printf (m,"pd gtt offset: 0x%08x\n" , ppgtt->pd.pd_offset); ppgtt->debug_dump (ppgtt, m); }
drivers/gpu/drm/i915/i915_gem_gtt.c +30 -55
220 lines not on this page · raw message
drivers/gpu/drm/i915/i915_gem_gtt.h +6 -8
@@ -189,10 +189,16 @@ struct i915_vma {struct i915_page_table_entry {struct page *page; +dma_addr_t daddr; };struct i915_page_directory_entry {struct page *page;/* NULL for GEN6-GEN7 */ +union { +uint32_t pd_offset; +dma_addr_t daddr; + }; +struct i915_page_table_entry *page_tables; };
@@ -286,14 +292,6 @@ struct i915_hw_ppgtt {unsigned num_pd_entries;unsigned num_pd_pages;/* gen8+ */ union { -uint32_t pd_offset; -dma_addr_t pd_dma_addr[GEN8_LEGACY_PDPES ]; - }; -union { -dma_addr_t *pt_dma_addr; -dma_addr_t *gen8_pt_dma_addr[GEN8_LEGACY_PDPES ]; - }; -union {struct i915_page_directory_pointer_entry pdp;struct i915_page_directory_entry pd; };
drivers/gpu/drm/i915/intel_lrc.c +8 -8
@@ -1731,14 +1731,14 @@ populate_lr_context(struct intel_context *ctx, struct drm_i915_gem_object *ctx_o reg_state[CTX_PDP1_LDW ] =GEN8_RING_PDP_LDW (ring,1 ); reg_state[CTX_PDP0_UDW ] =GEN8_RING_PDP_UDW (ring,0 ); reg_state[CTX_PDP0_LDW ] =GEN8_RING_PDP_LDW (ring,0 ); - reg_state[CTX_PDP3_UDW +1 ] =upper_32_bits (ppgtt->pd_dma_addr[3 ]); - reg_state[CTX_PDP3_LDW +1 ] =lower_32_bits (ppgtt->pd_dma_addr[3 ]); - reg_state[CTX_PDP2_UDW +1 ] =upper_32_bits (ppgtt->pd_dma_addr[2 ]); - reg_state[CTX_PDP2_LDW +1 ] =lower_32_bits (ppgtt->pd_dma_addr[2 ]); - reg_state[CTX_PDP1_UDW +1 ] =upper_32_bits (ppgtt->pd_dma_addr[1 ]); - reg_state[CTX_PDP1_LDW +1 ] =lower_32_bits (ppgtt->pd_dma_addr[1 ]); - reg_state[CTX_PDP0_UDW +1 ] =upper_32_bits (ppgtt->pd_dma_addr[0 ]); - reg_state[CTX_PDP0_LDW +1 ] =lower_32_bits (ppgtt->pd_dma_addr[0 ]); + reg_state[CTX_PDP3_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[3 ].daddr); + reg_state[CTX_PDP3_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[3 ].daddr); + reg_state[CTX_PDP2_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[2 ].daddr); + reg_state[CTX_PDP2_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[2 ].daddr); + reg_state[CTX_PDP1_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[1 ].daddr); + reg_state[CTX_PDP1_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[1 ].daddr); + reg_state[CTX_PDP0_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[0 ].daddr); + reg_state[CTX_PDP0_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[0 ].daddr);if (ring->id ==RCS ) { reg_state[CTX_LRI_HEADER_2 ] =MI_LOAD_REGISTER_IMM (1 ); reg_state[CTX_R_PWR_CLK_STATE ] =0x20c8 ;
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx