gen8 ppgtt dynamic page allocations
Subsystems (2)
cover gen8 ppgtt dynamic page allocations
These are the last remining patches to enable dynamic allocation in gen8+. All credit to Ben's original design and Mika's extensive reviews.
During stress testing, the light restore context corruption problem was observed in some systems (resubmission with HEAD==TAIL). The workaround to prevent to prevent this known problem should be in place as we also update the PDPx registers before the context is send to execution.
The last patch is only of interest in systems with less than 4GB of memory. Now that the PPGTT table overhead is not that big, we can use the full virtual space address range in these systems.
Michel Thierry (13):
drm/i915: Remove _entry from PPGTT page structures drm/i915: Remove unnecessary gen8_ppgtt_unmap_pages drm/i915/gen8: Initialize page tables
drm/i915/gen8: Add dynamic allocation macros and helper functions drm/i915/gen8: page directories rework allocation drm/i915/gen8: pagetable allocation rework
drm/i915/gen8: Update pdp switch and point unused PDPs to scratch page drm/i915: num_pd_pages/num_pd_entries isn't useful drm/i915: Extract PPGTT param from page_directory alloc drm/i915/gen8: Split out mappings
drm/i915/gen8: begin bitmap tracking
drm/i915/gen8: Dynamic page table allocations
drm/i915: Use complete address space in true PPGTT
drivers/gpu/drm/i915/i915_debugfs.c | 2 - drivers/gpu/drm/i915/i915_gem_gtt.c | 620 +++++++++++++++++++++++------------- drivers/gpu/drm/i915/i915_gem_gtt.h | 82 ++++- drivers/gpu/drm/i915/i915_trace.h | 4 +- drivers/gpu/drm/i915/intel_lrc.c | 40 ++- 5 files changed, 504 insertions(+), 244 deletions(-)
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH 1/13 drm/i915: Remove _entry from PPGTT page structures
Lets try to keep this consistent:
Page Directory Pointer (PDP).
Page Directory (PD), also known as page directory pointer entries. Page Table (PT), also known as page directory entries.
s/struct i915_page_table_entry/struct i915_page_table/ s/struct i915_page_directory_entry/struct i915_page_directory/ s/struct i915_page_directory_pointer_entry/struct i915_page_directory_pointer/
--- drivers/gpu/drm/i915/i915_gem_gtt.c | 56 ++++++++++++++++++------------------- drivers/gpu/drm/i915/i915_gem_gtt.h | 16 +++++------ drivers/gpu/drm/i915/i915_trace.h | 4 +-- 3 files changed, 38 insertions(+), 38 deletions(-)
drivers/gpu/drm/i915/i915_gem_gtt.c +28 -28
@@ -320,7 +320,7 @@ static inline int i915_dma_map_page_single(struct page *page,return 0 ; } -static void unmap_and_free_pt (struct i915_page_table_entry *pt, +static void unmap_and_free_pt (struct i915_page_table *pt,struct drm_device *dev) {if (WARN_ON (!pt->page))
@@ -332,9 +332,9 @@ static void unmap_and_free_pt(struct i915_page_table_entry *pt,kfree (pt); } -static struct i915_page_table_entry *alloc_pt_single (struct drm_device *dev) +static struct i915_page_table *alloc_pt_single (struct drm_device *dev) { -struct i915_page_table_entry *pt; +struct i915_page_table *pt;const size_t count =INTEL_INFO (dev)->gen >=8 ?GEN8_PTES :GEN6_PTES ;int ret = -ENOMEM ;
@@ -383,7 +383,7 @@ fail_bitmap:* * Return: 0 if allocation succeeded. */ -static int alloc_pt_range (struct i915_page_directory_entry *pd,uint16_t pde,size_t count, +static int alloc_pt_range (struct i915_page_directory *pd,uint16_t pde,size_t count,struct drm_device *dev) {int i, ret;
@@ -393,7 +393,7 @@ static int alloc_pt_range(struct i915_page_directory_entry *pd, uint16_t pde, sireturn -EINVAL ;for (i = pde; i < pde + count; i++) { -struct i915_page_table_entry *pt =alloc_pt_single (dev); +struct i915_page_table *pt =alloc_pt_single (dev);if (IS_ERR (pt)) { ret =PTR_ERR (pt);
@@ -413,7 +413,7 @@ err_out:return ret; } -static void unmap_and_free_pd (struct i915_page_directory_entry *pd) +static void unmap_and_free_pd (struct i915_page_directory *pd) {if (pd->page) {__free_page (pd->page);
@@ -421,9 +421,9 @@ static void unmap_and_free_pd(struct i915_page_directory_entry *pd) } } -static struct i915_page_directory_entry *alloc_pd_single (void ) +static struct i915_page_directory *alloc_pd_single (void ) { -struct i915_page_directory_entry *pd; +struct i915_page_directory *pd; pd =kzalloc (sizeof (*pd),GFP_KERNEL );if (!pd)
@@ -497,8 +497,8 @@ static void gen8_ppgtt_clear_range(struct i915_address_space *vm, I915_CACHE_LLC, use_scratch);while (num_entries) { -struct i915_page_directory_entry *pd; -struct i915_page_table_entry *pt; +struct i915_page_directory *pd; +struct i915_page_table *pt;struct page *page_table;if (WARN_ON (!ppgtt->pdp.page_directory[pdpe]))
@@ -559,8 +559,8 @@ static void gen8_ppgtt_insert_entries(struct i915_address_space *vm,break ;if (pt_vaddr ==NULL ) { -struct i915_page_directory_entry *pd = ppgtt->pdp.page_directory[pdpe]; -struct i915_page_table_entry *pt = pd->page_table[pde]; +struct i915_page_directory *pd = ppgtt->pdp.page_directory[pdpe]; +struct i915_page_table *pt = pd->page_table[pde];struct page *page_table = pt->page; pt_vaddr =kmap_atomic (page_table);
@@ -588,7 +588,7 @@ static void gen8_ppgtt_insert_entries(struct i915_address_space *vm, } } -static void gen8_free_page_tables (struct i915_page_directory_entry *pd,struct drm_device *dev) +static void gen8_free_page_tables (struct i915_page_directory *pd,struct drm_device *dev) {int i;
@@ -632,8 +632,8 @@ static void gen8_ppgtt_unmap_pages(struct i915_hw_ppgtt *ppgtt) PCI_DMA_BIDIRECTIONAL);for (j =0 ; j <I915_PDES ; j++) { -struct i915_page_directory_entry *pd = ppgtt->pdp.page_directory[i]; -struct i915_page_table_entry *pt; +struct i915_page_directory *pd = ppgtt->pdp.page_directory[i]; +struct i915_page_table *pt;dma_addr_t addr;if (WARN_ON (!pd->page_table[j]))
@@ -747,8 +747,8 @@ static int gen8_ppgtt_setup_page_tables(struct i915_hw_ppgtt *ppgtt, const int pt) {dma_addr_t pt_addr; -struct i915_page_directory_entry *pdir = ppgtt->pdp.page_directory[pd]; -struct i915_page_table_entry *ptab = pdir->page_table[pt]; +struct i915_page_directory *pdir = ppgtt->pdp.page_directory[pd]; +struct i915_page_table *ptab = pdir->page_table[pt];struct page *p = ptab->page;int ret;
@@ -815,11 +815,11 @@ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size)* will never need to touch the PDEs again. */ for (i =0 ; i <GEN8_LEGACY_PDPES ; i++) { -struct i915_page_directory_entry *pd = ppgtt->pdp.page_directory[i]; +struct i915_page_directory *pd = ppgtt->pdp.page_directory[i];gen8_pde_t *pd_vaddr; pd_vaddr =kmap_atomic (ppgtt->pdp.page_directory[i]->page);for (j =0 ; j <I915_PDES ; j++) { -struct i915_page_table_entry *pt = pd->page_table[j]; +struct i915_page_table *pt = pd->page_table[j];dma_addr_t addr = pt->daddr; pd_vaddr[j] =gen8_pde_encode (ppgtt->base.dev, addr,I915_CACHE_LLC );
@@ -914,8 +914,8 @@ static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m) }/* Write pde (index) from the page directory @pd to the page table @pt */ -static void gen6_write_pde (struct i915_page_directory_entry *pd, -const int pde,struct i915_page_table_entry *pt) +static void gen6_write_pde (struct i915_page_directory *pd, +const int pde,struct i915_page_table *pt) {/* Caller needs to make sure the write completes if necessary */ struct i915_hw_ppgtt *ppgtt =
@@ -931,10 +931,10 @@ static void gen6_write_pde(struct i915_page_directory_entry *pd,/* Write all the page tables found in the ppgtt structure to incrementing page * directories. */ static void gen6_write_page_range (struct drm_i915_private *dev_priv, -struct i915_page_directory_entry *pd, +struct i915_page_directory *pd,uint32_t start,uint32_t length) { -struct i915_page_table_entry *pt; +struct i915_page_table *pt;uint32_t pde, temp;gen6_for_each_pde (pt, pd, start, length, temp, pde)
@@ -1169,7 +1169,7 @@ static inline void mark_tlbs_dirty(struct i915_hw_ppgtt *ppgtt) }static void gen6_initialize_pt (struct i915_address_space *vm, -struct i915_page_table_entry *pt) +struct i915_page_table *pt) {gen6_pte_t *pt_vaddr, scratch_pte;int i;
@@ -1195,7 +1195,7 @@ static int gen6_alloc_va_range(struct i915_address_space *vm,struct drm_i915_private *dev_priv = dev->dev_private;struct i915_hw_ppgtt *ppgtt = container_of(vm, struct i915_hw_ppgtt, base); -struct i915_page_table_entry *pt; +struct i915_page_table *pt;const uint32_t start_save = start, length_save = length;uint32_t pde, temp;int ret;
@@ -1263,7 +1263,7 @@ static int gen6_alloc_va_range(struct i915_address_space *vm, unwind_out:for_each_set_bit (pde, new_page_tables,I915_PDES ) { -struct i915_page_table_entry *pt = ppgtt->pd.page_table[pde]; +struct i915_page_table *pt = ppgtt->pd.page_table[pde]; ppgtt->pd.page_table[pde] = ppgtt->scratch_pt;unmap_and_free_pt (pt, vm->dev);
@@ -1278,7 +1278,7 @@ static void gen6_ppgtt_free(struct i915_hw_ppgtt *ppgtt)int i;for (i =0 ; i < ppgtt->num_pd_entries; i++) { -struct i915_page_table_entry *pt = ppgtt->pd.page_table[i]; +struct i915_page_table *pt = ppgtt->pd.page_table[i];if (pt != ppgtt->scratch_pt)unmap_and_free_pt (ppgtt->pd.page_table[i], ppgtt->base.dev);
@@ -1358,7 +1358,7 @@ static int gen6_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt)static void gen6_scratch_va_range (struct i915_hw_ppgtt *ppgtt,uint64_t start,uint64_t length) { -struct i915_page_table_entry *unused; +struct i915_page_table *unused;uint32_t pde, temp;gen6_for_each_pde (unused, &ppgtt->pd, start, length, temp, pde)
drivers/gpu/drm/i915/i915_gem_gtt.h +8 -8
@@ -206,26 +206,26 @@ struct i915_vma { u32 flags); }; -struct i915_page_table_entry { +struct i915_page_table {struct page *page;dma_addr_t daddr;unsigned long *used_ptes; }; -struct i915_page_directory_entry { +struct i915_page_directory {struct page *page;/* NULL for GEN6-GEN7 */ union {uint32_t pd_offset;dma_addr_t daddr; }; -struct i915_page_table_entry *page_table[I915_PDES ];/* PDEs */ +struct i915_page_table *page_table[I915_PDES ];/* PDEs */ }; -struct i915_page_directory_pointer_entry { +struct i915_page_directory_pointer {/* struct page *page; */ -struct i915_page_directory_entry *page_directory[GEN8_LEGACY_PDPES ]; +struct i915_page_directory *page_directory[GEN8_LEGACY_PDPES ]; };struct i915_address_space {
@@ -317,11 +317,11 @@ struct i915_hw_ppgtt {unsigned num_pd_entries;unsigned num_pd_pages;/* gen8+ */ union { -struct i915_page_directory_pointer_entry pdp; -struct i915_page_directory_entry pd; +struct i915_page_directory_pointer pdp; +struct i915_page_directory pd; }; -struct i915_page_table_entry *scratch_pt; +struct i915_page_table *scratch_pt;struct drm_i915_file_private *file_priv;
drivers/gpu/drm/i915/i915_trace.h +2 -2
@@ -221,7 +221,7 @@ DEFINE_EVENT(i915_page_table_entry, i915_page_table_entry_alloc, DECLARE_EVENT_CLASS(i915_page_table_entry_update, TP_PROTO(struct i915_address_space *vm, u32 pde, - struct i915_page_table_entry *pt, u32 first, u32 count, u32 bits), + struct i915_page_table *pt, u32 first, u32 count, u32 bits), TP_ARGS(vm, pde, pt, first, count, bits), TP_STRUCT__entry(
@@ -251,7 +251,7 @@ DECLARE_EVENT_CLASS(i915_page_table_entry_update,DEFINE_EVENT (i915_page_table_entry_update, i915_page_table_entry_map, TP_PROTO(struct i915_address_space *vm, u32 pde, - struct i915_page_table_entry *pt, u32 first, u32 count, u32 bits), + struct i915_page_table *pt, u32 first, u32 count, u32 bits),TP_ARGS (vm, pde, pt, first, count, bits) );
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH 2/13 drm/i915: Remove unnecessary gen8_ppgtt_unmap_pages
In reply to Michel ThierryWe are already unmapping them in gen8_ppgtt_free. This function became redundant since commit 06fda602dbca9c59d87db7da71192e4b54c9f5ff ("drm/i915: Create page table allocators").
--- drivers/gpu/drm/i915/i915_gem_gtt.c | 34 ---------------------------------- 1 file changed, 34 deletions(-)
drivers/gpu/drm/i915/i915_gem_gtt.c +0 -34
@@ -617,44 +617,11 @@ static void gen8_ppgtt_free(struct i915_hw_ppgtt *ppgtt) } } -static void gen8_ppgtt_unmap_pages (struct i915_hw_ppgtt *ppgtt) -{ -struct pci_dev *hwdev = ppgtt->base.dev->pdev; -int i, j; - -for (i =0 ; i < ppgtt->num_pd_pages; i++) { -/* TODO: In the future we'll support sparse mappings, so this -* will have to change. */ -if (!ppgtt->pdp.page_directory[i]->daddr) -continue ; - -pci_unmap_page (hwdev, ppgtt->pdp.page_directory[i]->daddr,PAGE_SIZE , -PCI_DMA_BIDIRECTIONAL ); - -for (j =0 ; j <I915_PDES ; j++) { -struct i915_page_directory *pd = ppgtt->pdp.page_directory[i]; -struct i915_page_table *pt; -dma_addr_t addr; - -if (WARN_ON (!pd->page_table[j])) -continue ; - - pt = pd->page_table[j]; - addr = pt->daddr; - -if (addr) -pci_unmap_page (hwdev, addr,PAGE_SIZE , -PCI_DMA_BIDIRECTIONAL ); - } - } -} -static void gen8_ppgtt_cleanup (struct i915_address_space *vm) {struct i915_hw_ppgtt *ppgtt = container_of(vm, struct i915_hw_ppgtt, base); -gen8_ppgtt_unmap_pages (ppgtt);gen8_ppgtt_free (ppgtt); }
@@ -851,7 +818,6 @@ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size)return 0 ; bail: -gen8_ppgtt_unmap_pages (ppgtt);gen8_ppgtt_free (ppgtt);return ret; }
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH 3/13 drm/i915/gen8: Initialize page tables
In reply to Michel ThierrySimilar to gen6, while setting up a page table, make all entries of the pt point to the scratch page before mapping; this is to be safe in case of out of bound access or proactive prefetch.
Systems without LLC require an explicit flush.
v2: Expanded commit text and fixed indentation (Mika)
--- drivers/gpu/drm/i915/i915_gem_gtt.c | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-)
drivers/gpu/drm/i915/i915_gem_gtt.c +21 -1
@@ -332,6 +332,24 @@ static void unmap_and_free_pt(struct i915_page_table *pt,kfree (pt); } +static void gen8_initialize_pt (struct i915_address_space *vm, +struct i915_page_table *pt) +{ +gen8_pte_t *pt_vaddr, scratch_pte; +int i; + + pt_vaddr =kmap_atomic (pt->page); + scratch_pte =gen8_pte_encode (vm->scratch.addr, +I915_CACHE_LLC , true); + +for (i =0 ; i <GEN8_PTES ; i++) + pt_vaddr[i] = scratch_pte; + +if (!HAS_LLC (vm->dev)) +drm_clflush_virt_range (pt_vaddr,PAGE_SIZE ); +kunmap_atomic (pt_vaddr); +} +static struct i915_page_table *alloc_pt_single (struct drm_device *dev) {struct i915_page_table *pt;
@@ -429,7 +447,7 @@ static struct i915_page_directory *alloc_pd_single(void)if (!pd)return ERR_PTR (-ENOMEM ); - pd->page =alloc_page (GFP_KERNEL | __GFP_ZERO); + pd->page =alloc_page (GFP_KERNEL );if (!pd->page) {kfree (pd);return ERR_PTR (-ENOMEM );
@@ -719,6 +737,8 @@ static int gen8_ppgtt_setup_page_tables(struct i915_hw_ppgtt *ppgtt,struct page *p = ptab->page;int ret; +gen8_initialize_pt (&ppgtt->base, ptab); + pt_addr =pci_map_page (ppgtt->base.dev->pdev, p,0 ,PAGE_SIZE ,PCI_DMA_BIDIRECTIONAL ); ret =pci_dma_mapping_error (ppgtt->base.dev->pdev, pt_addr);
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH 4/13 drm/i915/gen8: Add dynamic allocation macros and helper functions
In reply to Michel ThierrySimilar to gen6, we will use for_each_pde/for_each_pdpe and pte/pde/pdpe_index to iterate over these new structures.
v2: Match trace_i915_va_teardown params
v3: Multiple rebases.
v4: Updated to use unmap_and_free_pt.
v5: teardown_va_range logic no longer needed.
v6: Rebase after s/page_tables/page_table/.
v7: Renamed commit to match what it does now (it was "Use dynamic allocation idioms on free").
v8: Prevent (harmless) out of range access in gen8_for_each_pde and gen8_for_each_pdpe_e.
--- drivers/gpu/drm/i915/i915_gem_gtt.h | 51 +++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+)
drivers/gpu/drm/i915/i915_gem_gtt.h +51 -0
@@ -397,6 +397,57 @@ static inline uint32_t gen6_pde_index(uint32_t addr)return i915_pde_index (addr,GEN6_PDE_SHIFT ); } +/* Equivalent to the gen6 version, For each pde iterates over every pde +* between from start until start + length. On gen8+ it simply iterates +* over every page directory entry in a page directory. +*/ +#define gen8_for_each_pde (pt, pd, start, length, temp, iter) \ + for (iter = gen8_pde_index(start); \ + pt = (pd)->page_table[iter], length > 0 && iter < I915_PDES; \ + iter++, \ + temp = ALIGN(start+1, 1 << GEN8_PDE_SHIFT) - start, \ + temp = min(temp, length), \ + start += temp, length -= temp) + +#define gen8_for_each_pdpe (pd, pdp, start, length, temp, iter) \ + for (iter = gen8_pdpe_index(start); \ + pd = (pdp)->page_directory[iter], length > 0 && iter < GEN8_LEGACY_PDPES; \ + iter++, \ + temp = ALIGN(start+1, 1 << GEN8_PDPE_SHIFT) - start, \ + temp = min(temp, length), \ + start += temp, length -= temp) + +/* Clamp length to the next page_directory boundary */ +static inline uint64_t gen8_clamp_pd (uint64_t start,uint64_t length) +{ +uint64_t next_pd =ALIGN (start +1 ,1 <<GEN8_PDPE_SHIFT ); + +if (next_pd > (start + length)) +return length; + +return next_pd - start; +} + +static inline uint32_t gen8_pte_index (uint64_t address) +{ +return i915_pte_index (address,GEN8_PDE_SHIFT ); +} + +static inline uint32_t gen8_pde_index (uint64_t address) +{ +return i915_pde_index (address,GEN8_PDE_SHIFT ); +} + +static inline uint32_t gen8_pdpe_index (uint64_t address) +{ +return (address >>GEN8_PDPE_SHIFT ) &GEN8_PDPE_MASK ; +} + +static inline uint32_t gen8_pml4e_index (uint64_t address) +{ +BUG ();/* For 64B */ +} +int i915_gem_gtt_init (struct drm_device *dev);void i915_gem_init_global_gtt (struct drm_device *dev);void i915_global_gtt_cleanup (struct drm_device *dev);
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
Re: [PATCH 04/13] drm/i915/gen8: Add dynamic allocation macros and helper functions
drivers/gpu/drm/i915/i915_gem_gtt.h static inline uint32_t gen6_pde_index(uint32_t addr) · line 448
Quoted 3 lines · Michel Thierry
drivers/gpu/drm/i915/i915_gem_gtt.h +3
-0+static inline uint32_t gen8_pml4e_index (uint64_t address) +{ +BUG ();/* For 64B */
Please never use BUG where a WARN would do too. Fixed while applying. -Daniel
Signature · 7 lines
Daniel Vetter Software Engineer, Intel Corporation http://blog.ffwll.ch _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH 5/13 drm/i915/gen8: page directories rework allocation
In reply to Michel ThierryStart using gen8_for_each_pdpe macro to allocate the page directories.
Similar to PTs, while setting up a page directory, make all entries of the pd point to the scratch pd before mapping (and make all its entries point to the scratch page); this is to be safe in case of out of bound access or proactive prefetch. Systems without LLC require an explicit flush.
v2: Rebased after s/free_pt_*/unmap_and_free_pt/ change.
v3: Rebased after teardown va range logic was removed.
v4: Keep setting up all page directories for systems with less than 4GB of memory.
v5: Initialize PDs. (Mika)
v6: Initialize also the extra PDs from systems with less than 4GB of memory. (Mika)
--- drivers/gpu/drm/i915/i915_gem_gtt.c | 100 ++++++++++++++++++++++++++++++------ 1 file changed, 85 insertions(+), 15 deletions(-)
drivers/gpu/drm/i915/i915_gem_gtt.c +85 -15
@@ -606,6 +606,36 @@ static void gen8_ppgtt_insert_entries(struct i915_address_space *vm, } } +static void __gen8_do_map_pt (gen8_pde_t *const pde, +struct i915_page_table *pt, +struct drm_device *dev) +{ +gen8_pde_t entry = +gen8_pde_encode (dev, pt->daddr,I915_CACHE_LLC ); + *pde = entry; +} + +static void gen8_initialize_pd (struct i915_address_space *vm, +struct i915_page_directory *pd) +{ +struct i915_hw_ppgtt *ppgtt = + container_of(vm, struct i915_hw_ppgtt, base); +gen8_pde_t *page_directory; +struct i915_page_table *pt; +int i; + + page_directory =kmap_atomic (pd->page); + pt = ppgtt->scratch_pt; +for (i =0 ; i <I915_PDES ; i++) +/* Map the PDE to the page table */ +__gen8_do_map_pt (page_directory + i, pt, vm->dev); + +if (!HAS_LLC (vm->dev)) +drm_clflush_virt_range (page_directory,PAGE_SIZE ); + +kunmap_atomic (page_directory); +} +static void gen8_free_page_tables (struct i915_page_directory *pd,struct drm_device *dev) {int i;
@@ -633,6 +663,8 @@ static void gen8_ppgtt_free(struct i915_hw_ppgtt *ppgtt)gen8_free_page_tables (ppgtt->pdp.page_directory[i], ppgtt->base.dev);unmap_and_free_pd (ppgtt->pdp.page_directory[i]); } + +unmap_and_free_pt (ppgtt->scratch_pt, ppgtt->base.dev); }static void gen8_ppgtt_cleanup (struct i915_address_space *vm)
@@ -663,25 +695,55 @@ unwind_out:return -ENOMEM ; } -static int gen8_ppgtt_allocate_page_directories (struct i915_hw_ppgtt *ppgtt, -const int max_pdp) +static int gen8_ppgtt_alloc_page_directories (struct i915_page_directory_pointer *pdp, +uint64_t start, +uint64_t length) { -int i; +struct i915_hw_ppgtt *ppgtt = + container_of(pdp, struct i915_hw_ppgtt, pdp); +struct i915_page_directory *unused; +uint64_t temp; +uint32_t pdpe; + +/* FIXME: PPGTT container_of won't work for 64b */ +WARN_ON ((start + length) >0x800000000ULL ); + +gen8_for_each_pdpe (unused, pdp, start, length, temp, pdpe) { +WARN_ON (unused); + pdp->page_directory[pdpe] =alloc_pd_single (); +if (IS_ERR (ppgtt->pdp.page_directory[pdpe])) + goto unwind_out; + +gen8_initialize_pd (&ppgtt->base, + ppgtt->pdp.page_directory[pdpe]); + ppgtt->num_pd_pages++; + } -for (i =0 ; i < max_pdp; i++) { - ppgtt->pdp.page_directory[i] =alloc_pd_single (); -if (IS_ERR (ppgtt->pdp.page_directory[i])) +/* XXX: Still alloc all page directories in systems with less than +* 4GB of memory. This won't be needed after a subsequent patch. +*/ +while (ppgtt->num_pd_pages <GEN8_LEGACY_PDPES ) { + ppgtt->pdp.page_directory[ppgtt->num_pd_pages] =alloc_pd_single (); +if (IS_ERR (ppgtt->pdp.page_directory[ppgtt->num_pd_pages])) goto unwind_out; + +gen8_initialize_pd (&ppgtt->base, + ppgtt->pdp.page_directory[ppgtt->num_pd_pages]); + pdpe++; + ppgtt->num_pd_pages++; } - ppgtt->num_pd_pages = max_pdp;BUG_ON (ppgtt->num_pd_pages >GEN8_LEGACY_PDPES );return 0 ; unwind_out: -while (i--) -unmap_and_free_pd (ppgtt->pdp.page_directory[i]); +while (pdpe--) { +unmap_and_free_pd (ppgtt->pdp.page_directory[pdpe]); + ppgtt->num_pd_pages--; + } + +WARN_ON (ppgtt->num_pd_pages);return -ENOMEM ; }
@@ -691,7 +753,8 @@ static int gen8_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt, {int ret; - ret =gen8_ppgtt_allocate_page_directories (ppgtt, max_pdp); + ret =gen8_ppgtt_alloc_page_directories (&ppgtt->pdp, ppgtt->base.start, + ppgtt->base.total);if (ret)return ret;
@@ -769,6 +832,17 @@ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size)if (size % (1 <<30 ))DRM_INFO ("Pages will be wasted unless GTT size (%llu) is divisible by 1GB\n" , size); + ppgtt->base.start =0 ; +/* This is the area that we advertise as usable for the caller */ + ppgtt->base.total = max_pdp *I915_PDES *GEN8_PTES *PAGE_SIZE ; +WARN_ON (ppgtt->base.total ==0 ); + + ppgtt->scratch_pt =alloc_pt_single (ppgtt->base.dev); +if (IS_ERR (ppgtt->scratch_pt)) +return PTR_ERR (ppgtt->scratch_pt); + +gen8_initialize_pt (&ppgtt->base, ppgtt->scratch_pt); +/* 1. Do all our allocations for page directories and page tables. * We allocate more than was asked so that we can point the unused parts * to valid entries that point to scratch page. Dynamic page tables
@@ -794,7 +868,7 @@ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size) }/* -* 3. Map all the page directory entires to point to the page tables +* 3. Map all the page directory entries to point to the page tables * we've allocated. * * For now, the PPGTT helper functions all require that the PDEs are
@@ -820,10 +894,6 @@ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size) ppgtt->base.clear_range = gen8_ppgtt_clear_range; ppgtt->base.insert_entries = gen8_ppgtt_insert_entries; ppgtt->base.cleanup = gen8_ppgtt_cleanup; - ppgtt->base.start =0 ; - -/* This is the area that we advertise as usable for the caller */ - ppgtt->base.total = max_pdp *I915_PDES *GEN8_PTES *PAGE_SIZE ;/* Set all ptes to a valid scratch page. Also above requested space */ ppgtt->base.clear_range(&ppgtt->base, 0,
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH 6/13 drm/i915/gen8: pagetable allocation rework
In reply to Michel ThierryStart using gen8_for_each_pde macro to allocate page tables.
v2: teardown_va_range references removed.
v3: Rebase after s/page_tables/page_table/.
v4: Keep setting up page tables for all page directories in systems with less than 4GB of memory.
v5: Also initialize the page tables. (Mika)
v6: Initialize all page tables, including the extra ones from systems with less than 4GB of memory. (Mika)
--- drivers/gpu/drm/i915/i915_gem_gtt.c | 83 ++++++++++++++++++++++++++----------- 1 file changed, 59 insertions(+), 24 deletions(-)
drivers/gpu/drm/i915/i915_gem_gtt.c +59 -24
@@ -675,22 +675,41 @@ static void gen8_ppgtt_cleanup(struct i915_address_space *vm)gen8_ppgtt_free (ppgtt); } -static int gen8_ppgtt_allocate_page_tables (struct i915_hw_ppgtt *ppgtt) +static int gen8_ppgtt_alloc_pagetabs (struct i915_page_directory *pd, +uint64_t start, +uint64_t length, +struct i915_address_space *vm) { -int i, ret; +struct i915_page_table *unused; +uint64_t temp; +uint32_t pde; -for (i =0 ; i < ppgtt->num_pd_pages; i++) { - ret =alloc_pt_range (ppgtt->pdp.page_directory[i], -0 ,I915_PDES , ppgtt->base.dev); -if (ret) +gen8_for_each_pde (unused, pd, start, length, temp, pde) { +WARN_ON (unused); + pd->page_table[pde] =alloc_pt_single (vm->dev); +if (IS_ERR (pd->page_table[pde])) + goto unwind_out; + +gen8_initialize_pt (vm, pd->page_table[pde]); + } + +/* XXX: Still alloc all page tables in systems with less than +* 4GB of memory. This won't be needed after a subsequent patch. +*/ +while (pde <I915_PDES ) { + pd->page_table[pde] =alloc_pt_single (vm->dev); +if (IS_ERR (pd->page_table[pde])) goto unwind_out; + +gen8_initialize_pt (vm, pd->page_table[pde]); + pde++; }return 0 ; unwind_out: -while (i--) -gen8_free_page_tables (ppgtt->pdp.page_directory[i], ppgtt->base.dev); +while (pde--) +unmap_and_free_pt (pd->page_table[pde], vm->dev);return -ENOMEM ; }
@@ -749,20 +768,42 @@ unwind_out: }static int gen8_ppgtt_alloc (struct i915_hw_ppgtt *ppgtt, -const int max_pdp) +uint64_t start, +uint64_t length) { +struct i915_page_directory *pd; +uint64_t temp; +uint32_t pdpe;int ret; - ret =gen8_ppgtt_alloc_page_directories (&ppgtt->pdp, ppgtt->base.start, - ppgtt->base.total); + ret =gen8_ppgtt_alloc_page_directories (&ppgtt->pdp, start, length);if (ret)return ret; - ret =gen8_ppgtt_allocate_page_tables (ppgtt); -if (ret) - goto err_out; +gen8_for_each_pdpe (pd, &ppgtt->pdp, start, length, temp, pdpe) { + ret =gen8_ppgtt_alloc_pagetabs (pd, start, length, + &ppgtt->base); +if (ret) + goto err_out; + + ppgtt->num_pd_entries +=I915_PDES ; + } - ppgtt->num_pd_entries = max_pdp *I915_PDES ; +/* XXX: We allocated all page directories in systems with less than +* 4GB of memory. So initalize page tables of all PDPs. +* This won't be needed after the next patch. +*/ +while (pdpe <GEN8_LEGACY_PDPES ) { + ret =gen8_ppgtt_alloc_pagetabs (ppgtt->pdp.page_directory[pdpe], start, length, + &ppgtt->base); +if (ret) + goto err_out; + + ppgtt->num_pd_entries +=I915_PDES ; + pdpe++; + } + +WARN_ON (pdpe > ppgtt->num_pd_pages);return 0 ;
@@ -833,9 +874,7 @@ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size)DRM_INFO ("Pages will be wasted unless GTT size (%llu) is divisible by 1GB\n" , size); ppgtt->base.start =0 ; -/* This is the area that we advertise as usable for the caller */ - ppgtt->base.total = max_pdp *I915_PDES *GEN8_PTES *PAGE_SIZE ; -WARN_ON (ppgtt->base.total ==0 ); + ppgtt->base.total = size; ppgtt->scratch_pt =alloc_pt_single (ppgtt->base.dev);if (IS_ERR (ppgtt->scratch_pt))
@@ -843,12 +882,8 @@ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size)gen8_initialize_pt (&ppgtt->base, ppgtt->scratch_pt); -/* 1. Do all our allocations for page directories and page tables. -* We allocate more than was asked so that we can point the unused parts -* to valid entries that point to scratch page. Dynamic page tables -* will fix this eventually. -*/ - ret =gen8_ppgtt_alloc (ppgtt,GEN8_LEGACY_PDPES ); +/* 1. Do all our allocations for page directories and page tables. */ + ret =gen8_ppgtt_alloc (ppgtt, ppgtt->base.start, ppgtt->base.total);if (ret)return ret;
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH 7/13 drm/i915/gen8: Update pdp switch and point unused PDPs to scratch page
In reply to Michel ThierryOne important part of this patch is we now write a scratch page directory into any unused PDP descriptors. This matters for 2 reasons, first, we're not allowed to just use 0, or an invalid pointer, and second, we must wipe out any previous contents from the last context.
The latter point only matters with full PPGTT. The former point only effect platforms with less than 4GB memory.
v2: Updated commit message to point that we must set unused PDPs to the scratch page.
v3: Unmap scratch_pd in gen8_ppgtt_free.
v4: Initialize scratch_pd. (Mika)
--- drivers/gpu/drm/i915/i915_gem_gtt.c | 32 +++++++++++++++++++++----------- drivers/gpu/drm/i915/i915_gem_gtt.h | 1 + 2 files changed, 22 insertions(+), 11 deletions(-)
drivers/gpu/drm/i915/i915_gem_gtt.c +21 -11
@@ -457,8 +457,9 @@ static struct i915_page_directory *alloc_pd_single(void) }/* Broadwell Page Directory Pointer Descriptors */ -static int gen8_write_pdp (struct intel_engine_cs *ring,unsigned entry, -uint64_t val) +static int gen8_write_pdp (struct intel_engine_cs *ring, +unsigned entry, +dma_addr_t addr) {int ret;
@@ -470,10 +471,10 @@ static int gen8_write_pdp(struct intel_engine_cs *ring, unsigned entry,intel_ring_emit (ring,MI_LOAD_REGISTER_IMM (1 ));intel_ring_emit (ring,GEN8_RING_PDP_UDW (ring, entry)); -intel_ring_emit (ring, (u32)(val >>32 )); +intel_ring_emit (ring,upper_32_bits (addr));intel_ring_emit (ring,MI_LOAD_REGISTER_IMM (1 ));intel_ring_emit (ring,GEN8_RING_PDP_LDW (ring, entry)); -intel_ring_emit (ring, (u32)(val)); +intel_ring_emit (ring,lower_32_bits (addr));intel_ring_advance (ring);return 0 ;
@@ -484,12 +485,12 @@ static int gen8_mm_switch(struct i915_hw_ppgtt *ppgtt, {int i, ret; -/* bit of a hack to find the actual last used pd */ -int used_pd = ppgtt->num_pd_entries /I915_PDES ; - -for (i = used_pd -1 ; i >=0 ; i--) { -dma_addr_t addr = ppgtt->pdp.page_directory[i]->daddr; - ret =gen8_write_pdp (ring, i, addr); +for (i =GEN8_LEGACY_PDPES -1 ; i >=0 ; i--) { +struct i915_page_directory *pd = ppgtt->pdp.page_directory[i]; +dma_addr_t pd_daddr = pd ? pd->daddr : ppgtt->scratch_pd->daddr; +/* The page directory might be NULL, but we need to clear out +* whatever the previous context might have used. */ + ret =gen8_write_pdp (ring, i, pd_daddr);if (ret)return ret; }
@@ -664,6 +665,7 @@ static void gen8_ppgtt_free(struct i915_hw_ppgtt *ppgtt)unmap_and_free_pd (ppgtt->pdp.page_directory[i]); } +unmap_and_free_pd (ppgtt->scratch_pd);unmap_and_free_pt (ppgtt->scratch_pt, ppgtt->base.dev); }
@@ -880,12 +882,20 @@ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size)if (IS_ERR (ppgtt->scratch_pt))return PTR_ERR (ppgtt->scratch_pt); + ppgtt->scratch_pd =alloc_pd_single (); +if (IS_ERR (ppgtt->scratch_pd)) +return PTR_ERR (ppgtt->scratch_pd); +gen8_initialize_pt (&ppgtt->base, ppgtt->scratch_pt); +gen8_initialize_pd (&ppgtt->base, ppgtt->scratch_pd);/* 1. Do all our allocations for page directories and page tables. */ ret =gen8_ppgtt_alloc (ppgtt, ppgtt->base.start, ppgtt->base.total); -if (ret) +if (ret) { +unmap_and_free_pd (ppgtt->scratch_pd); +unmap_and_free_pt (ppgtt->scratch_pt, ppgtt->base.dev);return ret; + }/* * 2. Create DMA mappings for the page directories and page tables.
drivers/gpu/drm/i915/i915_gem_gtt.h +1 -0
@@ -322,6 +322,7 @@ struct i915_hw_ppgtt { };struct i915_page_table *scratch_pt; +struct i915_page_directory *scratch_pd;struct drm_i915_file_private *file_priv;
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH 8/13 drm/i915: num_pd_pages/num_pd_entries isn't useful
In reply to Michel ThierryThese values are never quite useful for dynamic allocations of the page tables. Getting rid of them will help prevent later confusion.
v2: Updated to use unmap_and_free_pd functions.
v3: Updated gen8_ppgtt_free after teardown logic was removed.
v4: Rebase after s/page_tables/page_table/.
v5: Keep allocating all page directories in GEN8+ systems with less than 4GB of memory. Updated gen6_for_all_pdes.
v6: Prevent (harmless) out of range access in gen6_for_all_pdes.
--- drivers/gpu/drm/i915/i915_debugfs.c | 2 -- drivers/gpu/drm/i915/i915_gem_gtt.c | 68 ++++++++++--------------------------- drivers/gpu/drm/i915/i915_gem_gtt.h | 7 ++-- 3 files changed, 23 insertions(+), 54 deletions(-)
drivers/gpu/drm/i915/i915_debugfs.c +0 -2
@@ -2166,8 +2166,6 @@ static void gen8_ppgtt_info(struct seq_file *m, struct drm_device *dev)if (!ppgtt)return ; -seq_printf (m,"Page directories: %d\n" , ppgtt->num_pd_pages); -seq_printf (m,"Page tables: %d\n" , ppgtt->num_pd_entries);for_each_ring (ring, dev_priv, unused) {seq_printf (m,"%s\n" , ring->name);for (i =0 ; i <4 ; i++) {
drivers/gpu/drm/i915/i915_gem_gtt.c +18 -50
163 lines not on this page · raw message
drivers/gpu/drm/i915/i915_gem_gtt.h +5 -2
@@ -314,8 +314,6 @@ struct i915_hw_ppgtt {struct kref ref;struct drm_mm_node node;unsigned long pd_dirty_rings; -unsigned num_pd_entries; -unsigned num_pd_pages;/* gen8+ */ union {struct i915_page_directory_pointer pdp;struct i915_page_directory pd;
@@ -350,6 +348,11 @@ struct i915_hw_ppgtt { temp = min_t(unsigned, temp, length), \ start += temp, length -= temp) +#define gen6_for_all_pdes (pt, ppgtt, iter) \ + for (iter = 0; \ + pt = ppgtt->pd.page_table[iter], iter < I915_PDES; \ + iter++) +static inline uint32_t i915_pte_index (uint64_t address,uint32_t pde_shift) {const uint32_t mask =NUM_PTE (pde_shift) -1 ;
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH 9/13 drm/i915: Extract PPGTT param from page_directory alloc
In reply to Michel ThierryThis will be useful for when we move to 48b addressing, and the PDP isn't the root of the page table structure.
v2: Rebase after changes for Gen8+ systems with less than 4GB of memory.
v3: Rebase after Mika's code review.
--- drivers/gpu/drm/i915/i915_gem_gtt.c | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-)
drivers/gpu/drm/i915/i915_gem_gtt.c +7 -8
@@ -716,12 +716,11 @@ unwind_out:return -ENOMEM ; } -static int gen8_ppgtt_alloc_page_directories (struct i915_page_directory_pointer *pdp, +static int gen8_ppgtt_alloc_page_directories (struct i915_hw_ppgtt *ppgtt, +struct i915_page_directory_pointer *pdp,uint64_t start,uint64_t length) { -struct i915_hw_ppgtt *ppgtt = - container_of(pdp, struct i915_hw_ppgtt, pdp);struct i915_page_directory *unused;uint64_t temp;uint32_t pdpe;
@@ -732,7 +731,7 @@ static int gen8_ppgtt_alloc_page_directories(struct i915_page_directory_pointergen8_for_each_pdpe (unused, pdp, start, length, temp, pdpe) {WARN_ON (unused); pdp->page_directory[pdpe] =alloc_pd_single (); -if (IS_ERR (ppgtt->pdp.page_directory[pdpe])) +if (IS_ERR (pdp->page_directory[pdpe])) goto unwind_out; gen8_initialize_pd(&ppgtt->base,
@@ -743,8 +742,8 @@ static int gen8_ppgtt_alloc_page_directories(struct i915_page_directory_pointer* 4GB of memory. This won't be needed after a subsequent patch. */ while (pdpe <GEN8_LEGACY_PDPES ) { - ppgtt->pdp.page_directory[pdpe] =alloc_pd_single (); -if (IS_ERR (ppgtt->pdp.page_directory[pdpe])) + pdp->page_directory[pdpe] =alloc_pd_single (); +if (IS_ERR (pdp->page_directory[pdpe])) goto unwind_out; gen8_initialize_pd(&ppgtt->base,
@@ -756,7 +755,7 @@ static int gen8_ppgtt_alloc_page_directories(struct i915_page_directory_pointer unwind_out:while (pdpe--) -unmap_and_free_pd (ppgtt->pdp.page_directory[pdpe]); +unmap_and_free_pd (pdp->page_directory[pdpe]);return -ENOMEM ; }
@@ -770,7 +769,7 @@ static int gen8_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt,uint32_t pdpe;int ret; - ret =gen8_ppgtt_alloc_page_directories (&ppgtt->pdp, start, length); + ret =gen8_ppgtt_alloc_page_directories (ppgtt, &ppgtt->pdp, start, length);if (ret)return ret;
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH 10/13 drm/i915/gen8: Split out mappings
In reply to Michel ThierryWhen we do dynamic page table allocations for gen8, we'll need to have more control over how and when we map page tables, similar to gen6. In particular, DMA mappings for page directories/tables occur at allocation time.
This patch adds the functionality and calls it at init, which should have no functional change.
The PDPEs are still a special case for now. We'll need a function for that in the future as well.
v2: Handle renamed unmap_and_free_page functions.
v3: Updated after teardown_va logic was removed.
v4: Rebase after s/page_tables/page_table/.
v5: No longer allocate all PDPs in GEN8+ systems with less than 4GB of memory, and update populate_lr_context to handle this new case (proper tracking will be added later in the patch series).
v6: Assign lrc page directory pointer addresses using a macro. (Mika)
--- drivers/gpu/drm/i915/i915_gem_gtt.c | 224 +++++++++++------------------------- drivers/gpu/drm/i915/intel_lrc.c | 25 ++-- 2 files changed, 86 insertions(+), 163 deletions(-)
drivers/gpu/drm/i915/i915_gem_gtt.c +69 -155
@@ -333,7 +333,7 @@ static void unmap_and_free_pt(struct i915_page_table *pt, }static void gen8_initialize_pt (struct i915_address_space *vm, -struct i915_page_table *pt) +struct i915_page_table *pt) {gen8_pte_t *pt_vaddr, scratch_pte;int i;
@@ -431,17 +431,20 @@ err_out:return ret; } -static void unmap_and_free_pd (struct i915_page_directory *pd) +static void unmap_and_free_pd (struct i915_page_directory *pd, +struct drm_device *dev) {if (pd->page) { +i915_dma_unmap_single (pd, dev);__free_page (pd->page);kfree (pd); } } -static struct i915_page_directory *alloc_pd_single (void ) +static struct i915_page_directory *alloc_pd_single (struct drm_device *dev) {struct i915_page_directory *pd; +int ret; pd =kzalloc (sizeof (*pd),GFP_KERNEL );if (!pd)
@@ -453,6 +456,13 @@ static struct i915_page_directory *alloc_pd_single(void)return ERR_PTR (-ENOMEM ); } + ret =i915_dma_map_single (pd, dev); +if (ret) { +__free_page (pd->page); +kfree (pd); +return ERR_PTR (ret); + } +return pd; }
@@ -637,6 +647,27 @@ static void gen8_initialize_pd(struct i915_address_space *vm,kunmap_atomic (page_directory); } +/* It's likely we'll map more than one pagetable at a time. This function will +* save us unnecessary kmap calls, but do no more functionally than multiple +* calls to map_pt. */ +static void gen8_map_pagetable_range (struct i915_page_directory *pd, +uint64_t start, +uint64_t length, +struct drm_device *dev) +{ +gen8_pde_t *page_directory =kmap_atomic (pd->page); +struct i915_page_table *pt; +uint64_t temp, pde; + +gen8_for_each_pde (pt, pd, start, length, temp, pde) +__gen8_do_map_pt (page_directory + pde, pt, dev); + +if (!HAS_LLC (dev)) +drm_clflush_virt_range (page_directory,PAGE_SIZE ); + +kunmap_atomic (page_directory); +} +static void gen8_free_page_tables (struct i915_page_directory *pd,struct drm_device *dev) {int i;
@@ -662,10 +693,10 @@ static void gen8_ppgtt_free(struct i915_hw_ppgtt *ppgtt)continue ;gen8_free_page_tables (ppgtt->pdp.page_directory[i], ppgtt->base.dev); -unmap_and_free_pd (ppgtt->pdp.page_directory[i]); +unmap_and_free_pd (ppgtt->pdp.page_directory[i], ppgtt->base.dev); } -unmap_and_free_pd (ppgtt->scratch_pd); +unmap_and_free_pd (ppgtt->scratch_pd, ppgtt->base.dev);unmap_and_free_pt (ppgtt->scratch_pt, ppgtt->base.dev); }
@@ -677,41 +708,30 @@ static void gen8_ppgtt_cleanup(struct i915_address_space *vm)gen8_ppgtt_free (ppgtt); } -static int gen8_ppgtt_alloc_pagetabs (struct i915_page_directory *pd, +static int gen8_ppgtt_alloc_pagetabs (struct i915_hw_ppgtt *ppgtt, +struct i915_page_directory *pd,uint64_t start, -uint64_t length, -struct i915_address_space *vm) +uint64_t length) { +struct drm_device *dev = ppgtt->base.dev;struct i915_page_table *unused;uint64_t temp;uint32_t pde;gen8_for_each_pde (unused, pd, start, length, temp, pde) {WARN_ON (unused); - pd->page_table[pde] =alloc_pt_single (vm->dev); + pd->page_table[pde] =alloc_pt_single (dev);if (IS_ERR (pd->page_table[pde])) goto unwind_out; -gen8_initialize_pt (vm, pd->page_table[pde]); - } - -/* XXX: Still alloc all page tables in systems with less than -* 4GB of memory. This won't be needed after a subsequent patch. -*/ -while (pde <I915_PDES ) { - pd->page_table[pde] =alloc_pt_single (vm->dev); -if (IS_ERR (pd->page_table[pde])) - goto unwind_out; - -gen8_initialize_pt (vm, pd->page_table[pde]); - pde++; +gen8_initialize_pt (&ppgtt->base, pd->page_table[pde]); }return 0 ; unwind_out:while (pde--) -unmap_and_free_pt (pd->page_table[pde], vm->dev); +unmap_and_free_pt (pd->page_table[pde], dev);return -ENOMEM ; }
@@ -721,6 +741,7 @@ static int gen8_ppgtt_alloc_page_directories(struct i915_hw_ppgtt *ppgtt, uint64_t start, uint64_t length) { +struct drm_device *dev = ppgtt->base.dev;struct i915_page_directory *unused;uint64_t temp;uint32_t pdpe;
@@ -730,40 +751,29 @@ static int gen8_ppgtt_alloc_page_directories(struct i915_hw_ppgtt *ppgtt,gen8_for_each_pdpe (unused, pdp, start, length, temp, pdpe) {WARN_ON (unused); - pdp->page_directory[pdpe] =alloc_pd_single (); -if (IS_ERR (pdp->page_directory[pdpe])) - goto unwind_out; - -gen8_initialize_pd (&ppgtt->base, - ppgtt->pdp.page_directory[pdpe]); - } - -/* XXX: Still alloc all page directories in systems with less than -* 4GB of memory. This won't be needed after a subsequent patch. -*/ -while (pdpe <GEN8_LEGACY_PDPES ) { - pdp->page_directory[pdpe] =alloc_pd_single (); + pdp->page_directory[pdpe] =alloc_pd_single (dev);if (IS_ERR (pdp->page_directory[pdpe])) goto unwind_out;gen8_initialize_pd (&ppgtt->base, ppgtt->pdp.page_directory[pdpe]); - pdpe++; }return 0 ; unwind_out:while (pdpe--) -unmap_and_free_pd (pdp->page_directory[pdpe]); +unmap_and_free_pd (pdp->page_directory[pdpe], dev);return -ENOMEM ; } -static int gen8_ppgtt_alloc (struct i915_hw_ppgtt *ppgtt, -uint64_t start, -uint64_t length) +static int gen8_alloc_va_range (struct i915_address_space *vm, +uint64_t start, +uint64_t length) { +struct i915_hw_ppgtt *ppgtt = + container_of(vm, struct i915_hw_ppgtt, base);struct i915_page_directory *pd;uint64_t temp;uint32_t pdpe;
@@ -774,23 +784,9 @@ static int gen8_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt,return ret;gen8_for_each_pdpe (pd, &ppgtt->pdp, start, length, temp, pdpe) { - ret =gen8_ppgtt_alloc_pagetabs (pd, start, length, - &ppgtt->base); -if (ret) - goto err_out; - } - -/* XXX: We allocated all page directories in systems with less than -* 4GB of memory. So initalize page tables of all PDPs. -* This won't be needed after the next patch. -*/ -while (pdpe <GEN8_LEGACY_PDPES ) { - ret =gen8_ppgtt_alloc_pagetabs (ppgtt->pdp.page_directory[pdpe], start, length, - &ppgtt->base); + ret =gen8_ppgtt_alloc_pagetabs (ppgtt, pd, start, length);if (ret) goto err_out; - - pdpe++; }return 0 ;
@@ -800,136 +796,54 @@ err_out:return ret; } -static int gen8_ppgtt_setup_page_directories (struct i915_hw_ppgtt *ppgtt, -const int pd) -{ -dma_addr_t pd_addr; -int ret; - - pd_addr =pci_map_page (ppgtt->base.dev->pdev, - ppgtt->pdp.page_directory[pd]->page,0 , -PAGE_SIZE ,PCI_DMA_BIDIRECTIONAL ); - - ret =pci_dma_mapping_error (ppgtt->base.dev->pdev, pd_addr); -if (ret) -return ret; - - ppgtt->pdp.page_directory[pd]->daddr = pd_addr; - -return 0 ; -} - -static int gen8_ppgtt_setup_page_tables (struct i915_hw_ppgtt *ppgtt, -const int pd, -const int pt) -{ -dma_addr_t pt_addr; -struct i915_page_directory *pdir = ppgtt->pdp.page_directory[pd]; -struct i915_page_table *ptab = pdir->page_table[pt]; -struct page *p = ptab->page; -int ret; - -gen8_initialize_pt (&ppgtt->base, ptab); - - pt_addr =pci_map_page (ppgtt->base.dev->pdev, - p,0 ,PAGE_SIZE ,PCI_DMA_BIDIRECTIONAL ); - ret =pci_dma_mapping_error (ppgtt->base.dev->pdev, pt_addr); -if (ret) -return ret; - - ptab->daddr = pt_addr; - -return 0 ; -} -/* * GEN8 legacy ppgtt programming is accomplished through a max 4 PDP registers * with a net effect resembling a 2-level page table in normal x86 terms. Each * PDP represents 1GB of memory 4 * 512 * 512 * 4096 = 4GB legacy 32b address * space. * -* FIXME: split allocation into smaller pieces. For now we only ever do this -* once, but with full PPGTT, the multiple contiguous allocations will be bad. -* TODO: Do something with the size parameter */ static int gen8_ppgtt_init (struct i915_hw_ppgtt *ppgtt,uint64_t size) { -int i, j, ret; - -if (size % (1 <<30 )) -DRM_INFO ("Pages will be wasted unless GTT size (%llu) is divisible by 1GB\n" , size); +struct i915_page_directory *pd; +uint64_t temp, start =0 ; +const uint64_t orig_length = size; +uint32_t pdpe; +int ret; ppgtt->base.start =0 ; ppgtt->base.total = size; + ppgtt->base.clear_range = gen8_ppgtt_clear_range; + ppgtt->base.insert_entries = gen8_ppgtt_insert_entries; + ppgtt->base.cleanup = gen8_ppgtt_cleanup; + ppgtt->switch_mm = gen8_mm_switch; ppgtt->scratch_pt =alloc_pt_single (ppgtt->base.dev);if (IS_ERR (ppgtt->scratch_pt))return PTR_ERR (ppgtt->scratch_pt); - ppgtt->scratch_pd =alloc_pd_single (); + ppgtt->scratch_pd =alloc_pd_single (ppgtt->base.dev);if (IS_ERR (ppgtt->scratch_pd))return PTR_ERR (ppgtt->scratch_pd);gen8_initialize_pt (&ppgtt->base, ppgtt->scratch_pt);gen8_initialize_pd (&ppgtt->base, ppgtt->scratch_pd); -/* 1. Do all our allocations for page directories and page tables. */ - ret =gen8_ppgtt_alloc (ppgtt, ppgtt->base.start, ppgtt->base.total); + ret =gen8_alloc_va_range (&ppgtt->base, start, size);if (ret) { -unmap_and_free_pd (ppgtt->scratch_pd); +unmap_and_free_pd (ppgtt->scratch_pd, ppgtt->base.dev);unmap_and_free_pt (ppgtt->scratch_pt, ppgtt->base.dev);return ret; } -/* -* 2. Create DMA mappings for the page directories and page tables. -*/ -for (i =0 ; i <GEN8_LEGACY_PDPES ; i++) { - ret =gen8_ppgtt_setup_page_directories (ppgtt, i); -if (ret) - goto bail; - -for (j =0 ; j <I915_PDES ; j++) { - ret =gen8_ppgtt_setup_page_tables (ppgtt, i, j); -if (ret) - goto bail; - } - } - -/* -* 3. Map all the page directory entries to point to the page tables -* we've allocated. -* -* For now, the PPGTT helper functions all require that the PDEs are -* plugged in correctly. So we do that now/here. For aliasing PPGTT, we -* will never need to touch the PDEs again. -*/ -for (i =0 ; i <GEN8_LEGACY_PDPES ; i++) { -struct i915_page_directory *pd = ppgtt->pdp.page_directory[i]; -gen8_pde_t *pd_vaddr; - pd_vaddr =kmap_atomic (ppgtt->pdp.page_directory[i]->page); -for (j =0 ; j <I915_PDES ; j++) { -struct i915_page_table *pt = pd->page_table[j]; -dma_addr_t addr = pt->daddr; - pd_vaddr[j] =gen8_pde_encode (ppgtt->base.dev, addr, -I915_CACHE_LLC ); - } -if (!HAS_LLC (ppgtt->base.dev)) -drm_clflush_virt_range (pd_vaddr,PAGE_SIZE ); -kunmap_atomic (pd_vaddr); - } + start =0 ; + size = orig_length; - ppgtt->switch_mm = gen8_mm_switch; - ppgtt->base.clear_range = gen8_ppgtt_clear_range; - ppgtt->base.insert_entries = gen8_ppgtt_insert_entries; - ppgtt->base.cleanup = gen8_ppgtt_cleanup; +gen8_for_each_pdpe (pd, &ppgtt->pdp, start, size, temp, pdpe) +gen8_map_pagetable_range (pd, start, size, ppgtt->base.dev); ppgtt->base.clear_range (&ppgtt->base,0 , ppgtt->base.total, true);return 0 ; - -bail: -gen8_ppgtt_free (ppgtt); -return ret; }static void gen6_dump_ppgtt (struct i915_hw_ppgtt *ppgtt,struct seq_file *m)
@@ -1354,7 +1268,7 @@ static void gen6_ppgtt_free(struct i915_hw_ppgtt *ppgtt) }unmap_and_free_pt (ppgtt->scratch_pt, ppgtt->base.dev); -unmap_and_free_pd (&ppgtt->pd); +unmap_and_free_pd (&ppgtt->pd, ppgtt->base.dev); }static void gen6_ppgtt_cleanup (struct i915_address_space *vm)
drivers/gpu/drm/i915/intel_lrc.c +17 -8
@@ -188,6 +188,15 @@#define GEN8_CTX_FORCE_RESTORE (1<<2)#define GEN8_CTX_L3LLC_COHERENT (1<<5)#define GEN8_CTX_PRIVILEGE (1<<8) + +#define ASSIGN_CTX_PDP (ppgtt, reg_state, n) { \ + const u64 _addr = ppgtt->pdp.page_directory[n] ? \ + ppgtt->pdp.page_directory[n]->daddr : \ + ppgtt->scratch_pd->daddr; \ + reg_state[CTX_PDP ## n ## _UDW+1] = upper_32_bits(_addr); \ + reg_state[CTX_PDP ## n ## _LDW+1] = lower_32_bits(_addr); \ +} +enum {ADVANCED_CONTEXT =0 ,LEGACY_CONTEXT ,
@@ -1776,14 +1785,14 @@ populate_lr_context(struct intel_context *ctx, struct drm_i915_gem_object *ctx_o reg_state[CTX_PDP1_LDW ] =GEN8_RING_PDP_LDW (ring,1 ); reg_state[CTX_PDP0_UDW ] =GEN8_RING_PDP_UDW (ring,0 ); reg_state[CTX_PDP0_LDW ] =GEN8_RING_PDP_LDW (ring,0 ); - reg_state[CTX_PDP3_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[3 ]->daddr); - reg_state[CTX_PDP3_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[3 ]->daddr); - reg_state[CTX_PDP2_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[2 ]->daddr); - reg_state[CTX_PDP2_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[2 ]->daddr); - reg_state[CTX_PDP1_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[1 ]->daddr); - reg_state[CTX_PDP1_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[1 ]->daddr); - reg_state[CTX_PDP0_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[0 ]->daddr); - reg_state[CTX_PDP0_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[0 ]->daddr); +/* XXX: Systems with less than 4GB of memory do not have +* all PDPs. Proper PDP tracking will be added in a +* subsequent patch. +*/ +ASSIGN_CTX_PDP (ppgtt, reg_state,3 ); +ASSIGN_CTX_PDP (ppgtt, reg_state,2 ); +ASSIGN_CTX_PDP (ppgtt, reg_state,1 ); +ASSIGN_CTX_PDP (ppgtt, reg_state,0 );if (ring->id ==RCS ) { reg_state[CTX_LRI_HEADER_2 ] =MI_LOAD_REGISTER_IMM (1 ); reg_state[CTX_R_PWR_CLK_STATE ] =GEN8_R_PWR_CLK_STATE ;
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH 11/13 drm/i915/gen8: begin bitmap tracking
In reply to Michel ThierryLike with gen6/7, we can enable bitmap tracking with all the preallocations to make sure things actually don't blow up.
v2: Rebased to match changes from previous patches.
v3: Without teardown logic, rely on used_pdpes and used_pdes when freeing page tables.
v4: Rebased after s/page_tables/page_table/.
v5: Rebased after page table generalizations.
--- drivers/gpu/drm/i915/i915_gem_gtt.c | 59 +++++++++++++++++++++++++++++-------- drivers/gpu/drm/i915/i915_gem_gtt.h | 7 +++++ 2 files changed, 54 insertions(+), 12 deletions(-)
drivers/gpu/drm/i915/i915_gem_gtt.c +47 -12
@@ -437,6 +437,7 @@ static void unmap_and_free_pd(struct i915_page_directory *pd,if (pd->page) {i915_dma_unmap_single (pd, dev);__free_page (pd->page); +kfree (pd->used_pdes);kfree (pd); } }
@@ -444,26 +445,35 @@ static void unmap_and_free_pd(struct i915_page_directory *pd,static struct i915_page_directory *alloc_pd_single (struct drm_device *dev) {struct i915_page_directory *pd; -int ret; +int ret = -ENOMEM ; pd =kzalloc (sizeof (*pd),GFP_KERNEL );if (!pd)return ERR_PTR (-ENOMEM ); + pd->used_pdes =kcalloc (BITS_TO_LONGS (I915_PDES ), +sizeof (*pd->used_pdes),GFP_KERNEL ); +if (!pd->used_pdes) + goto free_pd; + pd->page =alloc_page (GFP_KERNEL ); -if (!pd->page) { -kfree (pd); -return ERR_PTR (-ENOMEM ); - } +if (!pd->page) + goto free_bitmap; ret =i915_dma_map_single (pd, dev); -if (ret) { -__free_page (pd->page); -kfree (pd); -return ERR_PTR (ret); - } +if (ret) + goto free_page;return pd; + +free_page: +__free_page (pd->page); +free_bitmap: +kfree (pd->used_pdes); +free_pd: +kfree (pd); + +return ERR_PTR (ret); }/* Broadwell Page Directory Pointer Descriptors */
@@ -675,7 +685,7 @@ static void gen8_free_page_tables(struct i915_page_directory *pd, struct drm_devif (!pd->page)return ; -for (i =0 ; i <I915_PDES ; i++) { +for_each_set_bit (i, pd->used_pdes,I915_PDES ) {if (WARN_ON (!pd->page_table[i]))continue ;
@@ -688,7 +698,7 @@ static void gen8_ppgtt_free(struct i915_hw_ppgtt *ppgtt) {int i; -for (i =0 ; i <GEN8_LEGACY_PDPES ; i++) { +for_each_set_bit (i, ppgtt->pdp.used_pdpes,GEN8_LEGACY_PDPES ) {if (WARN_ON (!ppgtt->pdp.page_directory[i]))continue ;
@@ -752,6 +762,7 @@ static int gen8_ppgtt_alloc_page_directories(struct i915_hw_ppgtt *ppgtt,gen8_for_each_pdpe (unused, pdp, start, length, temp, pdpe) {WARN_ON (unused); pdp->page_directory[pdpe] =alloc_pd_single (dev); +if (IS_ERR (pdp->page_directory[pdpe])) goto unwind_out;
@@ -775,10 +786,13 @@ static int gen8_alloc_va_range(struct i915_address_space *vm,struct i915_hw_ppgtt *ppgtt = container_of(vm, struct i915_hw_ppgtt, base);struct i915_page_directory *pd; +const uint64_t orig_start = start; +const uint64_t orig_length = length;uint64_t temp;uint32_t pdpe;int ret; +/* Do the allocations first so we can easily bail out */ ret =gen8_ppgtt_alloc_page_directories (ppgtt, &ppgtt->pdp, start, length);if (ret)return ret;
@@ -789,6 +803,27 @@ static int gen8_alloc_va_range(struct i915_address_space *vm, goto err_out; } +/* Now mark everything we've touched as used. This doesn't allow for +* robust error checking, but it makes the code a hell of a lot simpler. +*/ + start = orig_start; + length = orig_length; + +gen8_for_each_pdpe (pd, &ppgtt->pdp, start, length, temp, pdpe) { +struct i915_page_table *pt; +uint64_t pd_len =gen8_clamp_pd (start, length); +uint64_t pd_start = start; +uint32_t pde; + +gen8_for_each_pde (pt, &ppgtt->pd, pd_start, pd_len, temp, pde) { +bitmap_set (pd->page_table[pde]->used_ptes, +gen8_pte_index (start), +gen8_pte_count (start, length)); +set_bit (pde, pd->used_pdes); + } +set_bit (pdpe, ppgtt->pdp.used_pdpes); + } +return 0 ; err_out:
drivers/gpu/drm/i915/i915_gem_gtt.h +7 -0
@@ -220,11 +220,13 @@ struct i915_page_directory {dma_addr_t daddr; }; +unsigned long *used_pdes;struct i915_page_table *page_table[I915_PDES ];/* PDEs */ };struct i915_page_directory_pointer {/* struct page *page; */ + DECLARE_BITMAP(used_pdpes, GEN8_LEGACY_PDPES);struct i915_page_directory *page_directory[GEN8_LEGACY_PDPES ]; };
@@ -452,6 +454,11 @@ static inline uint32_t gen8_pml4e_index(uint64_t address)BUG ();/* For 64B */ } +static inline size_t gen8_pte_count (uint64_t address,uint64_t length) +{ +return i915_pte_count (address, length,GEN8_PDE_SHIFT ); +} +int i915_gem_gtt_init (struct drm_device *dev);void i915_gem_init_global_gtt (struct drm_device *dev);void i915_global_gtt_cleanup (struct drm_device *dev);
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH 12/13 drm/i915/gen8: Dynamic page table allocations
In reply to Michel ThierryThis finishes off the dynamic page tables allocations, in the legacy 3 level style that already exists. Most everything has already been setup to this point, the patch finishes off the enabling by setting the appropriate function pointers.
In LRC mode, contexts need to know the PDPs when they are populated. With dynamic page table allocations, these PDPs may not exist yet. Check if PDPs have been allocated and use the scratch page if they do not exist yet.
Before submission, update the PDPs in the logic ring context as PDPs have been allocated.
v2: Update aliasing/true ppgtt allocate/teardown/clear functions for gen 6 & 7.
v3: Rebase.
v4: Remove BUG() from ppgtt_unbind_vma, but keep checking that either teardown_va_range or clear_range functions exist (Daniel).
v5: Similar to gen6, in init, gen8_ppgtt_clear_range call is only needed for aliasing ppgtt. Zombie tracking was originally added for teardown function and is no longer required.
v6: Update err_out case in gen8_alloc_va_range (missed from lastest rebase).
v7: Rebase after s/page_tables/page_table/.
v8: Updated scratch_pt check after scratch flag was removed in previous patch.
v9: Note that lrc mode needs to be updated to support init state without any PDP.
v10: Unmap correct page_table in gen8_alloc_va_range's error case, clean-up gen8_aliasing_ppgtt_init (remove duplicated map), and initialize PTs during page table allocation.
v11: Squashed LRC enabling commit, otherwise LRC mode would be left broken until it was updated to handle the init case without any PDP.
v12: Do not overallocate new_pts bitmap, make alloc_gen8_temp_bitmaps static and don't abuse of inline functions. (Mika)
--- drivers/gpu/drm/i915/i915_gem_gtt.c | 307 +++++++++++++++++++++++++++--------- drivers/gpu/drm/i915/intel_lrc.c | 23 ++- 2 files changed, 253 insertions(+), 77 deletions(-)
drivers/gpu/drm/i915/i915_gem_gtt.c +236 -71
@@ -653,28 +653,6 @@ static void gen8_initialize_pd(struct i915_address_space *vm,if (!HAS_LLC (vm->dev))drm_clflush_virt_range (page_directory,PAGE_SIZE ); - -kunmap_atomic (page_directory); -} - -/* It's likely we'll map more than one pagetable at a time. This function will -* save us unnecessary kmap calls, but do no more functionally than multiple -* calls to map_pt. */ -static void gen8_map_pagetable_range (struct i915_page_directory *pd, -uint64_t start, -uint64_t length, -struct drm_device *dev) -{ -gen8_pde_t *page_directory =kmap_atomic (pd->page); -struct i915_page_table *pt; -uint64_t temp, pde; - -gen8_for_each_pde (pt, pd, start, length, temp, pde) -__gen8_do_map_pt (page_directory + pde, pt, dev); - -if (!HAS_LLC (dev)) -drm_clflush_virt_range (page_directory,PAGE_SIZE ); -kunmap_atomic (page_directory); }
@@ -718,73 +696,178 @@ static void gen8_ppgtt_cleanup(struct i915_address_space *vm)gen8_ppgtt_free (ppgtt); } +/** +* gen8_ppgtt_alloc_pagetabs() - Allocate page tables for VA range. +* @ppgtt: Master ppgtt structure. +* @pd: Page directory for this address range. +* @start: Starting virtual address to begin allocations. +* @length Size of the allocations. +* @new_pts: Bitmap set by function with new allocations. Likely used by the +* caller to free on error. +* +* Allocate the required number of page tables. Extremely similar to +* gen8_ppgtt_alloc_page_directories(). The main difference is here we are limited by +* the page directory boundary (instead of the page directory pointer). That +* boundary is 1GB virtual. Therefore, unlike gen8_ppgtt_alloc_page_directories(), it is +* possible, and likely that the caller will need to use multiple calls of this +* function to achieve the appropriate allocation. +* +* Return: 0 if success; negative error code otherwise. +*/ static int gen8_ppgtt_alloc_pagetabs (struct i915_hw_ppgtt *ppgtt,struct i915_page_directory *pd,uint64_t start, -uint64_t length) +uint64_t length, +unsigned long *new_pts) {struct drm_device *dev = ppgtt->base.dev; -struct i915_page_table *unused; +struct i915_page_table *pt;uint64_t temp;uint32_t pde; -gen8_for_each_pde (unused, pd, start, length, temp, pde) { -WARN_ON (unused); - pd->page_table[pde] =alloc_pt_single (dev); -if (IS_ERR (pd->page_table[pde])) +gen8_for_each_pde (pt, pd, start, length, temp, pde) { +/* Don't reallocate page tables */ +if (pt) { +/* Scratch is never allocated this way */ +WARN_ON (pt == ppgtt->scratch_pt); +continue ; + } + + pt =alloc_pt_single (dev); +if (IS_ERR (pt)) goto unwind_out; -gen8_initialize_pt (&ppgtt->base, pd->page_table[pde]); +gen8_initialize_pt (&ppgtt->base, pt); + pd->page_table[pde] = pt; +set_bit (pde, new_pts); }return 0 ; unwind_out: -while (pde--) +for_each_set_bit (pde, new_pts,I915_PDES )unmap_and_free_pt (pd->page_table[pde], dev);return -ENOMEM ; } +/** +* gen8_ppgtt_alloc_page_directories() - Allocate page directories for VA range. +* @ppgtt: Master ppgtt structure. +* @pdp: Page directory pointer for this address range. +* @start: Starting virtual address to begin allocations. +* @length Size of the allocations. +* @new_pds Bitmap set by function with new allocations. Likely used by the +* caller to free on error. +* +* Allocate the required number of page directories starting at the pde index of +* @start, and ending at the pde index @start + @length. This function will skip +* over already allocated page directories within the range, and only allocate +* new ones, setting the appropriate pointer within the pdp as well as the +* correct position in the bitmap @new_pds. +* +* The function will only allocate the pages within the range for a give page +* directory pointer. In other words, if @start + @length straddles a virtually +* addressed PDP boundary (512GB for 4k pages), there will be more allocations +* required by the caller, This is not currently possible, and the BUG in the +* code will prevent it. +* +* Return: 0 if success; negative error code otherwise. +*/ static int gen8_ppgtt_alloc_page_directories (struct i915_hw_ppgtt *ppgtt,struct i915_page_directory_pointer *pdp,uint64_t start, -uint64_t length) +uint64_t length, +unsigned long *new_pds) {struct drm_device *dev = ppgtt->base.dev; -struct i915_page_directory *unused; +struct i915_page_directory *pd;uint64_t temp;uint32_t pdpe; +WARN_ON (!bitmap_empty (new_pds,GEN8_LEGACY_PDPES )); +/* FIXME: PPGTT container_of won't work for 64b */ WARN_ON ((start + length) >0x800000000ULL ); -gen8_for_each_pdpe (unused, pdp, start, length, temp, pdpe) { -WARN_ON (unused); - pdp->page_directory[pdpe] =alloc_pd_single (dev); +gen8_for_each_pdpe (pd, pdp, start, length, temp, pdpe) { +if (pd) +continue ; -if (IS_ERR (pdp->page_directory[pdpe])) + pd =alloc_pd_single (dev); +if (IS_ERR (pd)) goto unwind_out; -gen8_initialize_pd (&ppgtt->base, - ppgtt->pdp.page_directory[pdpe]); +gen8_initialize_pd (&ppgtt->base, pd); + pdp->page_directory[pdpe] = pd; +set_bit (pdpe, new_pds); }return 0 ; unwind_out: -while (pdpe--) +for_each_set_bit (pdpe, new_pds,GEN8_LEGACY_PDPES )unmap_and_free_pd (pdp->page_directory[pdpe], dev);return -ENOMEM ; } +static void +free_gen8_temp_bitmaps (unsigned long *new_pds,unsigned long **new_pts) +{ +int i; + +for (i =0 ; i <GEN8_LEGACY_PDPES ; i++) +kfree (new_pts[i]); +kfree (new_pts); +kfree (new_pds); +} + +/* Fills in the page directory bitmap, and the array of page tables bitmap. Both +* of these are based on the number of PDPEs in the system. +*/ +static +int __must_check alloc_gen8_temp_bitmaps (unsigned long **new_pds, +unsigned long ***new_pts) +{ +int i; +unsigned long *pds; +unsigned long **pts; + + pds =kcalloc (BITS_TO_LONGS (GEN8_LEGACY_PDPES ),sizeof (unsigned long ),GFP_KERNEL ); +if (!pds) +return -ENOMEM ; + + pts =kcalloc (GEN8_LEGACY_PDPES ,sizeof (unsigned long *),GFP_KERNEL ); +if (!pts) { +kfree (pds); +return -ENOMEM ; + } + +for (i =0 ; i <GEN8_LEGACY_PDPES ; i++) { + pts[i] =kcalloc (BITS_TO_LONGS (I915_PDES ), +sizeof (unsigned long ),GFP_KERNEL ); +if (!pts[i]) + goto err_out; + } + + *new_pds = pds; + *new_pts = pts; + +return 0 ; + +err_out: +free_gen8_temp_bitmaps (pds, pts); +return -ENOMEM ; +} +static int gen8_alloc_va_range (struct i915_address_space *vm,uint64_t start,uint64_t length) {struct i915_hw_ppgtt *ppgtt = container_of(vm, struct i915_hw_ppgtt, base); +unsigned long *new_page_dirs, **new_page_tables;struct i915_page_directory *pd;const uint64_t orig_start = start;const uint64_t orig_length = length;
@@ -792,42 +875,98 @@ static int gen8_alloc_va_range(struct i915_address_space *vm,uint32_t pdpe;int ret; -/* Do the allocations first so we can easily bail out */ - ret =gen8_ppgtt_alloc_page_directories (ppgtt, &ppgtt->pdp, start, length); +#ifndef CONFIG_64BIT +/* Disallow 64b address on 32b platforms. Nothing is wrong with doing +* this in hardware, but a lot of the drm code is not prepared to handle +* 64b offset on 32b platforms. +* This will be addressed when 48b PPGTT is added */ +if (start + length >0x100000000ULL ) +return -E2BIG ; +#endif + +/* Wrap is never okay since we can only represent 48b, and we don't +* actually use the other side of the canonical address space. +*/ +if (WARN_ON (start + length < start)) +return -ERANGE ; + + ret =alloc_gen8_temp_bitmaps (&new_page_dirs, &new_page_tables);if (ret)return ret; +/* Do the allocations first so we can easily bail out */ + ret =gen8_ppgtt_alloc_page_directories (ppgtt, &ppgtt->pdp, start, length, + new_page_dirs); +if (ret) { +free_gen8_temp_bitmaps (new_page_dirs, new_page_tables); +return ret; + } + +/* For every page directory referenced, allocate page tables */ gen8_for_each_pdpe (pd, &ppgtt->pdp, start, length, temp, pdpe) { - ret =gen8_ppgtt_alloc_pagetabs (ppgtt, pd, start, length); + ret =gen8_ppgtt_alloc_pagetabs (ppgtt, pd, start, length, + new_page_tables[pdpe]);if (ret) goto err_out; } -/* Now mark everything we've touched as used. This doesn't allow for -* robust error checking, but it makes the code a hell of a lot simpler. -*/ start = orig_start; length = orig_length; +/* Allocations have completed successfully, so set the bitmaps, and do +* the mappings. */ gen8_for_each_pdpe (pd, &ppgtt->pdp, start, length, temp, pdpe) { +gen8_pde_t *const page_directory =kmap_atomic (pd->page);struct i915_page_table *pt;uint64_t pd_len =gen8_clamp_pd (start, length);uint64_t pd_start = start;uint32_t pde; -gen8_for_each_pde (pt, &ppgtt->pd, pd_start, pd_len, temp, pde) { -bitmap_set (pd->page_table[pde]->used_ptes, -gen8_pte_index (start), -gen8_pte_count (start, length)); +/* Every pd should be allocated, we just did that above. */ +WARN_ON (!pd); + +gen8_for_each_pde (pt, pd, pd_start, pd_len, temp, pde) { +/* Same reasoning as pd */ +WARN_ON (!pt); +WARN_ON (!pd_len); +WARN_ON (!gen8_pte_count (pd_start, pd_len)); + +/* Set our used ptes within the page table */ +bitmap_set (pt->used_ptes, +gen8_pte_index (pd_start), +gen8_pte_count (pd_start, pd_len)); + +/* Our pde is now pointing to the pagetable, pt */ set_bit (pde, pd->used_pdes); + +/* Map the PDE to the page table */ +__gen8_do_map_pt (page_directory + pde, pt, vm->dev); + +/* NB: We haven't yet mapped ptes to pages. At this +* point we're still relying on insert_entries() */ } + +if (!HAS_LLC (vm->dev)) +drm_clflush_virt_range (page_directory,PAGE_SIZE ); + +kunmap_atomic (page_directory); +set_bit (pdpe, ppgtt->pdp.used_pdpes); } +free_gen8_temp_bitmaps (new_page_dirs, new_page_tables);return 0 ; err_out: -gen8_ppgtt_free (ppgtt); +while (pdpe--) { +for_each_set_bit (temp, new_page_tables[pdpe],I915_PDES ) +unmap_and_free_pt (ppgtt->pdp.page_directory[pdpe]->page_table[temp], vm->dev); + } + +for_each_set_bit (pdpe, new_page_dirs,GEN8_LEGACY_PDPES ) +unmap_and_free_pd (ppgtt->pdp.page_directory[pdpe], vm->dev); + +free_gen8_temp_bitmaps (new_page_dirs, new_page_tables);return ret; }
@@ -838,21 +977,8 @@ err_out:* space. * */ -static int gen8_ppgtt_init (struct i915_hw_ppgtt *ppgtt,uint64_t size) +static int gen8_ppgtt_init_common (struct i915_hw_ppgtt *ppgtt,uint64_t size) { -struct i915_page_directory *pd; -uint64_t temp, start =0 ; -const uint64_t orig_length = size; -uint32_t pdpe; -int ret; - - ppgtt->base.start =0 ; - ppgtt->base.total = size; - ppgtt->base.clear_range = gen8_ppgtt_clear_range; - ppgtt->base.insert_entries = gen8_ppgtt_insert_entries; - ppgtt->base.cleanup = gen8_ppgtt_cleanup; - ppgtt->switch_mm = gen8_mm_switch; - ppgtt->scratch_pt =alloc_pt_single (ppgtt->base.dev);if (IS_ERR (ppgtt->scratch_pt))return PTR_ERR (ppgtt->scratch_pt);
@@ -864,6 +990,30 @@ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size)gen8_initialize_pt (&ppgtt->base, ppgtt->scratch_pt);gen8_initialize_pd (&ppgtt->base, ppgtt->scratch_pd); + ppgtt->base.start =0 ; + ppgtt->base.total = size; + ppgtt->base.cleanup = gen8_ppgtt_cleanup; + ppgtt->base.insert_entries = gen8_ppgtt_insert_entries; + + ppgtt->switch_mm = gen8_mm_switch; + +return 0 ; +} + +static int gen8_aliasing_ppgtt_init (struct i915_hw_ppgtt *ppgtt) +{ +struct drm_device *dev = ppgtt->base.dev; +struct drm_i915_private *dev_priv = dev->dev_private; +uint64_t start =0 , size = dev_priv->gtt.base.total; +int ret; + + ret =gen8_ppgtt_init_common (ppgtt, dev_priv->gtt.base.total); +if (ret) +return ret; + +/* Aliasing PPGTT has to always work and be mapped because of the way we +* use RESTORE_INHIBIT in the context switch. This will be fixed +* eventually. */ ret =gen8_alloc_va_range (&ppgtt->base, start, size);if (ret) {unmap_and_free_pd (ppgtt->scratch_pd, ppgtt->base.dev);
@@ -871,13 +1021,26 @@ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size)return ret; } - start =0 ; - size = orig_length; + ppgtt->base.allocate_va_range =NULL ; + ppgtt->base.clear_range = gen8_ppgtt_clear_range; + ppgtt->base.clear_range (&ppgtt->base,0 , ppgtt->base.total, true); -gen8_for_each_pdpe (pd, &ppgtt->pdp, start, size, temp, pdpe) -gen8_map_pagetable_range (pd, start, size, ppgtt->base.dev); +return 0 ; +} + +static int gen8_ppgtt_init (struct i915_hw_ppgtt *ppgtt) +{ +struct drm_device *dev = ppgtt->base.dev; +struct drm_i915_private *dev_priv = dev->dev_private; +int ret; + + ret =gen8_ppgtt_init_common (ppgtt, dev_priv->gtt.base.total); +if (ret) +return ret; + + ppgtt->base.allocate_va_range = gen8_alloc_va_range; + ppgtt->base.clear_range = gen8_ppgtt_clear_range; - ppgtt->base.clear_range (&ppgtt->base,0 , ppgtt->base.total, true);return 0 ; }
@@ -1416,7 +1579,7 @@ static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt, bool aliasing) } } - ppgtt->base.allocate_va_range = gen6_alloc_va_range; + ppgtt->base.allocate_va_range = aliasing ?NULL : gen6_alloc_va_range; ppgtt->base.clear_range = gen6_ppgtt_clear_range; ppgtt->base.insert_entries = gen6_ppgtt_insert_entries; ppgtt->base.cleanup = gen6_ppgtt_cleanup;
@@ -1457,8 +1620,10 @@ static int __hw_ppgtt_init(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt,if (INTEL_INFO (dev)->gen <8 )return gen6_ppgtt_init (ppgtt, aliasing); +else if (aliasing) +return gen8_aliasing_ppgtt_init (ppgtt);else -return gen8_ppgtt_init (ppgtt, dev_priv->gtt.base.total); +return gen8_ppgtt_init (ppgtt); }int i915_ppgtt_init (struct drm_device *dev,struct i915_hw_ppgtt *ppgtt) {
drivers/gpu/drm/i915/intel_lrc.c +17 -6
@@ -190,7 +190,7 @@#define GEN8_CTX_PRIVILEGE (1<<8)#define ASSIGN_CTX_PDP (ppgtt, reg_state, n) { \ - const u64 _addr = ppgtt->pdp.page_directory[n] ? \ + const u64 _addr = test_bit(n, ppgtt->pdp.used_pdpes) ? \ ppgtt->pdp.page_directory[n]->daddr : \ ppgtt->scratch_pd->daddr; \ reg_state[CTX_PDP ## n ## _UDW+1] = upper_32_bits(_addr); \
@@ -329,6 +329,7 @@ static void execlists_elsp_write(struct intel_engine_cs *ring,static int execlists_update_context (struct drm_i915_gem_object *ctx_obj,struct drm_i915_gem_object *ring_obj, +struct i915_hw_ppgtt *ppgtt,u32 tail) {struct page *page;
@@ -340,6 +341,16 @@ static int execlists_update_context(struct drm_i915_gem_object *ctx_obj, reg_state[CTX_RING_TAIL +1 ] = tail; reg_state[CTX_RING_BUFFER_START +1 ] =i915_gem_obj_ggtt_offset (ring_obj); +/* True PPGTT with dynamic page allocation: update PDP registers and +* point the unallocated PDPs to the scratch page +*/ +if (ppgtt) { +ASSIGN_CTX_PDP (ppgtt, reg_state,3 ); +ASSIGN_CTX_PDP (ppgtt, reg_state,2 ); +ASSIGN_CTX_PDP (ppgtt, reg_state,1 ); +ASSIGN_CTX_PDP (ppgtt, reg_state,0 ); + } +kunmap_atomic (reg_state);return 0 ;
@@ -358,7 +369,7 @@ static void execlists_submit_contexts(struct intel_engine_cs *ring,WARN_ON (!i915_gem_obj_is_pinned (ctx_obj0));WARN_ON (!i915_gem_obj_is_pinned (ringbuf0->obj)); -execlists_update_context (ctx_obj0, ringbuf0->obj, tail0); +execlists_update_context (ctx_obj0, ringbuf0->obj, to0->ppgtt, tail0);if (to1) { ringbuf1 = to1->engine[ring->id].ringbuf;
@@ -367,7 +378,7 @@ static void execlists_submit_contexts(struct intel_engine_cs *ring,WARN_ON (!i915_gem_obj_is_pinned (ctx_obj1));WARN_ON (!i915_gem_obj_is_pinned (ringbuf1->obj)); -execlists_update_context (ctx_obj1, ringbuf1->obj, tail1); +execlists_update_context (ctx_obj1, ringbuf1->obj, to1->ppgtt, tail1); }execlists_elsp_write (ring, ctx_obj0, ctx_obj1);
@@ -1785,9 +1796,9 @@ populate_lr_context(struct intel_context *ctx, struct drm_i915_gem_object *ctx_o reg_state[CTX_PDP1_LDW ] =GEN8_RING_PDP_LDW (ring,1 ); reg_state[CTX_PDP0_UDW ] =GEN8_RING_PDP_UDW (ring,0 ); reg_state[CTX_PDP0_LDW ] =GEN8_RING_PDP_LDW (ring,0 ); -/* XXX: Systems with less than 4GB of memory do not have -* all PDPs. Proper PDP tracking will be added in a -* subsequent patch. + +/* With dynamic page allocation, PDPs may not be allocated at this point, +* Point the unallocated PDPs to the scratch page */ ASSIGN_CTX_PDP (ppgtt, reg_state,3 );ASSIGN_CTX_PDP (ppgtt, reg_state,2 );
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH 13/13 drm/i915: Use complete address space in true PPGTT
In reply to Michel ThierryTrue PPGTT is capable of having a full address space, even if the system has less allocated memory.
Note that aliasing PPGTT always aliases the GGTT and thus should remain of the same size.
--- drivers/gpu/drm/i915/i915_gem_gtt.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-)
drivers/gpu/drm/i915/i915_gem_gtt.c +1 -3
@@ -1030,11 +1030,9 @@ static int gen8_aliasing_ppgtt_init(struct i915_hw_ppgtt *ppgtt)static int gen8_ppgtt_init (struct i915_hw_ppgtt *ppgtt) { -struct drm_device *dev = ppgtt->base.dev; -struct drm_i915_private *dev_priv = dev->dev_private;int ret; - ret =gen8_ppgtt_init_common (ppgtt, dev_priv->gtt.base.total); + ret =gen8_ppgtt_init_common (ppgtt, (1ULL <<32 ));if (ret)return ret;
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
Re: [PATCH 13/13] drm/i915: Use complete address space in true PPGTT
Task id: 6148 -------------------------------------Summary-------------------------------------
Platform Delta drm-intel-nightly Series Applied
PNV -2 272/272 270/272
ILK 302/302 302/302 SNB 303/303 303/303 IVB 338/338 338/338 BYT -1 287/287 286/287 HSW 361/361 361/361 BDW 308/308 308/308 -------------------------------------Detailed------------------------------------- Platform Test drm-intel-nightly Series Applied *PNV igt@gem_fence_thrash@bo-write-verify-threaded-none PASS(6) FAIL(1)PASS(1) PNV igt@gen3_render_tiledx_blits FAIL(9)PASS(6) FAIL(2) *BYT igt@gem_exec_bad_domains@conflicting-write-domain PASS(23) FAIL(1)PASS(1)
Note: You need to pay more attention to line start with '*'
_______________________________________________
Intel-gfx mailing list [email protected]
http://lists.freedesktop.org/mailman/listinfo/intel-gfx
Quoted 12 lines · Michel Thierry
These are the last remining patches to enable dynamic allocation in gen8+. All credit to Ben's original design and Mika's extensive reviews.
During stress testing, the light restore context corruption problem was observed in some systems (resubmission with HEAD==TAIL). The workaround to prevent to prevent this known problem should be in place as we also update the PDPx registers before the context is send to execution.
The last patch is only of interest in systems with less than 4GB of memory. Now that the PPGTT table overhead is not that big, we can use the full virtual space address range in these systems.
Quoted 15 lines
Michel Thierry (13):
drm/i915: Remove_entryfrom PPGTT page structures drm/i915: Remove unnecessarygen8_ppgtt_unmap_pagesdrm/i915/gen8: Initialize page tables
drm/i915/gen8: Add dynamic allocation macros and helper functions drm/i915/gen8: page directories rework allocation drm/i915/gen8: pagetable allocation rework
drm/i915/gen8: Update pdp switch and point unused PDPs to scratch page drm/i915: num_pd_pages/num_pd_entries isn't useful drm/i915: Extract PPGTT param frompage_directoryalloc drm/i915/gen8: Split out mappings
drm/i915/gen8: begin bitmap tracking
drm/i915/gen8: Dynamic page table allocations
drm/i915: Use complete address space in true PPGTT
Patches 1-13,
Quoted 9 lines
drivers/gpu/drm/i915/i915_debugfs.c | 2 - drivers/gpu/drm/i915/i915_gem_gtt.c | 620 +++++++++++++++++++++++------------- drivers/gpu/drm/i915/i915_gem_gtt.h | 82 ++++- drivers/gpu/drm/i915/i915_trace.h | 4 +- drivers/gpu/drm/i915/intel_lrc.c | 40 ++- 5 files changed, 504 insertions(+), 244 deletions(-)Signature · 1 line
2.1.1
_______________________________________________
Intel-gfx mailing list
[email protected]
http://lists.freedesktop.org/mailman/listinfo/intel-gfx
Quoted 30 lines · Mika Kuoppala
Quoted 12 lines · Michel Thierry
These are the last remining patches to enable dynamic allocation in gen8+. All credit to Ben's original design and Mika's extensive reviews.
During stress testing, the light restore context corruption problem was observed in some systems (resubmission with HEAD==TAIL). The workaround to prevent to prevent this known problem should be in place as we also update the PDPx registers before the context is send to execution.
The last patch is only of interest in systems with less than 4GB of memory. Now that the PPGTT table overhead is not that big, we can use the full virtual space address range in these systems.
Quoted 15 lines
Michel Thierry (13):
drm/i915: Remove_entryfrom PPGTT page structures drm/i915: Remove unnecessarygen8_ppgtt_unmap_pagesdrm/i915/gen8: Initialize page tables
drm/i915/gen8: Add dynamic allocation macros and helper functions drm/i915/gen8: page directories rework allocation drm/i915/gen8: pagetable allocation rework
drm/i915/gen8: Update pdp switch and point unused PDPs to scratch page drm/i915: num_pd_pages/num_pd_entries isn't useful drm/i915: Extract PPGTT param frompage_directoryalloc drm/i915/gen8: Split out mappings
drm/i915/gen8: begin bitmap tracking
drm/i915/gen8: Dynamic page table allocations
drm/i915: Use complete address space in true PPGTT
Quoted 2 lines
Patches 1-13,
Reviewed-by Mika Kuoppala <[email protected]>
Merged the entire patch series, thanks. A few things to take care of once this settles a bit (i.e. after the 48b stuff is merged):
- checkpatch wasn't too happy about the line breaks and long lines and stuff in here. But there's already a bit of that going on, so I figured a checkpatch pass over i915_gem_gtt.[hc] at the end is easier.
- A lot of the comments are now outdated, there's full-blown kerneldoc comments for static functions (which we generally don't do, that should all be evident from context) and I guess the existing kerneldoc also needs updating.
- i915_gem_gtt.h has way too many static inline functions. gcc will do a better job. Imo those should all/most be moved into i915_gem_gtt.c.
- i915_gem_gtt.c has a bunch of unecessary forward declarations.
Anyway just polish.
-Daniel
Signature · 7 lines
Daniel Vetter Software Engineer, Intel Corporation http://blog.ffwll.ch _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx