PPGTT dynamic page allocations
Subsystems (2)
PATCH v3 8/25 drm/i915: Create page table allocators
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
As we move toward dynamic page table allocation, it becomes much easier to manage our data structures if break do things less coarsely by breaking up all of our actions into individual tasks. This makes the code easier to write, read, and verify.
Aside from the dissection of the allocation functions, the patch statically allocates the page table structures without a page directory. This remains the same for all platforms,
The patch itself should not have much functional difference. The primary noticeable difference is the fact that page tables are no longer allocated, but rather statically declared as part of the page directory. This has non-zero overhead, but things gain non-trivial complexity as a result.
This patch exists for a few reasons:
1. Splitting out the functions allows easily combining GEN6 and GEN8 code. Page tables have no difference based on GEN8. As we'll see in a future patch when we add the DMA mappings to the allocations, it requires only one small change to make work, and error handling should just fall into place.
2. Unless we always want to allocate all page tables under a given PDE, we'll have to eventually break this up into an array of pointers (or pointer to pointer).
3. Having the discrete functions is easier to review, and understand. All allocations and frees now take place in just a couple of locations. Reviewing, and catching leaks should be easy.
4. Less important: the GFP flags are confined to one location, which makes playing around with such things trivial.
v2: Updated commit message to explain why this patch exists
v3: For lrc, s/pdp.page_directory[i].daddr/pdp.page_directory[i]->daddr/
v4: Renamed free_pt/pd_single functions to unmap_and_free_pt/pd (Daniel)
--- drivers/gpu/drm/i915/i915_gem_gtt.c | 224 +++++++++++++++++++++++------------- drivers/gpu/drm/i915/i915_gem_gtt.h | 4 +- drivers/gpu/drm/i915/intel_lrc.c | 16 +-- 3 files changed, 151 insertions(+), 93 deletions(-)
drivers/gpu/drm/i915/i915_gem_gtt.c +141 -83
@@ -275,6 +275,98 @@ static gen6_gtt_pte_t iris_pte_encode(dma_addr_t addr,return pte; } +static void unmap_and_free_pt (struct i915_page_table_entry *pt) +{ +if (WARN_ON (!pt->page)) +return ; +__free_page (pt->page); +kfree (pt); +} + +static struct i915_page_table_entry *alloc_pt_single (void ) +{ +struct i915_page_table_entry *pt; + + pt =kzalloc (sizeof (*pt),GFP_KERNEL ); +if (!pt) +return ERR_PTR (-ENOMEM ); + + pt->page =alloc_page (GFP_KERNEL | __GFP_ZERO); +if (!pt->page) { +kfree (pt); +return ERR_PTR (-ENOMEM ); + } + +return pt; +} + +/** +* alloc_pt_range() - Allocate a multiple page tables +* @pd: The page directory which will have at least @count entries +* available to point to the allocated page tables. +* @pde: First page directory entry for which we are allocating. +* @count: Number of pages to allocate. +* +* Allocates multiple page table pages and sets the appropriate entries in the +* page table structure within the page directory. Function cleans up after +* itself on any failures. +* +* Return: 0 if allocation succeeded. +*/ +static int alloc_pt_range (struct i915_page_directory_entry *pd,uint16_t pde,size_t count) +{ +int i, ret; + +/* 512 is the max page tables per page_directory on any platform. +* TODO: make WARN after patch series is done +*/ +BUG_ON (pde + count >GEN6_PPGTT_PD_ENTRIES ); + +for (i = pde; i < pde + count; i++) { +struct i915_page_table_entry *pt =alloc_pt_single (); +if (IS_ERR (pt)) { + ret =PTR_ERR (pt); + goto err_out; + } +WARN (pd->page_tables[i], +"Leaking page directory entry %d (%pa)\n" , + i, pd->page_tables[i]); + pd->page_tables[i] = pt; + } + +return 0 ; + +err_out: +while (i--) +unmap_and_free_pt (pd->page_tables[i]); +return ret; +} + +static void unmap_and_free_pd (struct i915_page_directory_entry *pd) +{ +if (pd->page) { +__free_page (pd->page); +kfree (pd); + } +} + +static struct i915_page_directory_entry *alloc_pd_single (void ) +{ +struct i915_page_directory_entry *pd; + + pd =kzalloc (sizeof (*pd),GFP_KERNEL ); +if (!pd) +return ERR_PTR (-ENOMEM ); + + pd->page =alloc_page (GFP_KERNEL | __GFP_ZERO); +if (!pd->page) { +kfree (pd); +return ERR_PTR (-ENOMEM ); + } + +return pd; +} +/* Broadwell Page Directory Pointer Descriptors */ static int gen8_write_pdp (struct intel_engine_cs *ring,unsigned entry,uint64_t val)
@@ -307,7 +399,7 @@ static int gen8_mm_switch(struct i915_hw_ppgtt *ppgtt,int used_pd = ppgtt->num_pd_entries /GEN8_PDES_PER_PAGE ;for (i = used_pd -1 ; i >=0 ; i--) { -dma_addr_t addr = ppgtt->pdp.page_directory[i].daddr; +dma_addr_t addr = ppgtt->pdp.page_directory[i]->daddr; ret =gen8_write_pdp (ring, i, addr);if (ret)return ret;
@@ -334,8 +426,9 @@ static void gen8_ppgtt_clear_range(struct i915_address_space *vm, I915_CACHE_LLC, use_scratch);while (num_entries) { -struct i915_page_directory_entry *pd = &ppgtt->pdp.page_directory[pdpe]; -struct page *page_table = pd->page_tables[pde].page; +struct i915_page_directory_entry *pd = ppgtt->pdp.page_directory[pdpe]; +struct i915_page_table_entry *pt = pd->page_tables[pde]; +struct page *page_table = pt->page; last_pte = pte + num_entries;if (last_pte >GEN8_PTES_PER_PAGE )
@@ -380,8 +473,9 @@ static void gen8_ppgtt_insert_entries(struct i915_address_space *vm,break ;if (pt_vaddr ==NULL ) { -struct i915_page_directory_entry *pd = &ppgtt->pdp.page_directory[pdpe]; -struct page *page_table = pd->page_tables[pde].page; +struct i915_page_directory_entry *pd = ppgtt->pdp.page_directory[pdpe]; +struct i915_page_table_entry *pt = pd->page_tables[pde]; +struct page *page_table = pt->page; pt_vaddr =kmap_atomic (page_table); }
@@ -412,18 +506,13 @@ static void gen8_free_page_tables(struct i915_page_directory_entry *pd) {int i; -if (pd->page_tables ==NULL ) +if (!pd->page)return ; -for (i =0 ; i <GEN8_PDES_PER_PAGE ; i++) -if (pd->page_tables[i].page) -__free_page (pd->page_tables[i].page); -} - -static void gen8_free_page_directories (struct i915_page_directory_entry *pd) -{ -kfree (pd->page_tables); -__free_page (pd->page); +for (i =0 ; i <GEN8_PDES_PER_PAGE ; i++) { +unmap_and_free_pt (pd->page_tables[i]); + pd->page_tables[i] =NULL ; + } }static void gen8_ppgtt_free (struct i915_hw_ppgtt *ppgtt)
@@ -431,8 +520,8 @@ static void gen8_ppgtt_free(struct i915_hw_ppgtt *ppgtt)int i;for (i =0 ; i < ppgtt->num_pd_pages; i++) { -gen8_free_page_tables (&ppgtt->pdp.page_directory[i]); -gen8_free_page_directories (&ppgtt->pdp.page_directory[i]); +gen8_free_page_tables (ppgtt->pdp.page_directory[i]); +unmap_and_free_pd (ppgtt->pdp.page_directory[i]); } }
@@ -444,14 +533,16 @@ static void gen8_ppgtt_unmap_pages(struct i915_hw_ppgtt *ppgtt)for (i =0 ; i < ppgtt->num_pd_pages; i++) {/* TODO: In the future we'll support sparse mappings, so this * will have to change. */ -if (!ppgtt->pdp.page_directory[i].daddr) +if (!ppgtt->pdp.page_directory[i]->daddr)continue ; -pci_unmap_page (hwdev, ppgtt->pdp.page_directory[i].daddr,PAGE_SIZE , +pci_unmap_page (hwdev, ppgtt->pdp.page_directory[i]->daddr,PAGE_SIZE ,PCI_DMA_BIDIRECTIONAL );for (j =0 ; j <GEN8_PDES_PER_PAGE ; j++) { -dma_addr_t addr = ppgtt->pdp.page_directory[i].page_tables[j].daddr; +struct i915_page_directory_entry *pd = ppgtt->pdp.page_directory[i]; +struct i915_page_table_entry *pt = pd->page_tables[j]; +dma_addr_t addr = pt->daddr;if (addr)pci_unmap_page (hwdev, addr,PAGE_SIZE ,PCI_DMA_BIDIRECTIONAL );
@@ -470,25 +561,20 @@ static void gen8_ppgtt_cleanup(struct i915_address_space *vm)static int gen8_ppgtt_allocate_page_tables (struct i915_hw_ppgtt *ppgtt) { -int i, j; +int i, ret;for (i =0 ; i < ppgtt->num_pd_pages; i++) { -struct i915_page_directory_entry *pd = &ppgtt->pdp.page_directory[i]; -for (j =0 ; j <GEN8_PDES_PER_PAGE ; j++) { -struct i915_page_table_entry *pt = &pd->page_tables[j]; - - pt->page =alloc_page (GFP_KERNEL | __GFP_ZERO); -if (!pt->page) - goto unwind_out; - - } + ret =alloc_pt_range (ppgtt->pdp.page_directory[i], +0 ,GEN8_PDES_PER_PAGE ); +if (ret) + goto unwind_out; }return 0 ; unwind_out:while (i--) -gen8_free_page_tables (&ppgtt->pdp.page_directory[i]); +gen8_free_page_tables (ppgtt->pdp.page_directory[i]);return -ENOMEM ; }
@@ -499,17 +585,9 @@ static int gen8_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt,int i;for (i =0 ; i < max_pdp; i++) { -struct i915_page_table_entry *pt; - - pt =kcalloc (GEN8_PDES_PER_PAGE ,sizeof (*pt),GFP_KERNEL ); -if (!pt) - goto unwind_out; - - ppgtt->pdp.page_directory[i].page =alloc_page (GFP_KERNEL ); -if (!ppgtt->pdp.page_directory[i].page) + ppgtt->pdp.page_directory[i] =alloc_pd_single (); +if (IS_ERR (ppgtt->pdp.page_directory[i])) goto unwind_out; - - ppgtt->pdp.page_directory[i].page_tables = pt; } ppgtt->num_pd_pages = max_pdp;
@@ -518,10 +596,8 @@ static int gen8_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt,return 0 ; unwind_out: -while (i--) { -kfree (ppgtt->pdp.page_directory[i].page_tables); -__free_page (ppgtt->pdp.page_directory[i].page); - } +while (i--) +unmap_and_free_pd (ppgtt->pdp.page_directory[i]);return -ENOMEM ; }
@@ -556,14 +632,14 @@ static int gen8_ppgtt_setup_page_directories(struct i915_hw_ppgtt *ppgtt,int ret; pd_addr =pci_map_page (ppgtt->base.dev->pdev, - ppgtt->pdp.page_directory[pd].page,0 , + ppgtt->pdp.page_directory[pd]->page,0 ,PAGE_SIZE ,PCI_DMA_BIDIRECTIONAL ); ret =pci_dma_mapping_error (ppgtt->base.dev->pdev, pd_addr);if (ret)return ret; - ppgtt->pdp.page_directory[pd].daddr = pd_addr; + ppgtt->pdp.page_directory[pd]->daddr = pd_addr;return 0 ; }
@@ -573,8 +649,8 @@ static int gen8_ppgtt_setup_page_tables(struct i915_hw_ppgtt *ppgtt, const int pt) {dma_addr_t pt_addr; -struct i915_page_directory_entry *pdir = &ppgtt->pdp.page_directory[pd]; -struct i915_page_table_entry *ptab = &pdir->page_tables[pt]; +struct i915_page_directory_entry *pdir = ppgtt->pdp.page_directory[pd]; +struct i915_page_table_entry *ptab = pdir->page_tables[pt];struct page *p = ptab->page;int ret;
@@ -637,10 +713,12 @@ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size)* will never need to touch the PDEs again. */ for (i =0 ; i < max_pdp; i++) { +struct i915_page_directory_entry *pd = ppgtt->pdp.page_directory[i];gen8_ppgtt_pde_t *pd_vaddr; - pd_vaddr =kmap_atomic (ppgtt->pdp.page_directory[i].page); + pd_vaddr =kmap_atomic (ppgtt->pdp.page_directory[i]->page);for (j =0 ; j <GEN8_PDES_PER_PAGE ; j++) { -dma_addr_t addr = ppgtt->pdp.page_directory[i].page_tables[j].daddr; +struct i915_page_table_entry *pt = pd->page_tables[j]; +dma_addr_t addr = pt->daddr; pd_vaddr[j] =gen8_pde_encode (ppgtt->base.dev, addr,I915_CACHE_LLC ); }
@@ -689,7 +767,7 @@ static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m)for (pde =0 ; pde < ppgtt->num_pd_entries; pde++) {u32 expected;gen6_gtt_pte_t *pt_vaddr; -dma_addr_t pt_addr = ppgtt->pd.page_tables[pde].daddr; +dma_addr_t pt_addr = ppgtt->pd.page_tables[pde]->daddr; pd_entry =readl (pd_addr + pde); expected = (GEN6_PDE_ADDR_ENCODE (pt_addr) |GEN6_PDE_VALID );
@@ -700,7 +778,7 @@ static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m) expected);seq_printf (m,"\tPDE: %x\n" , pd_entry); - pt_vaddr =kmap_atomic (ppgtt->pd.page_tables[pde].page); + pt_vaddr =kmap_atomic (ppgtt->pd.page_tables[pde]->page);for (pte =0 ; pte <I915_PPGTT_PT_ENTRIES ; pte+=4 ) {unsigned long va = (pde * PAGE_SIZE * I915_PPGTT_PT_ENTRIES) +
@@ -739,7 +817,7 @@ static void gen6_write_pdes(struct i915_hw_ppgtt *ppgtt)for (i =0 ; i < ppgtt->num_pd_entries; i++) {dma_addr_t pt_addr; - pt_addr = ppgtt->pd.page_tables[i].daddr; + pt_addr = ppgtt->pd.page_tables[i]->daddr; pd_entry =GEN6_PDE_ADDR_ENCODE (pt_addr); pd_entry |=GEN6_PDE_VALID ;
@@ -905,7 +983,7 @@ static void gen6_ppgtt_clear_range(struct i915_address_space *vm,if (last_pte >I915_PPGTT_PT_ENTRIES ) last_pte =I915_PPGTT_PT_ENTRIES ; - pt_vaddr =kmap_atomic (ppgtt->pd.page_tables[act_pt].page); + pt_vaddr =kmap_atomic (ppgtt->pd.page_tables[act_pt]->page);for (i = first_pte; i < last_pte; i++) pt_vaddr[i] = scratch_pte;
@@ -934,7 +1012,7 @@ static void gen6_ppgtt_insert_entries(struct i915_address_space *vm, pt_vaddr =NULL ;for_each_sg_page (pages->sgl, &sg_iter, pages->nents,0 ) {if (pt_vaddr ==NULL ) - pt_vaddr =kmap_atomic (ppgtt->pd.page_tables[act_pt].page); + pt_vaddr =kmap_atomic (ppgtt->pd.page_tables[act_pt]->page); pt_vaddr[act_pte] = vm->pte_encode(sg_page_iter_dma_address(&sg_iter),
@@ -957,7 +1035,7 @@ static void gen6_ppgtt_unmap_pages(struct i915_hw_ppgtt *ppgtt)for (i =0 ; i < ppgtt->num_pd_entries; i++)pci_unmap_page (ppgtt->base.dev->pdev, - ppgtt->pd.page_tables[i].daddr, + ppgtt->pd.page_tables[i]->daddr,4096 ,PCI_DMA_BIDIRECTIONAL ); }
@@ -966,8 +1044,9 @@ static void gen6_ppgtt_free(struct i915_hw_ppgtt *ppgtt)int i;for (i =0 ; i < ppgtt->num_pd_entries; i++) -__free_page (ppgtt->pd.page_tables[i].page); -kfree (ppgtt->pd.page_tables); +unmap_and_free_pt (ppgtt->pd.page_tables[i]); + +unmap_and_free_pd (&ppgtt->pd); }static void gen6_ppgtt_cleanup (struct i915_address_space *vm)
@@ -1022,27 +1101,6 @@ alloc:return 0 ; } -static int gen6_ppgtt_allocate_page_tables (struct i915_hw_ppgtt *ppgtt) -{ -struct i915_page_table_entry *pt; -int i; - - pt =kcalloc (ppgtt->num_pd_entries,sizeof (*pt),GFP_KERNEL ); -if (!pt) -return -ENOMEM ; - -for (i =0 ; i < ppgtt->num_pd_entries; i++) { - pt[i].page =alloc_page (GFP_KERNEL ); -if (!pt->page) { -gen6_ppgtt_free (ppgtt); -return -ENOMEM ; - } - } - - ppgtt->pd.page_tables = pt; -return 0 ; -} -static int gen6_ppgtt_alloc (struct i915_hw_ppgtt *ppgtt) {int ret;
@@ -1051,7 +1109,7 @@ static int gen6_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt)if (ret)return ret; - ret =gen6_ppgtt_allocate_page_tables (ppgtt); + ret =alloc_pt_range (&ppgtt->pd,0 , ppgtt->num_pd_entries);if (ret) {drm_mm_remove_node (&ppgtt->node);return ret;
@@ -1069,7 +1127,7 @@ static int gen6_ppgtt_setup_page_tables(struct i915_hw_ppgtt *ppgtt)struct page *page;dma_addr_t pt_addr; - page = ppgtt->pd.page_tables[i].page; + page = ppgtt->pd.page_tables[i]->page; pt_addr =pci_map_page (dev->pdev, page,0 ,4096 ,PCI_DMA_BIDIRECTIONAL );
@@ -1078,7 +1136,7 @@ static int gen6_ppgtt_setup_page_tables(struct i915_hw_ppgtt *ppgtt)return -EIO ; } - ppgtt->pd.page_tables[i].daddr = pt_addr; + ppgtt->pd.page_tables[i]->daddr = pt_addr; }return 0 ;
drivers/gpu/drm/i915/i915_gem_gtt.h +2 -2
@@ -199,12 +199,12 @@ struct i915_page_directory_entry {dma_addr_t daddr; }; -struct i915_page_table_entry *page_tables; +struct i915_page_table_entry *page_tables[GEN6_PPGTT_PD_ENTRIES ];/* PDEs */ };struct i915_page_directory_pointer_entry {/* struct page *page; */ -struct i915_page_directory_entry page_directory[GEN8_LEGACY_PDPES ]; +struct i915_page_directory_entry *page_directory[GEN8_LEGACY_PDPES ]; };struct i915_address_space {
drivers/gpu/drm/i915/intel_lrc.c +8 -8
@@ -1731,14 +1731,14 @@ populate_lr_context(struct intel_context *ctx, struct drm_i915_gem_object *ctx_o reg_state[CTX_PDP1_LDW ] =GEN8_RING_PDP_LDW (ring,1 ); reg_state[CTX_PDP0_UDW ] =GEN8_RING_PDP_UDW (ring,0 ); reg_state[CTX_PDP0_LDW ] =GEN8_RING_PDP_LDW (ring,0 ); - reg_state[CTX_PDP3_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[3 ].daddr); - reg_state[CTX_PDP3_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[3 ].daddr); - reg_state[CTX_PDP2_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[2 ].daddr); - reg_state[CTX_PDP2_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[2 ].daddr); - reg_state[CTX_PDP1_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[1 ].daddr); - reg_state[CTX_PDP1_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[1 ].daddr); - reg_state[CTX_PDP0_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[0 ].daddr); - reg_state[CTX_PDP0_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[0 ].daddr); + reg_state[CTX_PDP3_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[3 ]->daddr); + reg_state[CTX_PDP3_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[3 ]->daddr); + reg_state[CTX_PDP2_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[2 ]->daddr); + reg_state[CTX_PDP2_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[2 ]->daddr); + reg_state[CTX_PDP1_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[1 ]->daddr); + reg_state[CTX_PDP1_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[1 ]->daddr); + reg_state[CTX_PDP0_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[0 ]->daddr); + reg_state[CTX_PDP0_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[0 ]->daddr);if (ring->id ==RCS ) { reg_state[CTX_LRI_HEADER_2 ] =MI_LOAD_REGISTER_IMM (1 ); reg_state[CTX_R_PWR_CLK_STATE ] =0x20c8 ;
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH v3 9/25 drm/i915: Plumb drm_device through page tables operations
In reply to Michel ThierryThe next patch in the series will require it for alloc_pt_single.
--- drivers/gpu/drm/i915/i915_gem_gtt.c | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-)
drivers/gpu/drm/i915/i915_gem_gtt.c +16 -13
@@ -138,7 +138,6 @@ static int sanitize_enable_ppgtt(struct drm_device *dev, int enable_ppgtt)return has_aliasing_ppgtt ?1 :0 ; } -static void ppgtt_bind_vma (struct i915_vma *vma,enum i915_cache_level cache_level,u32 flags);
@@ -275,7 +274,7 @@ static gen6_gtt_pte_t iris_pte_encode(dma_addr_t addr,return pte; } -static void unmap_and_free_pt (struct i915_page_table_entry *pt) +static void unmap_and_free_pt (struct i915_page_table_entry *pt,struct drm_device *dev) {if (WARN_ON (!pt->page))return ;
@@ -283,7 +282,7 @@ static void unmap_and_free_pt(struct i915_page_table_entry *pt)kfree (pt); } -static struct i915_page_table_entry *alloc_pt_single (void ) +static struct i915_page_table_entry *alloc_pt_single (struct drm_device *dev) {struct i915_page_table_entry *pt;
@@ -313,7 +312,9 @@ static struct i915_page_table_entry *alloc_pt_single(void)* * Return: 0 if allocation succeeded. */ -static int alloc_pt_range (struct i915_page_directory_entry *pd,uint16_t pde,size_t count) +static int alloc_pt_range (struct i915_page_directory_entry *pd,uint16_t pde,size_t count, +struct drm_device *dev) + {int i, ret;
@@ -323,7 +324,7 @@ static int alloc_pt_range(struct i915_page_directory_entry *pd, uint16_t pde, siBUG_ON (pde + count >GEN6_PPGTT_PD_ENTRIES );for (i = pde; i < pde + count; i++) { -struct i915_page_table_entry *pt =alloc_pt_single (); +struct i915_page_table_entry *pt =alloc_pt_single (dev);if (IS_ERR (pt)) { ret =PTR_ERR (pt); goto err_out;
@@ -338,7 +339,7 @@ static int alloc_pt_range(struct i915_page_directory_entry *pd, uint16_t pde, si err_out:while (i--) -unmap_and_free_pt (pd->page_tables[i]); +unmap_and_free_pt (pd->page_tables[i], dev);return ret; }
@@ -502,7 +503,7 @@ static void gen8_ppgtt_insert_entries(struct i915_address_space *vm, } } -static void gen8_free_page_tables (struct i915_page_directory_entry *pd) +static void gen8_free_page_tables (struct i915_page_directory_entry *pd,struct drm_device *dev) {int i;
@@ -510,7 +511,7 @@ static void gen8_free_page_tables(struct i915_page_directory_entry *pd)return ;for (i =0 ; i <GEN8_PDES_PER_PAGE ; i++) { -unmap_and_free_pt (pd->page_tables[i]); +unmap_and_free_pt (pd->page_tables[i], dev); pd->page_tables[i] =NULL ; } }
@@ -520,7 +521,7 @@ static void gen8_ppgtt_free(struct i915_hw_ppgtt *ppgtt)int i;for (i =0 ; i < ppgtt->num_pd_pages; i++) { -gen8_free_page_tables (ppgtt->pdp.page_directory[i]); +gen8_free_page_tables (ppgtt->pdp.page_directory[i], ppgtt->base.dev);unmap_and_free_pd (ppgtt->pdp.page_directory[i]); } }
@@ -565,7 +566,7 @@ static int gen8_ppgtt_allocate_page_tables(struct i915_hw_ppgtt *ppgtt)for (i =0 ; i < ppgtt->num_pd_pages; i++) { ret =alloc_pt_range (ppgtt->pdp.page_directory[i], -0 ,GEN8_PDES_PER_PAGE ); +0 ,GEN8_PDES_PER_PAGE , ppgtt->base.dev);if (ret) goto unwind_out; }
@@ -574,7 +575,7 @@ static int gen8_ppgtt_allocate_page_tables(struct i915_hw_ppgtt *ppgtt) unwind_out:while (i--) -gen8_free_page_tables (ppgtt->pdp.page_directory[i]); +gen8_free_page_tables (ppgtt->pdp.page_directory[i], ppgtt->base.dev);return -ENOMEM ; }
@@ -1044,7 +1045,7 @@ static void gen6_ppgtt_free(struct i915_hw_ppgtt *ppgtt)int i;for (i =0 ; i < ppgtt->num_pd_entries; i++) -unmap_and_free_pt (ppgtt->pd.page_tables[i]); +unmap_and_free_pt (ppgtt->pd.page_tables[i], ppgtt->base.dev);unmap_and_free_pd (&ppgtt->pd); }
@@ -1109,7 +1110,9 @@ static int gen6_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt)if (ret)return ret; - ret =alloc_pt_range (&ppgtt->pd,0 , ppgtt->num_pd_entries); + ret =alloc_pt_range (&ppgtt->pd,0 , ppgtt->num_pd_entries, + ppgtt->base.dev); +if (ret) {drm_mm_remove_node (&ppgtt->node);return ret;
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH v3 10/25 drm/i915: Track GEN6 page table usage
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
Instead of implementing the full tracking + dynamic allocation, this patch does a bit less than half of the work, by tracking and warning on unexpected conditions. The tracking itself follows which PTEs within a page table are currently being used for objects. The next patch will modify this to actually allocate the page tables only when necessary.
With the current patch there isn't much in the way of making a gen agnostic range allocation function. However, in the next patch we'll add more specificity which makes having separate functions a bit easier to manage.
One important change introduced here is that DMA mappings are created/destroyed at the same page directories/tables are allocated/deallocated.
Notice that aliasing PPGTT is not managed here. The patch which actually begins dynamic allocation/teardown explains the reasoning for this.
v2: s/pdp.page_directory/pdp.page_directorys
Make a scratch page allocation helper
v3: Rebase and expand commit message.
v4: Allocate required pagetables only when it is needed, _bind_to_vm instead of bind_vma (Daniel).
v5: Rebased to remove the unnecessary noise in the diff, also:
- PDE mask is GEN agnostic, renamed GEN6_PDE_MASK to I915_PDE_MASK.
- Removed unnecessary checks in gen6_alloc_va_range.
- Changed map/unmap_px_single macros to use dma functions directly and be part of a static inline function instead.
- Moved drm_device plumbing through page tables operation to its own patch.
- Moved allocate/teardown_va_range calls until they are fully implemented (in subsequent patch).
- Merged pt and scratch_pt unmap_and_free path.
- Moved scratch page allocator helper to the patch that will use it.
--- drivers/gpu/drm/i915/i915_gem_gtt.c | 206 +++++++++++++++++++++++++----------- drivers/gpu/drm/i915/i915_gem_gtt.h | 77 ++++++++++++++ 2 files changed, 223 insertions(+), 60 deletions(-)
drivers/gpu/drm/i915/i915_gem_gtt.c +146 -60
@@ -274,29 +274,89 @@ static gen6_gtt_pte_t iris_pte_encode(dma_addr_t addr,return pte; } -static void unmap_and_free_pt (struct i915_page_table_entry *pt,struct drm_device *dev) +#define i915_dma_unmap_single (px, dev) do { \ + __i915_dma_unmap_single((px)->daddr, dev); \ +} while (0) + +static inline void __i915_dma_unmap_single (dma_addr_t daddr, +struct drm_device *dev) +{ +struct device *device = &dev->pdev->dev; + +dma_unmap_page (device, daddr,4096 ,PCI_DMA_BIDIRECTIONAL ); +} + +/** +* i915_dma_map_px_single() - Create a dma mapping for a page table/dir/etc. +* @px: Page table/dir/etc to get a DMA map for +* @dev: drm device +* +* Page table allocations are unified across all gens. They always require a +* single 4k allocation, as well as a DMA mapping. If we keep the structs +* symmetric here, the simple macro covers us for every page table type. +* +* Return: 0 if success. +*/ +#define i915_dma_map_px_single (px, dev) \ + i915_dma_map_page_single((px)->page, (dev), &(px)->daddr) + +static inline int i915_dma_map_page_single (struct page *page, +struct drm_device *dev, +dma_addr_t *daddr) +{ +struct device *device = &dev->pdev->dev; + + *daddr =dma_map_page (device, page,0 ,4096 ,PCI_DMA_BIDIRECTIONAL ); +return dma_mapping_error (device, *daddr); +} + +static void unmap_and_free_pt (struct i915_page_table_entry *pt, +struct drm_device *dev) {if (WARN_ON (!pt->page))return ; + +i915_dma_unmap_single (pt, dev);__free_page (pt->page); +kfree (pt->used_ptes);kfree (pt); }static struct i915_page_table_entry *alloc_pt_single (struct drm_device *dev) {struct i915_page_table_entry *pt; +const size_t count =INTEL_INFO (dev)->gen >=8 ? +GEN8_PTES_PER_PAGE :I915_PPGTT_PT_ENTRIES ; +int ret = -ENOMEM ; pt =kzalloc (sizeof (*pt),GFP_KERNEL );if (!pt)return ERR_PTR (-ENOMEM ); + pt->used_ptes =kcalloc (BITS_TO_LONGS (count),sizeof (*pt->used_ptes), +GFP_KERNEL ); + +if (!pt->used_ptes) + goto fail_bitmap; + pt->page =alloc_page (GFP_KERNEL | __GFP_ZERO); -if (!pt->page) { -kfree (pt); -return ERR_PTR (-ENOMEM ); - } +if (!pt->page) + goto fail_page; + + ret =i915_dma_map_px_single (pt, dev); +if (ret) + goto fail_dma;return pt; + +fail_dma: +__free_page (pt->page); +fail_page: +kfree (pt->used_ptes); +fail_bitmap: +kfree (pt); + +return ERR_PTR (ret); }/**
@@ -805,26 +865,36 @@ static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m) } } -static void gen6_write_pdes (struct i915_hw_ppgtt *ppgtt) +/* Write pde (index) from the page directory @pd to the page table @pt */ +static void gen6_write_pdes (struct i915_page_directory_entry *pd, +const int pde,struct i915_page_table_entry *pt) { -struct drm_i915_private *dev_priv = ppgtt->base.dev->dev_private; -gen6_gtt_pte_t __iomem *pd_addr; -uint32_t pd_entry; -int i; +struct i915_hw_ppgtt *ppgtt = + container_of(pd, struct i915_hw_ppgtt, pd); +u32 pd_entry; -WARN_ON (ppgtt->pd.pd_offset &0x3f ); - pd_addr = (gen6_gtt_pte_t __iomem *)dev_priv->gtt.gsm + - ppgtt->pd.pd_offset /sizeof (gen6_gtt_pte_t); -for (i =0 ; i < ppgtt->num_pd_entries; i++) { -dma_addr_t pt_addr; + pd_entry =GEN6_PDE_ADDR_ENCODE (pt->daddr); + pd_entry |=GEN6_PDE_VALID ; - pt_addr = ppgtt->pd.page_tables[i]->daddr; - pd_entry =GEN6_PDE_ADDR_ENCODE (pt_addr); - pd_entry |=GEN6_PDE_VALID ; +writel (pd_entry, ppgtt->pd_addr + pde); -writel (pd_entry, pd_addr + i); - } -readl (pd_addr); +/* XXX: Caller needs to make sure the write completes if necessary */ +} + +/* Write all the page tables found in the ppgtt structure to incrementing page +* directories. */ +static void gen6_write_page_range (struct drm_i915_private *dev_priv, +struct i915_page_directory_entry *pd,uint32_t start,uint32_t length) +{ +struct i915_page_table_entry *pt; +uint32_t pde, temp; + +gen6_for_each_pde (pt, pd, start, length, temp, pde) +gen6_write_pdes (pd, pde, pt); + +/* Make sure write is complete before other code can use this page +* table. Also require for WC mapped PTEs */ +readl (dev_priv->gtt.gsm); }static uint32_t get_pd_offset (struct i915_hw_ppgtt *ppgtt)
@@ -1040,6 +1110,41 @@ static void gen6_ppgtt_unmap_pages(struct i915_hw_ppgtt *ppgtt) 4096, PCI_DMA_BIDIRECTIONAL); } +static int gen6_alloc_va_range (struct i915_address_space *vm, +uint64_t start,uint64_t length) +{ +struct i915_hw_ppgtt *ppgtt = + container_of(vm, struct i915_hw_ppgtt, base); +struct i915_page_table_entry *pt; +uint32_t pde, temp; + +gen6_for_each_pde (pt, &ppgtt->pd, start, length, temp, pde) { +DECLARE_BITMAP (tmp_bitmap,I915_PPGTT_PT_ENTRIES ); +bitmap_zero (tmp_bitmap,I915_PPGTT_PT_ENTRIES ); +bitmap_set (tmp_bitmap,gen6_pte_index (start), +gen6_pte_count (start, length)); + +bitmap_or (pt->used_ptes, pt->used_ptes, tmp_bitmap, +I915_PPGTT_PT_ENTRIES ); + } + +return 0 ; +} + +static void gen6_teardown_va_range (struct i915_address_space *vm, +uint64_t start,uint64_t length) +{ +struct i915_hw_ppgtt *ppgtt = + container_of(vm, struct i915_hw_ppgtt, base); +struct i915_page_table_entry *pt; +uint32_t pde, temp; + +gen6_for_each_pde (pt, &ppgtt->pd, start, length, temp, pde) { +bitmap_clear (pt->used_ptes,gen6_pte_index (start), +gen6_pte_count (start, length)); + } +} +static void gen6_ppgtt_free (struct i915_hw_ppgtt *ppgtt) {int i;
@@ -1086,20 +1191,24 @@ alloc:0 , dev_priv->gtt.base.total, 0);if (ret) -return ret; + goto err_out; retried = true; goto alloc; }if (ret) -return ret; + goto err_out; +if (ppgtt->node.start < dev_priv->gtt.mappable_end)DRM_DEBUG ("Forced to use aperture for PDEs\n" ); ppgtt->num_pd_entries =GEN6_PPGTT_PD_ENTRIES ;return 0 ; + +err_out: +return ret; }static int gen6_ppgtt_alloc (struct i915_hw_ppgtt *ppgtt)
@@ -1121,30 +1230,6 @@ static int gen6_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt)return 0 ; } -static int gen6_ppgtt_setup_page_tables (struct i915_hw_ppgtt *ppgtt) -{ -struct drm_device *dev = ppgtt->base.dev; -int i; - -for (i =0 ; i < ppgtt->num_pd_entries; i++) { -struct page *page; -dma_addr_t pt_addr; - - page = ppgtt->pd.page_tables[i]->page; - pt_addr =pci_map_page (dev->pdev, page,0 ,4096 , -PCI_DMA_BIDIRECTIONAL ); - -if (pci_dma_mapping_error (dev->pdev, pt_addr)) { -gen6_ppgtt_unmap_pages (ppgtt); -return -EIO ; - } - - ppgtt->pd.page_tables[i]->daddr = pt_addr; - } - -return 0 ; -} -static int gen6_ppgtt_init (struct i915_hw_ppgtt *ppgtt) {struct drm_device *dev = ppgtt->base.dev;
@@ -1165,12 +1250,8 @@ static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt)if (ret)return ret; - ret =gen6_ppgtt_setup_page_tables (ppgtt); -if (ret) { -gen6_ppgtt_free (ppgtt); -return ret; - } - + ppgtt->base.allocate_va_range = gen6_alloc_va_range; + ppgtt->base.teardown_va_range = gen6_teardown_va_range; ppgtt->base.clear_range = gen6_ppgtt_clear_range; ppgtt->base.insert_entries = gen6_ppgtt_insert_entries; ppgtt->base.cleanup = gen6_ppgtt_cleanup;
@@ -1181,11 +1262,15 @@ static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt) ppgtt->pd.pd_offset = ppgtt->node.start /PAGE_SIZE *sizeof (gen6_gtt_pte_t); + ppgtt->pd_addr = (gen6_gtt_pte_t __iomem *)dev_priv->gtt.gsm + + ppgtt->pd.pd_offset /sizeof (gen6_gtt_pte_t); + +gen6_write_page_range (dev_priv, &ppgtt->pd,0 , ppgtt->base.total); +DRM_DEBUG_DRIVER ("Allocated pde space (%ldM) at GTT entry: %lx\n" , ppgtt->node.size >>20 , ppgtt->node.start /PAGE_SIZE ); -gen6_write_pdes (ppgtt);DRM_DEBUG ("Adding PPGTT at offset %x\n" , ppgtt->pd.pd_offset <<10 );
@@ -1460,13 +1545,14 @@ void i915_gem_restore_gtt_mappings(struct drm_device *dev)list_for_each_entry (vm, &dev_priv->vm_list, global_link) {/* TODO: Perhaps it shouldn't be gen6 specific */ -if (i915_is_ggtt (vm)) { -if (dev_priv->mm.aliasing_ppgtt) -gen6_write_pdes (dev_priv->mm.aliasing_ppgtt); -continue ; - } -gen6_write_pdes (container_of(vm ,struct i915_hw_ppgtt ,base )); +struct i915_hw_ppgtt *ppgtt = + container_of(vm, struct i915_hw_ppgtt, base); + +if (i915_is_ggtt (vm)) + ppgtt = dev_priv->mm.aliasing_ppgtt; + +gen6_write_page_range (dev_priv, &ppgtt->pd,0 , ppgtt->num_pd_entries); }i915_ggtt_flush (dev_priv);
drivers/gpu/drm/i915/i915_gem_gtt.h +77 -0
108 lines not on this page · raw message
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH v3 11/25 drm/i915: Extract context switch skip and pd load logic
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
We have some fanciness coming up. This patch just breaks out the logic of context switch skip, pd load pre, and pd load post.
v2: Use new functions to replace the logic right away (Daniel)
--- drivers/gpu/drm/i915/i915_gem_context.c | 40 +++++++++++++++++++++++++-------- 1 file changed, 31 insertions(+), 9 deletions(-)
drivers/gpu/drm/i915/i915_gem_context.c +31 -9
@@ -565,6 +565,33 @@ mi_set_context(struct intel_engine_cs *ring,return ret; } +static inline bool should_skip_switch (struct intel_engine_cs *ring, +struct intel_context *from, +struct intel_context *to) +{ +if (from == to && !to->remap_slice) +return true; + +return false; +} + +static bool +needs_pd_load_pre (struct intel_engine_cs *ring,struct intel_context *to) +{ +struct drm_i915_private *dev_priv = ring->dev->dev_private; + +return ((INTEL_INFO (ring->dev)->gen <8 ) || + (ring != &dev_priv->ring[RCS ])) && to->ppgtt; +} + +static bool +needs_pd_load_post (struct intel_engine_cs *ring,struct intel_context *to) +{ +return (!to->legacy_hw_ctx.initialized || +i915_gem_context_is_default (to)) && + to->ppgtt &&IS_GEN8 (ring->dev); +} +static int do_switch (struct intel_engine_cs *ring,struct intel_context *to) {
@@ -573,9 +600,6 @@ static int do_switch(struct intel_engine_cs *ring,u32 hw_flags =0 ;bool uninitialized = false;struct i915_vma *vma; -bool needs_pd_load_pre = ((INTEL_INFO (ring->dev)->gen <8 ) || - (ring != &dev_priv->ring[RCS ])) && to->ppgtt; -bool needs_pd_load_post = false;int ret, i;if (from !=NULL && ring == &dev_priv->ring[RCS ]) {
@@ -583,7 +607,7 @@ static int do_switch(struct intel_engine_cs *ring,BUG_ON (!i915_gem_obj_is_pinned (from->legacy_hw_ctx.rcs_state)); } -if (from == to && !to->remap_slice) +if (should_skip_switch (ring, from, to))return 0 ;/* Trying to pin first makes error handling easier. */
@@ -601,7 +625,7 @@ static int do_switch(struct intel_engine_cs *ring,*/ from = ring->last_context; -if (needs_pd_load_pre) { +if (needs_pd_load_pre (ring, to)) {/* Older GENs and non render rings still want the load first, * "PP_DCLV followed by PP_DIR_BASE register through Load * Register Immediate commands in Ring Buffer before submitting
@@ -646,16 +670,14 @@ static int do_switch(struct intel_engine_cs *ring,* XXX: If we implemented page directory eviction code, this * optimization needs to be removed. */ -if (!to->legacy_hw_ctx.initialized ||i915_gem_context_is_default (to)) { +if (!to->legacy_hw_ctx.initialized ||i915_gem_context_is_default (to)) hw_flags |=MI_RESTORE_INHIBIT ; - needs_pd_load_post = to->ppgtt &&IS_GEN8 (ring->dev); - } ret =mi_set_context (ring, to, hw_flags);if (ret) goto unpin_out; -if (needs_pd_load_post) { +if (needs_pd_load_post (ring, to)) { ret = to->ppgtt->switch_mm (to->ppgtt, ring);/* The hardware context switch is emitted, but we haven't * actually changed the state - so it's probably safe to bail
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH v3 12/25 drm/i915: Track page table reload need
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
This patch was formerly known as, "Force pd restore when PDEs change, gen6-7." I had to change the name because it is needed for GEN8 too.
The real issue this is trying to solve is when a new object is mapped into the current address space. The GPU does not snoop the new mapping so we must do the gen specific action to reload the page tables.
GEN8 and GEN7 do differ in the way they load page tables for the RCS. GEN8 does so with the context restore, while GEN7 requires the proper load commands in the command streamer. Non-render is similar for both.
Caveat for GEN7
The docs say you cannot change the PDEs of a currently running context. We never map new PDEs of a running context, and expect them to be present - so I think this is okay. (We can unmap, but this should also be okay since we only unmap unreferenced objects that the GPU shouldn't be tryingto va->pa xlate.) The MI_SET_CONTEXT command does have a flag to signal that even if the context is the same, force a reload. It's unclear exactly what this does, but I have a hunch it's the right thing to do.
The logic assumes that we always emit a context switch after mapping new PDEs, and before we submit a batch. This is the case today, and has been the case since the inception of hardware contexts. A note in the comment let's the user know.
squash! drm/i915: Force pd restore when PDEs change, gen6-7
It's not just for gen8. If the current context has mappings change, we need a context reload to switch
v2: Rebased after ppgtt clean up patches. Split the warning for aliasing and true ppgtt options. And do not break aliasing ppgtt, where to->ppgtt is always null.
v3: Invalidate PPGTT TLBs inside alloc_va_range and teardown_va_range.
v4: Rename ppgtt_invalidate_tlbs to mark_tlbs_dirty and move pd_dirty_rings from i915_address_space to i915_hw_ppgtt.
--- drivers/gpu/drm/i915/i915_gem_context.c | 27 ++++++++++++++++++++++----- drivers/gpu/drm/i915/i915_gem_execbuffer.c | 11 +++++++++++ drivers/gpu/drm/i915/i915_gem_gtt.c | 13 +++++++++++++ drivers/gpu/drm/i915/i915_gem_gtt.h | 1 + 4 files changed, 47 insertions(+), 5 deletions(-)
drivers/gpu/drm/i915/i915_gem_context.c +22 -5
@@ -569,8 +569,18 @@ static inline bool should_skip_switch(struct intel_engine_cs *ring,struct intel_context *from, struct intel_context *to) { -if (from == to && !to->remap_slice) -return true; + struct drm_i915_private *dev_priv = ring->dev->dev_private; + +if (to->remap_slice) +return false; + +if (to->ppgtt) { +if (from == to && !test_bit (ring->id, &to->ppgtt->pd_dirty_rings)) +return true; + }else { +if (from == to && !test_bit (ring->id, &dev_priv->mm.aliasing_ppgtt->pd_dirty_rings)) +return true; + }return false; }
@@ -587,9 +597,8 @@ needs_pd_load_pre(struct intel_engine_cs *ring, struct intel_context *to)static bool needs_pd_load_post (struct intel_engine_cs *ring,struct intel_context *to) { -return (!to->legacy_hw_ctx.initialized || -i915_gem_context_is_default (to)) && - to->ppgtt &&IS_GEN8 (ring->dev); +return IS_GEN8 (ring->dev) && + (to->ppgtt || &to->ppgtt->pd_dirty_rings); } static int do_switch(struct intel_engine_cs *ring,
@@ -634,6 +643,12 @@ static int do_switch(struct intel_engine_cs *ring, ret = to->ppgtt->switch_mm (to->ppgtt, ring);if (ret) goto unpin_out; + +/* Doing a PD load always reloads the page dirs */ +if (to->ppgtt) +clear_bit (ring->id, &to->ppgtt->pd_dirty_rings); +else +clear_bit (ring->id, &dev_priv->mm.aliasing_ppgtt->pd_dirty_rings); }if (ring != &dev_priv->ring[RCS ]) {
@@ -672,6 +687,8 @@ static int do_switch(struct intel_engine_cs *ring,*/ if (!to->legacy_hw_ctx.initialized ||i915_gem_context_is_default (to)) hw_flags |=MI_RESTORE_INHIBIT ; +else if (to->ppgtt &&test_and_clear_bit (ring->id, &to->ppgtt->pd_dirty_rings)) + hw_flags |=MI_FORCE_RESTORE ; ret =mi_set_context (ring, to, hw_flags);if (ret)
drivers/gpu/drm/i915/i915_gem_execbuffer.c +11 -0
@@ -1198,6 +1198,13 @@ i915_gem_ringbuffer_submission(struct drm_device *dev, struct drm_file *file,if (ret) goto error; +if (ctx->ppgtt) +WARN (ctx->ppgtt->pd_dirty_rings & (1 <<ring->id), +"%s didn't clear reload\n" , ring->name); +else +WARN (dev_priv->mm.aliasing_ppgtt->pd_dirty_rings & + (1 <<ring->id),"%s didn't clear reload\n" , ring->name); + instp_mode = args->flags &I915_EXEC_CONSTANTS_MASK ; instp_mask =I915_EXEC_CONSTANTS_MASK ;switch (instp_mode) {
@@ -1445,6 +1452,10 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,if (ret) goto err; +/* XXX: Reserve has possibly change PDEs which means we must do a +* context switch before we can coherently read some of the reserved +* VMAs. */ +/* The objects are in their final locations, apply the relocations. */ if (need_relocs) ret =i915_gem_execbuffer_relocate (eb);
drivers/gpu/drm/i915/i915_gem_gtt.c +13 -0
@@ -1110,6 +1110,16 @@ static void gen6_ppgtt_unmap_pages(struct i915_hw_ppgtt *ppgtt) 4096, PCI_DMA_BIDIRECTIONAL); } +/* PDE TLBs are a pain invalidate pre GEN8. It requires a context reload. If we +* are switching between contexts with the same LRCA, we also must do a force +* restore. +*/ +static inline void mark_tlbs_dirty (struct i915_hw_ppgtt *ppgtt) +{ +/* If current vm != vm, */ \ + ppgtt->pd_dirty_rings =INTEL_INFO (ppgtt->base.dev)->ring_mask; +} +static int gen6_alloc_va_range (struct i915_address_space *vm,uint64_t start,uint64_t length) {
@@ -1128,6 +1138,7 @@ static int gen6_alloc_va_range(struct i915_address_space *vm, I915_PPGTT_PT_ENTRIES); } +mark_tlbs_dirty (ppgtt);return 0 ; }
@@ -1143,6 +1154,8 @@ static void gen6_teardown_va_range(struct i915_address_space *vm,bitmap_clear (pt->used_ptes,gen6_pte_index (start),gen6_pte_count (start, length)); } + +mark_tlbs_dirty (ppgtt); }static void gen6_ppgtt_free (struct i915_hw_ppgtt *ppgtt)
drivers/gpu/drm/i915/i915_gem_gtt.h +1 -0
@@ -300,6 +300,7 @@ struct i915_hw_ppgtt {struct i915_address_space base;struct kref ref;struct drm_mm_node node; +unsigned long pd_dirty_rings;unsigned num_pd_entries;unsigned num_pd_pages;/* gen8+ */ union {
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH v3 13/25 drm/i915: Initialize all contexts
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
The problem is we're going to switch to a new context, which could be the default context. The plan was to use restore inhibit, which would be fine, except if we are using dynamic page tables (which we will). If we use dynamic page tables and we don't load new page tables, the previous page tables might go away, and future operations will fault.
CTXA runs.
switch to default, restore inhibit
CTXA dies and has its address space taken away.
Run CTXB, tries to save using the context A's address space - this fails.
The general solution is to make sure every context has it's own state, and its own address space. For cases when we must restore inhibit, first thing we do is load a valid address space. I thought this would be enough, but apparently there are references within the context itself which will refer to the old address space - therefore, we also must reinitialize.
It was tricky to track this down as we don't have much insight into what happens in a context save.
This is required for the next patch which enables dynamic page tables.
v2: to->ppgtt is only valid in full ppgtt.
--- drivers/gpu/drm/i915/i915_gem_context.c | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-)
drivers/gpu/drm/i915/i915_gem_context.c +11 -14
@@ -594,13 +594,6 @@ needs_pd_load_pre(struct intel_engine_cs *ring, struct intel_context *to) (ring != &dev_priv->ring[RCS])) && to->ppgtt; } -static bool -needs_pd_load_post (struct intel_engine_cs *ring,struct intel_context *to) -{ -return IS_GEN8 (ring->dev) && - (to->ppgtt || &to->ppgtt->pd_dirty_rings); -} -static int do_switch (struct intel_engine_cs *ring,struct intel_context *to) {
@@ -681,20 +674,24 @@ static int do_switch(struct intel_engine_cs *ring,/* GEN8 does *not* require an explicit reload if the PDPs have been * setup, and we do not wish to move them. -* -* XXX: If we implemented page directory eviction code, this -* optimization needs to be removed. */ -if (!to->legacy_hw_ctx.initialized ||i915_gem_context_is_default (to)) +if (!to->legacy_hw_ctx.initialized) { hw_flags |=MI_RESTORE_INHIBIT ; -else if (to->ppgtt &&test_and_clear_bit (ring->id, &to->ppgtt->pd_dirty_rings)) +/* NB: If we inhibit the restore, the context is not allowed to +* die because future work may end up depending on valid address +* space. This means we must enforce that a page table load +* occur when this occurs. */ + }else if (to->ppgtt &&test_and_clear_bit (ring->id, &to->ppgtt->pd_dirty_rings)) hw_flags |=MI_FORCE_RESTORE ; ret =mi_set_context (ring, to, hw_flags);if (ret) goto unpin_out; -if (needs_pd_load_post (ring, to)) { +if (IS_GEN8 (ring->dev) && to->ppgtt && (hw_flags &MI_RESTORE_INHIBIT )) { +/* We have a valid page directory (scratch) to switch to. This +* allows the old VM to be freed. Note that if anything occurs +* between the set context, and here, we are f*cked */ ret = to->ppgtt->switch_mm (to->ppgtt, ring);/* The hardware context switch is emitted, but we haven't * actually changed the state - so it's probably safe to bail
@@ -744,7 +741,7 @@ static int do_switch(struct intel_engine_cs *ring,i915_gem_context_unreference (from); } - uninitialized = !to->legacy_hw_ctx.initialized && from ==NULL ; + uninitialized = !to->legacy_hw_ctx.initialized; to->legacy_hw_ctx.initialized = true; done:
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH v3 14/25 drm/i915: Finish gen6/7 dynamic page table allocation
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
This patch continues on the idea from the previous patch. From here on, in the steady state, PDEs are all pointing to the scratch page table (as recommended in the spec). When an object is allocated in the VA range, the code will determine if we need to allocate a page for the page table. Similarly when the object is destroyed, we will remove, and free the page table pointing the PDE back to the scratch page.
Following patches will work to unify the code a bit as we bring in GEN8 support. GEN6 and GEN8 are different enough that I had a hard time to get to this point with as much common code as I do.
The aliasing PPGTT must pre-allocate all of the page tables. There are a few reasons for this. Two trivial ones: aliasing ppgtt goes through the ggtt paths, so it's hard to maintain, we currently do not restore the default context (assuming the previous force reload is indeed necessary). Most importantly though, the only way (it seems from empirical evidence) to invalidate the CS TLBs on non-render ring is to either use ring sync (which requires actually stopping the rings in order to synchronize when the sync completes vs. where you are in execution), or to reload DCLV. Since without full PPGTT we do not ever reload the DCLV register, there is no good way to achieve this. The simplest solution is just to not support dynamic page table creation/destruction in the aliasing PPGTT.
We could always reload DCLV, but this seems like quite a bit of excess overhead only to save at most 2MB-4k of memory for the aliasing PPGTT page tables.
v2: Make the page table bitmap declared inside the function (Chris) Simplify the way scratching address space works.
Move the alloc/teardown tracepoints up a level in the call stack so that both all implementations get the trace.
v3: Updated trace event to spit out a name
v4: Aliasing ppgtt is now initialized differently (in setup global gtt)
v5: Rebase to latest code. Also removed unnecessary aliasing ppgtt check for trace, as it is no longer possible after the PPGTT cleanup patch series of a couple of months ago (Daniel).
v6: Implement changes from code review (Daniel):
- allocate/teardown_va_range calls added.
- Add a scratch page allocation helper (only need the address).
- Move trace events to a new patch.
- Use updated mark_tlbs_dirty.
- Moved pt preallocation for aliasing ppgtt into gen6_ppgtt_init.
--- drivers/gpu/drm/i915/i915_debugfs.c | 3 +- drivers/gpu/drm/i915/i915_gem.c | 9 +++ drivers/gpu/drm/i915/i915_gem_gtt.c | 143 ++++++++++++++++++++++++++++++++---- drivers/gpu/drm/i915/i915_gem_gtt.h | 3 + 4 files changed, 142 insertions(+), 16 deletions(-)
drivers/gpu/drm/i915/i915_debugfs.c +2 -1
@@ -2149,6 +2149,8 @@ static void gen6_ppgtt_info(struct seq_file *m, struct drm_device *dev)seq_printf (m,"PP_DIR_BASE_READ: 0x%08x\n" ,I915_READ (RING_PP_DIR_BASE_READ (ring)));seq_printf (m,"PP_DIR_DCLV: 0x%08x\n" ,I915_READ (RING_PP_DIR_DCLV (ring))); } +seq_printf (m,"ECOCHK: 0x%08x\n\n" ,I915_READ (GAM_ECOCHK )); +if (dev_priv->mm.aliasing_ppgtt) {struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
@@ -2165,7 +2167,6 @@ static void gen6_ppgtt_info(struct seq_file *m, struct drm_device *dev) get_pid_task(file->pid, PIDTYPE_PID)->comm);idr_for_each (&file_priv->context_idr, per_file_ctx, m); } -seq_printf (m,"ECOCHK: 0x%08x\n" ,I915_READ (GAM_ECOCHK )); }static int i915_ppgtt_info (struct seq_file *m,void *data)
drivers/gpu/drm/i915/i915_gem.c +9 -0
@@ -3585,6 +3585,15 @@ search_free:if (ret) goto err_remove_node; +/* allocate before insert / bind */ +if (vma->vm->allocate_va_range) { + ret = vma->vm->allocate_va_range (vma->vm, + vma->node.start, + vma->node.size); +if (ret) + goto err_remove_node; + } +trace_i915_vma_bind (vma, flags); ret =i915_vma_bind (vma, obj->cache_level, flags &PIN_GLOBAL ?GLOBAL_BIND :0 );
drivers/gpu/drm/i915/i915_gem_gtt.c +128 -15
255 lines not on this page · raw message
drivers/gpu/drm/i915/i915_gem_gtt.h +3 -0
@@ -195,6 +195,7 @@ struct i915_page_table_entry {dma_addr_t daddr;unsigned long *used_ptes; + unsigned int scratch:1; };struct i915_page_directory_entry {
@@ -308,6 +309,8 @@ struct i915_hw_ppgtt {struct i915_page_directory_entry pd; }; +struct i915_page_table_entry *scratch_pt; +struct drm_i915_file_private *file_priv;gen6_gtt_pte_t __iomem *pd_addr;
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH v3 15/25 drm/i915: Add dynamic page trace events
In reply to Michel ThierryTraces for page directories and tables allocation/destroy and map/unmap.
--- drivers/gpu/drm/i915/i915_gem.c | 2 + drivers/gpu/drm/i915/i915_gem_gtt.c | 17 ++++++ drivers/gpu/drm/i915/i915_trace.h | 115 ++++++++++++++++++++++++++++++++++++ 3 files changed, 134 insertions(+)
drivers/gpu/drm/i915/i915_gem.c +2 -0
@@ -3587,6 +3587,8 @@ search_free:/* allocate before insert / bind */ if (vma->vm->allocate_va_range) { +trace_i915_va_alloc (vma->vm, vma->node.start, vma->node.size, +VM_TO_TRACE_NAME (vma->vm)); ret = vma->vm->allocate_va_range (vma->vm, vma->node.start, vma->node.size);
drivers/gpu/drm/i915/i915_gem_gtt.c +17 -0
@@ -1176,6 +1176,7 @@ static int gen6_alloc_va_range(struct i915_address_space *vm, ppgtt->pd.page_tables[pde] = pt;set_bit (pde, new_page_tables); +trace_i915_page_table_entry_alloc (vm, pde, start,GEN6_PDE_SHIFT ); } start = start_save;
@@ -1190,6 +1191,10 @@ static int gen6_alloc_va_range(struct i915_address_space *vm,if (test_and_clear_bit (pde, new_page_tables))gen6_write_pdes (&ppgtt->pd, pde, pt); +trace_i915_page_table_entry_map (vm, pde, pt, +gen6_pte_index (start), +gen6_pte_count (start, length), +I915_PPGTT_PT_ENTRIES );bitmap_or (pt->used_ptes, tmp_bitmap, pt->used_ptes,I915_PPGTT_PT_ENTRIES ); }
@@ -1229,10 +1234,18 @@ static void gen6_teardown_va_range(struct i915_address_space *vm, vm, pde, start, start + length))continue ; +trace_i915_page_table_entry_unmap (vm, pde, pt, +gen6_pte_index (start), +gen6_pte_count (start, length), +I915_PPGTT_PT_ENTRIES ); +bitmap_clear (pt->used_ptes,gen6_pte_index (start),gen6_pte_count (start, length));if (bitmap_empty (pt->used_ptes,I915_PPGTT_PT_ENTRIES )) { +trace_i915_page_table_entry_destroy (vm, pde, + start &GENMASK_ULL (63 ,GEN6_PDE_SHIFT ), +GEN6_PDE_SHIFT );gen6_write_pdes (&ppgtt->pd, pde, ppgtt->scratch_pt); ppgtt->pd.page_tables[pde] = ppgtt->scratch_pt; }
@@ -1525,6 +1538,10 @@ static void ppgtt_unbind_vma(struct i915_vma *vma) vma->obj->base.size, true);if (vma->vm->teardown_va_range) { +trace_i915_va_teardown (vma->vm, + vma->node.start, vma->node.size, +VM_TO_TRACE_NAME (vma->vm)); + vma->vm->teardown_va_range (vma->vm, vma->node.start, vma->node.size); }
drivers/gpu/drm/i915/i915_trace.h +115 -0
121 lines not on this page · raw message
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH v3 16/25 drm/i915/bdw: Use dynamic allocation idioms on free
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
The page directory freer is left here for now as it's still useful given that GEN8 still preallocates. Once the allocation functions are broken up into more discrete chunks, we'll follow suit and destroy this leftover piece.
v2: Match trace_i915_va_teardown params
v3: Multiple rebases.
v4: Updated to use unmap_and_free_pt.
--- drivers/gpu/drm/i915/i915_gem_gtt.c | 54 +++++++++++++++++++++++-------------- drivers/gpu/drm/i915/i915_gem_gtt.h | 46 +++++++++++++++++++++++++++++++ 2 files changed, 80 insertions(+), 20 deletions(-)
drivers/gpu/drm/i915/i915_gem_gtt.c +34 -20
@@ -580,27 +580,32 @@ static void gen8_ppgtt_insert_entries(struct i915_address_space *vm, } } -static void gen8_free_page_tables (struct i915_page_directory_entry *pd,struct drm_device *dev) +static void gen8_teardown_va_range (struct i915_address_space *vm, +uint64_t start,uint64_t length) { -int i; - -if (!pd->page) -return ; - -for (i =0 ; i <GEN8_PDES_PER_PAGE ; i++) { -unmap_and_free_pt (pd->page_tables[i], dev); - pd->page_tables[i] =NULL ; +struct i915_hw_ppgtt *ppgtt = + container_of(vm, struct i915_hw_ppgtt, base); +struct i915_page_directory_entry *pd; +struct i915_page_table_entry *pt; +uint64_t temp; +uint32_t pdpe, pde; + +gen8_for_each_pdpe (pd, &ppgtt->pdp, start, length, temp, pdpe) { +uint64_t pd_len =gen8_clamp_pd (start, length); +uint64_t pd_start = start; +gen8_for_each_pde (pt, pd, pd_start, pd_len, temp, pde) { +unmap_and_free_pt (pt, vm->dev); + } +unmap_and_free_pd (pd); } } -static void gen8_ppgtt_free (struct i915_hw_ppgtt *ppgtt) +/* This function will die soon */ +static void gen8_free_full_page_directory (struct i915_hw_ppgtt *ppgtt,int i) { -int i; - -for (i =0 ; i < ppgtt->num_pd_pages; i++) { -gen8_free_page_tables (ppgtt->pdp.page_directory[i], ppgtt->base.dev); -unmap_and_free_pd (ppgtt->pdp.page_directory[i]); - } +gen8_teardown_va_range (&ppgtt->base, + i <<GEN8_PDPE_SHIFT , + (1 <<GEN8_PDPE_SHIFT )); }static void gen8_ppgtt_unmap_pages (struct i915_hw_ppgtt *ppgtt)
@@ -615,19 +620,28 @@ static void gen8_ppgtt_unmap_pages(struct i915_hw_ppgtt *ppgtt)continue ;pci_unmap_page (hwdev, ppgtt->pdp.page_directory[i]->daddr,PAGE_SIZE , -PCI_DMA_BIDIRECTIONAL ); +PCI_DMA_BIDIRECTIONAL );for (j =0 ; j <GEN8_PDES_PER_PAGE ; j++) {struct i915_page_directory_entry *pd = ppgtt->pdp.page_directory[i]; -struct i915_page_table_entry *pt = pd->page_tables[j]; +struct i915_page_table_entry *pt = pd->page_tables[j];dma_addr_t addr = pt->daddr;if (addr)pci_unmap_page (hwdev, addr,PAGE_SIZE , -PCI_DMA_BIDIRECTIONAL ); +PCI_DMA_BIDIRECTIONAL ); } } } +static void gen8_ppgtt_free (struct i915_hw_ppgtt *ppgtt) +{ +trace_i915_va_teardown (&ppgtt->base, + ppgtt->base.start, ppgtt->base.total, +VM_TO_TRACE_NAME (&ppgtt->base)); +gen8_teardown_va_range (&ppgtt->base, + ppgtt->base.start, ppgtt->base.total); +} +static void gen8_ppgtt_cleanup (struct i915_address_space *vm) { struct i915_hw_ppgtt *ppgtt =
@@ -652,7 +666,7 @@ static int gen8_ppgtt_allocate_page_tables(struct i915_hw_ppgtt *ppgtt) unwind_out:while (i--) -gen8_free_page_tables (ppgtt->pdp.page_directory[i], ppgtt->base.dev); +gen8_free_full_page_directory (ppgtt, i);return -ENOMEM ; }
drivers/gpu/drm/i915/i915_gem_gtt.h +46 -0
52 lines not on this page · raw message
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH v3 17/25 drm/i915/bdw: page directories rework allocation
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
Start using gen8_for_each_pdpe macro to allocate the page directories.
v2: Rebased after s/free_pt_*/unmap_and_free_pt/ change.
--- drivers/gpu/drm/i915/i915_gem_gtt.c | 43 ++++++++++++++++++++++++++----------- 1 file changed, 31 insertions(+), 12 deletions(-)
drivers/gpu/drm/i915/i915_gem_gtt.c +31 -12
@@ -595,8 +595,10 @@ static void gen8_teardown_va_range(struct i915_address_space *vm,uint64_t pd_start = start;gen8_for_each_pde (pt, pd, pd_start, pd_len, temp, pde) {unmap_and_free_pt (pt, vm->dev); + pd->page_tables[pde] =NULL ; }unmap_and_free_pd (pd); + ppgtt->pdp.page_directory[pdpe] =NULL ; } }
@@ -671,25 +673,39 @@ unwind_out:return -ENOMEM ; } -static int gen8_ppgtt_allocate_page_directories (struct i915_hw_ppgtt *ppgtt, -const int max_pdp) +static int gen8_ppgtt_alloc_page_directories (struct i915_page_directory_pointer_entry *pdp, +uint64_t start, +uint64_t length) { -int i; +struct i915_hw_ppgtt *ppgtt = + container_of(pdp, struct i915_hw_ppgtt, pdp); +struct i915_page_directory_entry *unused; +uint64_t temp; +uint32_t pdpe; -for (i =0 ; i < max_pdp; i++) { - ppgtt->pdp.page_directory[i] =alloc_pd_single (); -if (IS_ERR (ppgtt->pdp.page_directory[i])) +/* FIXME: PPGTT container_of won't work for 64b */ +BUG_ON ((start + length) >0x800000000ULL ); + +gen8_for_each_pdpe (unused, pdp, start, length, temp, pdpe) { +BUG_ON (unused); + pdp->page_directory[pdpe] =alloc_pd_single (); +if (IS_ERR (ppgtt->pdp.page_directory[pdpe])) goto unwind_out; + + ppgtt->num_pd_pages++; } - ppgtt->num_pd_pages = max_pdp;BUG_ON (ppgtt->num_pd_pages >GEN8_LEGACY_PDPES );return 0 ; unwind_out: -while (i--) -unmap_and_free_pd (ppgtt->pdp.page_directory[i]); +while (pdpe--) { +unmap_and_free_pd (ppgtt->pdp.page_directory[pdpe]); + ppgtt->num_pd_pages--; + } + +WARN_ON (ppgtt->num_pd_pages);return -ENOMEM ; }
@@ -699,7 +715,8 @@ static int gen8_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt, {int ret; - ret =gen8_ppgtt_allocate_page_directories (ppgtt, max_pdp); + ret =gen8_ppgtt_alloc_page_directories (&ppgtt->pdp, ppgtt->base.start, + ppgtt->base.total);if (ret)return ret;
@@ -776,6 +793,10 @@ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size)if (size % (1 <<30 ))DRM_INFO ("Pages will be wasted unless GTT size (%llu) is divisible by 1GB\n" , size); + ppgtt->base.start =0 ; + ppgtt->base.total = size; +BUG_ON (ppgtt->base.total ==0 ); +/* 1. Do all our allocations for page directories and page tables. */ ret =gen8_ppgtt_alloc (ppgtt, max_pdp);if (ret)
@@ -823,8 +844,6 @@ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size) ppgtt->base.clear_range = gen8_ppgtt_clear_range; ppgtt->base.insert_entries = gen8_ppgtt_insert_entries; ppgtt->base.cleanup = gen8_ppgtt_cleanup; - ppgtt->base.start =0 ; - ppgtt->base.total = ppgtt->num_pd_entries *GEN8_PTES_PER_PAGE *PAGE_SIZE ;DRM_DEBUG_DRIVER ("Allocated %d pages for page directories (%d wasted)\n" , ppgtt->num_pd_pages, ppgtt->num_pd_pages - max_pdp);
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH v3 18/25 drm/i915/bdw: pagetable allocation rework
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
Start using gen8_for_each_pde macro to allocate page tables.
--- drivers/gpu/drm/i915/i915_gem_gtt.c | 54 ++++++++++++++++++++----------------- drivers/gpu/drm/i915/i915_gem_gtt.h | 10 +++++++ 2 files changed, 39 insertions(+), 25 deletions(-)
drivers/gpu/drm/i915/i915_gem_gtt.c +29 -25
@@ -602,14 +602,6 @@ static void gen8_teardown_va_range(struct i915_address_space *vm, } } -/* This function will die soon */ -static void gen8_free_full_page_directory (struct i915_hw_ppgtt *ppgtt,int i) -{ -gen8_teardown_va_range (&ppgtt->base, - i <<GEN8_PDPE_SHIFT , - (1 <<GEN8_PDPE_SHIFT )); -} -static void gen8_ppgtt_unmap_pages (struct i915_hw_ppgtt *ppgtt) {struct pci_dev *hwdev = ppgtt->base.dev->pdev;
@@ -653,22 +645,27 @@ static void gen8_ppgtt_cleanup(struct i915_address_space *vm)gen8_ppgtt_free (ppgtt); } -static int gen8_ppgtt_allocate_page_tables (struct i915_hw_ppgtt *ppgtt) +static int gen8_ppgtt_alloc_pagetabs (struct i915_page_directory_entry *pd, +uint64_t start, +uint64_t length, +struct drm_device *dev) { -int i, ret; +struct i915_page_table_entry *unused; +uint64_t temp; +uint32_t pde; -for (i =0 ; i < ppgtt->num_pd_pages; i++) { - ret =alloc_pt_range (ppgtt->pdp.page_directory[i], -0 ,GEN8_PDES_PER_PAGE , ppgtt->base.dev); -if (ret) +gen8_for_each_pde (unused, pd, start, length, temp, pde) { +BUG_ON (unused); + pd->page_tables[pde] =alloc_pt_single (dev); +if (IS_ERR (pd->page_tables[pde])) goto unwind_out; }return 0 ; unwind_out: -while (i--) -gen8_free_full_page_directory (ppgtt, i); +while (pde--) +unmap_and_free_pt (pd->page_tables[pde], dev);return -ENOMEM ; }
@@ -711,20 +708,28 @@ unwind_out: }static int gen8_ppgtt_alloc (struct i915_hw_ppgtt *ppgtt, -const int max_pdp) +uint64_t start, +uint64_t length) { +struct i915_page_directory_entry *pd; +uint64_t temp; +uint32_t pdpe;int ret; - ret =gen8_ppgtt_alloc_page_directories (&ppgtt->pdp, ppgtt->base.start, - ppgtt->base.total); + ret =gen8_ppgtt_alloc_page_directories (&ppgtt->pdp, start, length);if (ret)return ret; - ret =gen8_ppgtt_allocate_page_tables (ppgtt); -if (ret) - goto err_out; +gen8_for_each_pdpe (pd, &ppgtt->pdp, start, length, temp, pdpe) { + ret =gen8_ppgtt_alloc_pagetabs (pd, start, length, + ppgtt->base.dev); +if (ret) + goto err_out; + + ppgtt->num_pd_entries +=GEN8_PDES_PER_PAGE ; + } - ppgtt->num_pd_entries = max_pdp *GEN8_PDES_PER_PAGE ; +BUG_ON (pdpe > ppgtt->num_pd_pages);return 0 ;
@@ -795,10 +800,9 @@ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size) ppgtt->base.start =0 ; ppgtt->base.total = size; -BUG_ON (ppgtt->base.total ==0 );/* 1. Do all our allocations for page directories and page tables. */ - ret =gen8_ppgtt_alloc (ppgtt, max_pdp); + ret =gen8_ppgtt_alloc (ppgtt, ppgtt->base.start, ppgtt->base.total);if (ret)return ret;
drivers/gpu/drm/i915/i915_gem_gtt.h +10 -0
@@ -401,6 +401,16 @@ static inline uint32_t gen6_pde_index(uint32_t addr) temp = min(temp, length), \ start += temp, length -= temp) +/* Clamp length to the next pagetab boundary */ +static inline uint64_t gen8_clamp_pt (uint64_t start,uint64_t length) +{ +uint64_t next_pt =ALIGN (start +1 ,1 <<GEN8_PDE_SHIFT ); +if (next_pt > (start + length)) +return length; + +return next_pt - start; +} +/* Clamp length to the next page_directory boundary */ static inline uint64_t gen8_clamp_pd (uint64_t start,uint64_t length) {
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH v3 19/25 drm/i915/bdw: Update pdp switch and point unused PDPs to scratch page
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
One important part of this patch is we now write a scratch page directory into any unused PDP descriptors. This matters for 2 reasons, first, we're not allowed to just use 0, or an invalid pointer, and second, we must wipe out any previous contents from the last context.
The latter point only matters with full PPGTT. The former point only effect platforms with less than 4GB memory.
v2: Updated commit message to point that we must set unused PDPs to the scratch page.
--- drivers/gpu/drm/i915/i915_gem_gtt.c | 29 ++++++++++++++++++----------- drivers/gpu/drm/i915/i915_gem_gtt.h | 5 ++++- 2 files changed, 22 insertions(+), 12 deletions(-)
drivers/gpu/drm/i915/i915_gem_gtt.c +18 -11
@@ -446,8 +446,9 @@ static struct i915_page_directory_entry *alloc_pd_single(void) }/* Broadwell Page Directory Pointer Descriptors */ -static int gen8_write_pdp (struct intel_engine_cs *ring,unsigned entry, -uint64_t val) +static int gen8_write_pdp (struct intel_engine_cs *ring, +unsigned entry, +dma_addr_t addr) {int ret;
@@ -459,10 +460,10 @@ static int gen8_write_pdp(struct intel_engine_cs *ring, unsigned entry,intel_ring_emit (ring,MI_LOAD_REGISTER_IMM (1 ));intel_ring_emit (ring,GEN8_RING_PDP_UDW (ring, entry)); -intel_ring_emit (ring, (u32)(val >>32 )); +intel_ring_emit (ring,upper_32_bits (addr));intel_ring_emit (ring,MI_LOAD_REGISTER_IMM (1 ));intel_ring_emit (ring,GEN8_RING_PDP_LDW (ring, entry)); -intel_ring_emit (ring, (u32)(val)); +intel_ring_emit (ring,lower_32_bits (addr));intel_ring_advance (ring);return 0 ;
@@ -473,12 +474,12 @@ static int gen8_mm_switch(struct i915_hw_ppgtt *ppgtt, {int i, ret; -/* bit of a hack to find the actual last used pd */ -int used_pd = ppgtt->num_pd_entries /GEN8_PDES_PER_PAGE ; - -for (i = used_pd -1 ; i >=0 ; i--) { -dma_addr_t addr = ppgtt->pdp.page_directory[i]->daddr; - ret =gen8_write_pdp (ring, i, addr); +for (i =GEN8_LEGACY_PDPES -1 ; i >=0 ; i--) { +struct i915_page_directory_entry *pd = ppgtt->pdp.page_directory[i]; +dma_addr_t pd_daddr = pd ? pd->daddr : ppgtt->scratch_pd->daddr; +/* The page directory might be NULL, but we need to clear out +* whatever the previous context might have used. */ + ret =gen8_write_pdp (ring, i, pd_daddr);if (ret)return ret; }
@@ -801,10 +802,16 @@ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size) ppgtt->base.start =0 ; ppgtt->base.total = size; + ppgtt->scratch_pd =alloc_pt_scratch (ppgtt->base.dev); +if (IS_ERR (ppgtt->scratch_pd)) +return PTR_ERR (ppgtt->scratch_pd); +/* 1. Do all our allocations for page directories and page tables. */ ret =gen8_ppgtt_alloc (ppgtt, ppgtt->base.start, ppgtt->base.total); -if (ret) +if (ret) { +unmap_and_free_pt (ppgtt->scratch_pd, ppgtt->base.dev);return ret; + }/* * 2. Create DMA mappings for the page directories and page tables.
drivers/gpu/drm/i915/i915_gem_gtt.h +4 -1
@@ -309,7 +309,10 @@ struct i915_hw_ppgtt {struct i915_page_directory_entry pd; }; -struct i915_page_table_entry *scratch_pt; +union { +struct i915_page_table_entry *scratch_pt; +struct i915_page_table_entry *scratch_pd;/* Just need the daddr */ + };struct drm_i915_file_private *file_priv;
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH v3 20/25 drm/i915: num_pd_pages/num_pd_entries isn't useful
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
These values are never quite useful for dynamic allocations of the page tables. Getting rid of them will help prevent later confusion.
v2: Updated to use unmap_and_free_pd functions.
--- drivers/gpu/drm/i915/i915_debugfs.c | 2 -- drivers/gpu/drm/i915/i915_gem_gtt.c | 66 +++++++++++++------------------------ drivers/gpu/drm/i915/i915_gem_gtt.h | 7 ++-- 3 files changed, 27 insertions(+), 48 deletions(-)
drivers/gpu/drm/i915/i915_debugfs.c +0 -2
@@ -2117,8 +2117,6 @@ static void gen8_ppgtt_info(struct seq_file *m, struct drm_device *dev)if (!ppgtt)return ; -seq_printf (m,"Page directories: %d\n" , ppgtt->num_pd_pages); -seq_printf (m,"Page tables: %d\n" , ppgtt->num_pd_entries);for_each_ring (ring, dev_priv, unused) {seq_printf (m,"%s\n" , ring->name);for (i =0 ; i <4 ; i++) {
drivers/gpu/drm/i915/i915_gem_gtt.c +22 -44
158 lines not on this page · raw message
drivers/gpu/drm/i915/i915_gem_gtt.h +5 -2
@@ -302,8 +302,6 @@ struct i915_hw_ppgtt {struct kref ref;struct drm_mm_node node;unsigned long pd_dirty_rings; -unsigned num_pd_entries; -unsigned num_pd_pages;/* gen8+ */ union {struct i915_page_directory_pointer_entry pdp;struct i915_page_directory_entry pd;
@@ -341,6 +339,11 @@ struct i915_hw_ppgtt { temp = min(temp, (unsigned)length), \ start += temp, length -= temp) +#define gen6_for_all_pdes (pt, ppgtt, iter) \ + for (iter = 0, pt = ppgtt->pd.page_tables[iter]; \ + iter < gen6_pde_index(ppgtt->base.total); \ + pt = ppgtt->pd.page_tables[++iter]) +static inline uint32_t i915_pte_index (uint64_t address,uint32_t pde_shift) {const uint32_t mask =NUM_PTE (pde_shift) -1 ;
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH v3 21/25 drm/i915: Extract PPGTT param from page_directory alloc
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
Now that we don't need to trace num_pd_pages, we may as well kill all need for the PPGTT structure in the alloc_page_directorys. This is very useful for when we move to 48b addressing, and the PDP isn't the root of the page table structure.
The param is replaced with drm_device, which is an unavoidable wart throughout the series. (in other words, not extra flagrant).
--- drivers/gpu/drm/i915/i915_gem_gtt.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-)
drivers/gpu/drm/i915/i915_gem_gtt.c +2 -4
@@ -675,8 +675,6 @@ static int gen8_ppgtt_alloc_page_directories(struct i915_page_directory_pointer_ uint64_t start, uint64_t length) { -struct i915_hw_ppgtt *ppgtt = - container_of(pdp, struct i915_hw_ppgtt, pdp);struct i915_page_directory_entry *unused;uint64_t temp;uint32_t pdpe;
@@ -687,7 +685,7 @@ static int gen8_ppgtt_alloc_page_directories(struct i915_page_directory_pointer_gen8_for_each_pdpe (unused, pdp, start, length, temp, pdpe) {BUG_ON (unused); pdp->page_directory[pdpe] =alloc_pd_single (); -if (IS_ERR (ppgtt->pdp.page_directory[pdpe])) +if (IS_ERR (pdp->page_directory[pdpe])) goto unwind_out; }
@@ -695,7 +693,7 @@ static int gen8_ppgtt_alloc_page_directories(struct i915_page_directory_pointer_ unwind_out:while (pdpe--) -unmap_and_free_pd (ppgtt->pdp.page_directory[pdpe]); +unmap_and_free_pd (pdp->page_directory[pdpe]);return -ENOMEM ; }
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH v3 22/25 drm/i915/bdw: Split out mappings
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
When we do dynamic page table allocations for gen8, we'll need to have more control over how and when we map page tables, similar to gen6. In particular, DMA mappings for page directories/tables occur at allocation time.
This patch adds the functionality and calls it at init, which should have no functional change.
The PDPEs are still a special case for now. We'll need a function for that in the future as well.
v2: Handle renamed unmap_and_free_page functions.
--- drivers/gpu/drm/i915/i915_gem_gtt.c | 183 ++++++++++++++---------------------- 1 file changed, 71 insertions(+), 112 deletions(-)
drivers/gpu/drm/i915/i915_gem_gtt.c +71 -112
@@ -420,17 +420,20 @@ err_out:return ret; } -static void unmap_and_free_pd (struct i915_page_directory_entry *pd) +static void unmap_and_free_pd (struct i915_page_directory_entry *pd, +struct drm_device *dev) {if (pd->page) { +i915_dma_unmap_single (pd, dev);__free_page (pd->page);kfree (pd); } } -static struct i915_page_directory_entry *alloc_pd_single (void ) +static struct i915_page_directory_entry *alloc_pd_single (struct drm_device *dev) {struct i915_page_directory_entry *pd; +int ret; pd =kzalloc (sizeof (*pd),GFP_KERNEL );if (!pd)
@@ -442,6 +445,13 @@ static struct i915_page_directory_entry *alloc_pd_single(void)return ERR_PTR (-ENOMEM ); } + ret =i915_dma_map_px_single (pd, dev); +if (ret) { +__free_page (pd->page); +kfree (pd); +return ERR_PTR (ret); + } +return pd; }
@@ -581,6 +591,36 @@ static void gen8_ppgtt_insert_entries(struct i915_address_space *vm, } } +static void __gen8_do_map_pt (gen8_ppgtt_pde_t *pde, +struct i915_page_table_entry *pt, +struct drm_device *dev) +{ +gen8_ppgtt_pde_t entry = +gen8_pde_encode (dev, pt->daddr,I915_CACHE_LLC ); + *pde = entry; +} + +/* It's likely we'll map more than one pagetable at a time. This function will +* save us unnecessary kmap calls, but do no more functionally than multiple +* calls to map_pt. */ +static void gen8_map_pagetable_range (struct i915_page_directory_entry *pd, +uint64_t start, +uint64_t length, +struct drm_device *dev) +{ +gen8_ppgtt_pde_t *page_directory =kmap_atomic (pd->page); +struct i915_page_table_entry *pt; +uint64_t temp, pde; + +gen8_for_each_pde (pt, pd, start, length, temp, pde) +__gen8_do_map_pt (page_directory + pde, pt, dev); + +if (!HAS_LLC (dev)) +drm_clflush_virt_range (page_directory,PAGE_SIZE ); + +kunmap_atomic (page_directory); +} +static void gen8_teardown_va_range (struct i915_address_space *vm,uint64_t start,uint64_t length) {
@@ -598,7 +638,7 @@ static void gen8_teardown_va_range(struct i915_address_space *vm,unmap_and_free_pt (pt, vm->dev); pd->page_tables[pde] =NULL ; } -unmap_and_free_pd (pd); +unmap_and_free_pd (pd, vm->dev); ppgtt->pdp.page_directory[pdpe] =NULL ; } }
@@ -630,9 +670,6 @@ static void gen8_ppgtt_unmap_pages(struct i915_hw_ppgtt *ppgtt)static void gen8_ppgtt_free (struct i915_hw_ppgtt *ppgtt) { -trace_i915_va_teardown (&ppgtt->base, - ppgtt->base.start, ppgtt->base.total, -VM_TO_TRACE_NAME (&ppgtt->base));gen8_teardown_va_range (&ppgtt->base, ppgtt->base.start, ppgtt->base.total); }
@@ -673,7 +710,8 @@ unwind_out:static int gen8_ppgtt_alloc_page_directories (struct i915_page_directory_pointer_entry *pdp,uint64_t start, -uint64_t length) +uint64_t length, +struct drm_device *dev) {struct i915_page_directory_entry *unused;uint64_t temp;
@@ -684,7 +722,7 @@ static int gen8_ppgtt_alloc_page_directories(struct i915_page_directory_pointer_gen8_for_each_pdpe (unused, pdp, start, length, temp, pdpe) {BUG_ON (unused); - pdp->page_directory[pdpe] =alloc_pd_single (); + pdp->page_directory[pdpe] =alloc_pd_single (dev);if (IS_ERR (pdp->page_directory[pdpe])) goto unwind_out; }
@@ -693,21 +731,25 @@ static int gen8_ppgtt_alloc_page_directories(struct i915_page_directory_pointer_ unwind_out:while (pdpe--) -unmap_and_free_pd (pdp->page_directory[pdpe]); +unmap_and_free_pd (pdp->page_directory[pdpe], dev);return -ENOMEM ; } -static int gen8_ppgtt_alloc (struct i915_hw_ppgtt *ppgtt, -uint64_t start, -uint64_t length) +static int gen8_alloc_va_range (struct i915_address_space *vm, +uint64_t start, +uint64_t length) { +struct i915_hw_ppgtt *ppgtt = + container_of(vm, struct i915_hw_ppgtt, base);struct i915_page_directory_entry *pd; +const uint64_t orig_start = start;uint64_t temp;uint32_t pdpe;int ret; - ret =gen8_ppgtt_alloc_page_directories (&ppgtt->pdp, start, length); + ret =gen8_ppgtt_alloc_page_directories (&ppgtt->pdp, start, length, + ppgtt->base.dev);if (ret)return ret;
@@ -720,133 +762,50 @@ static int gen8_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt,return 0 ; -/* TODO: Check this for all cases */ err_out: -gen8_ppgtt_free (ppgtt); +gen8_teardown_va_range (vm, orig_start, start);return ret; } -static int gen8_ppgtt_setup_page_directories (struct i915_hw_ppgtt *ppgtt, -const int pd) -{ -dma_addr_t pd_addr; -int ret; - - pd_addr =pci_map_page (ppgtt->base.dev->pdev, - ppgtt->pdp.page_directory[pd]->page,0 , -PAGE_SIZE ,PCI_DMA_BIDIRECTIONAL ); - - ret =pci_dma_mapping_error (ppgtt->base.dev->pdev, pd_addr); -if (ret) -return ret; - - ppgtt->pdp.page_directory[pd]->daddr = pd_addr; - -return 0 ; -} - -static int gen8_ppgtt_setup_page_tables (struct i915_hw_ppgtt *ppgtt, -const int pd, -const int pt) -{ -dma_addr_t pt_addr; -struct i915_page_directory_entry *pdir = ppgtt->pdp.page_directory[pd]; -struct i915_page_table_entry *ptab = pdir->page_tables[pt]; -struct page *p = ptab->page; -int ret; - - pt_addr =pci_map_page (ppgtt->base.dev->pdev, - p,0 ,PAGE_SIZE ,PCI_DMA_BIDIRECTIONAL ); - ret =pci_dma_mapping_error (ppgtt->base.dev->pdev, pt_addr); -if (ret) -return ret; - - ptab->daddr = pt_addr; - -return 0 ; -} -/** * GEN8 legacy ppgtt programming is accomplished through a max 4 PDP registers * with a net effect resembling a 2-level page table in normal x86 terms. Each * PDP represents 1GB of memory 4 * 512 * 512 * 4096 = 4GB legacy 32b address * space. * -* FIXME: split allocation into smaller pieces. For now we only ever do this -* once, but with full PPGTT, the multiple contiguous allocations will be bad. -* TODO: Do something with the size parameter */ static int gen8_ppgtt_init (struct i915_hw_ppgtt *ppgtt,uint64_t size) { -const int max_pdp =DIV_ROUND_UP (size,1 <<30 ); -int i, j, ret; - -if (size % (1 <<30 )) -DRM_INFO ("Pages will be wasted unless GTT size (%llu) is divisible by 1GB\n" , size); +struct i915_page_directory_entry *pd; +uint64_t temp, start =0 ; +const uint64_t orig_length = size; +uint32_t pdpe; +int ret; ppgtt->base.start =0 ; ppgtt->base.total = size; + ppgtt->base.clear_range = gen8_ppgtt_clear_range; + ppgtt->base.insert_entries = gen8_ppgtt_insert_entries; + ppgtt->base.cleanup = gen8_ppgtt_cleanup; + ppgtt->switch_mm = gen8_mm_switch; ppgtt->scratch_pd =alloc_pt_scratch (ppgtt->base.dev);if (IS_ERR (ppgtt->scratch_pd))return PTR_ERR (ppgtt->scratch_pd); -/* 1. Do all our allocations for page directories and page tables. */ - ret =gen8_ppgtt_alloc (ppgtt, ppgtt->base.start, ppgtt->base.total); + ret =gen8_alloc_va_range (&ppgtt->base, start, size);if (ret) {unmap_and_free_pt (ppgtt->scratch_pd, ppgtt->base.dev);return ret; } -/* -* 2. Create DMA mappings for the page directories and page tables. -*/ -for (i =0 ; i < max_pdp; i++) { - ret =gen8_ppgtt_setup_page_directories (ppgtt, i); -if (ret) - goto bail; + start =0 ; + size = orig_length; -for (j =0 ; j <GEN8_PDES_PER_PAGE ; j++) { - ret =gen8_ppgtt_setup_page_tables (ppgtt, i, j); -if (ret) - goto bail; - } - } - -/* -* 3. Map all the page directory entires to point to the page tables -* we've allocated. -* -* For now, the PPGTT helper functions all require that the PDEs are -* plugged in correctly. So we do that now/here. For aliasing PPGTT, we -* will never need to touch the PDEs again. -*/ -for (i =0 ; i < max_pdp; i++) { -struct i915_page_directory_entry *pd = ppgtt->pdp.page_directory[i]; -gen8_ppgtt_pde_t *pd_vaddr; - pd_vaddr =kmap_atomic (ppgtt->pdp.page_directory[i]->page); -for (j =0 ; j <GEN8_PDES_PER_PAGE ; j++) { -struct i915_page_table_entry *pt = pd->page_tables[j]; -dma_addr_t addr = pt->daddr; - pd_vaddr[j] =gen8_pde_encode (ppgtt->base.dev, addr, -I915_CACHE_LLC ); - } -if (!HAS_LLC (ppgtt->base.dev)) -drm_clflush_virt_range (pd_vaddr,PAGE_SIZE ); -kunmap_atomic (pd_vaddr); - } - - ppgtt->switch_mm = gen8_mm_switch; - ppgtt->base.clear_range = gen8_ppgtt_clear_range; - ppgtt->base.insert_entries = gen8_ppgtt_insert_entries; - ppgtt->base.cleanup = gen8_ppgtt_cleanup; +gen8_for_each_pdpe (pd, &ppgtt->pdp, start, size, temp, pdpe) +gen8_map_pagetable_range (pd, start, size, ppgtt->base.dev);return 0 ; - -bail: -gen8_ppgtt_unmap_pages (ppgtt); -gen8_ppgtt_free (ppgtt); -return ret; }static void gen6_dump_ppgtt (struct i915_hw_ppgtt *ppgtt,struct seq_file *m)
@@ -1286,7 +1245,7 @@ static void gen6_ppgtt_free(struct i915_hw_ppgtt *ppgtt) }unmap_and_free_pt (ppgtt->scratch_pt, ppgtt->base.dev); -unmap_and_free_pd (&ppgtt->pd); +unmap_and_free_pd (&ppgtt->pd, ppgtt->base.dev); }static void gen6_ppgtt_cleanup (struct i915_address_space *vm)
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH v3 23/25 drm/i915/bdw: begin bitmap tracking
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
Like with gen6/7, we can enable bitmap tracking with all the preallocations to make sure things actually don't blow up.
v2: Rebased to match changes from previous patches.
--- drivers/gpu/drm/i915/i915_gem_gtt.c | 121 +++++++++++++++++++++++++----------- drivers/gpu/drm/i915/i915_gem_gtt.h | 24 +++++++ 2 files changed, 108 insertions(+), 37 deletions(-)
drivers/gpu/drm/i915/i915_gem_gtt.c +84 -37
@@ -424,8 +424,12 @@ static void unmap_and_free_pd(struct i915_page_directory_entry *pd, struct drm_device *dev) {if (pd->page) { +WARN (!bitmap_empty (pd->used_pdes,GEN8_PDES_PER_PAGE ), +"Free page directory with %d used pages\n" , +bitmap_weight (pd->used_pdes,GEN8_PDES_PER_PAGE ));i915_dma_unmap_single (pd, dev);__free_page (pd->page); +kfree (pd->used_pdes);kfree (pd); } }
@@ -433,26 +437,35 @@ static void unmap_and_free_pd(struct i915_page_directory_entry *pd,static struct i915_page_directory_entry *alloc_pd_single (struct drm_device *dev) {struct i915_page_directory_entry *pd; -int ret; +int ret = -ENOMEM ; pd =kzalloc (sizeof (*pd),GFP_KERNEL );if (!pd)return ERR_PTR (-ENOMEM ); + pd->used_pdes =kcalloc (BITS_TO_LONGS (GEN8_PDES_PER_PAGE ), +sizeof (*pd->used_pdes),GFP_KERNEL ); +if (!pd->used_pdes) + goto free_pd; + pd->page =alloc_page (GFP_KERNEL | __GFP_ZERO); -if (!pd->page) { -kfree (pd); -return ERR_PTR (-ENOMEM ); - } +if (!pd->page) + goto free_bitmap; ret =i915_dma_map_px_single (pd, dev); -if (ret) { -__free_page (pd->page); -kfree (pd); -return ERR_PTR (ret); - } +if (ret) + goto free_page;return pd; + +free_page: +__free_page (pd->page); +free_bitmap: +kfree (pd->used_pdes); +free_pd: +kfree (pd); + +return ERR_PTR (ret); }/* Broadwell Page Directory Pointer Descriptors */
@@ -634,36 +647,47 @@ static void gen8_teardown_va_range(struct i915_address_space *vm,gen8_for_each_pdpe (pd, &ppgtt->pdp, start, length, temp, pdpe) {uint64_t pd_len =gen8_clamp_pd (start, length);uint64_t pd_start = start; -gen8_for_each_pde (pt, pd, pd_start, pd_len, temp, pde) { -unmap_and_free_pt (pt, vm->dev); - pd->page_tables[pde] =NULL ; - } -unmap_and_free_pd (pd, vm->dev); - ppgtt->pdp.page_directory[pdpe] =NULL ; - } -} -static void gen8_ppgtt_unmap_pages (struct i915_hw_ppgtt *ppgtt) -{ -struct pci_dev *hwdev = ppgtt->base.dev->pdev; -int i, j; - -for (i =0 ; i <GEN8_PDES_PER_PAGE ; i++) { -/* TODO: In the future we'll support sparse mappings, so this -* will have to change. */ -if (!ppgtt->pdp.page_directory[i]->daddr) +/* Page directories might not be present since the macro rounds +* down, and up. +*/ +if (!pd) { +WARN (test_bit (pdpe, ppgtt->pdp.used_pdpes), +"PDPE %d is not allocated, but is reserved (%p)\n" , + pdpe, vm);continue ; + }else { +WARN (!test_bit (pdpe, ppgtt->pdp.used_pdpes), +"PDPE %d not reserved, but is allocated (%p)" , + pdpe, vm); + } -pci_unmap_page (hwdev, ppgtt->pdp.page_directory[i]->daddr,PAGE_SIZE , -PCI_DMA_BIDIRECTIONAL ); +gen8_for_each_pde (pt, pd, pd_start, pd_len, temp, pde) { +if (!pt) { +WARN (test_bit (pde, pd->used_pdes), +"PDE %d is not allocated, but is reserved (%p)\n" , + pde, vm); +continue ; + }else +WARN (!test_bit (pde, pd->used_pdes), +"PDE %d not reserved, but is allocated (%p)" , + pde, vm); + +bitmap_clear (pt->used_ptes, +gen8_pte_index (pd_start), +gen8_pte_count (pd_start, pd_len)); + +if (bitmap_empty (pt->used_ptes,GEN8_PTES_PER_PAGE )) { +unmap_and_free_pt (pt, vm->dev); + pd->page_tables[pde] =NULL ; +WARN_ON (!test_and_clear_bit (pde, pd->used_pdes)); + } + } -for (j =0 ; j <GEN8_PDES_PER_PAGE ; j++) { -struct i915_page_directory_entry *pd = ppgtt->pdp.page_directory[i]; -struct i915_page_table_entry *pt = pd->page_tables[j]; -dma_addr_t addr = pt->daddr; -if (addr) -pci_unmap_page (hwdev, addr,PAGE_SIZE , -PCI_DMA_BIDIRECTIONAL ); +if (bitmap_empty (pd->used_pdes,GEN8_PDES_PER_PAGE )) { +unmap_and_free_pd (pd, vm->dev); + ppgtt->pdp.page_directory[pdpe] =NULL ; +WARN_ON (!test_and_clear_bit (pdpe, ppgtt->pdp.used_pdpes)); } } }
@@ -679,7 +703,6 @@ static void gen8_ppgtt_cleanup(struct i915_address_space *vm)struct i915_hw_ppgtt *ppgtt = container_of(vm, struct i915_hw_ppgtt, base); -gen8_ppgtt_unmap_pages (ppgtt);gen8_ppgtt_free (ppgtt); }
@@ -708,6 +731,7 @@ unwind_out:return -ENOMEM ; } +/* bitmap of new page_directories */ static int gen8_ppgtt_alloc_page_directories(struct i915_page_directory_pointer_entry *pdp, uint64_t start, uint64_t length,
@@ -723,6 +747,7 @@ static int gen8_ppgtt_alloc_page_directories(struct i915_page_directory_pointer_gen8_for_each_pdpe (unused, pdp, start, length, temp, pdpe) {BUG_ON (unused); pdp->page_directory[pdpe] =alloc_pd_single (dev); +if (IS_ERR (pdp->page_directory[pdpe])) goto unwind_out; }
@@ -744,10 +769,12 @@ static int gen8_alloc_va_range(struct i915_address_space *vm, container_of(vm, struct i915_hw_ppgtt, base);struct i915_page_directory_entry *pd;const uint64_t orig_start = start; +const uint64_t orig_length = length;uint64_t temp;uint32_t pdpe;int ret; +/* Do the allocations first so we can easily bail out */ ret =gen8_ppgtt_alloc_page_directories (&ppgtt->pdp, start, length, ppgtt->base.dev);if (ret)
@@ -760,6 +787,26 @@ static int gen8_alloc_va_range(struct i915_address_space *vm, goto err_out; } +/* Now mark everything we've touched as used. This doesn't allow for +* robust error checking, but it makes the code a hell of a lot simpler. +*/ + start = orig_start; + length = orig_length; + +gen8_for_each_pdpe (pd, &ppgtt->pdp, start, length, temp, pdpe) { +struct i915_page_table_entry *pt; +uint64_t pd_len =gen8_clamp_pd (start, length); +uint64_t pd_start = start; +uint32_t pde; +gen8_for_each_pde (pt, &ppgtt->pd, pd_start, pd_len, temp, pde) { +bitmap_set (pd->page_tables[pde]->used_ptes, +gen8_pte_index (start), +gen8_pte_count (start, length)); +set_bit (pde, pd->used_pdes); + } +set_bit (pdpe, ppgtt->pdp.used_pdpes); + } +return 0 ; err_out:
drivers/gpu/drm/i915/i915_gem_gtt.h +24 -0
@@ -205,11 +205,13 @@ struct i915_page_directory_entry {dma_addr_t daddr; }; +unsigned long *used_pdes;struct i915_page_table_entry *page_tables[GEN6_PPGTT_PD_ENTRIES ];/* PDEs */ };struct i915_page_directory_pointer_entry {/* struct page *page; */ + DECLARE_BITMAP(used_pdpes, GEN8_LEGACY_PDPES);struct i915_page_directory_entry *page_directory[GEN8_LEGACY_PDPES ]; };
@@ -447,6 +449,28 @@ static inline uint32_t gen8_pml4e_index(uint64_t address)BUG ();/* For 64B */ } +static inline size_t gen8_pte_count (uint64_t addr,uint64_t length) +{ +return i915_pte_count (addr, length,GEN8_PDE_SHIFT ); +} + +static inline size_t gen8_pde_count (uint64_t addr,uint64_t length) +{ +const uint32_t pdp_shift =GEN8_PDE_SHIFT +9 ; +const uint64_t mask = ~((1 << pdp_shift) -1 ); +uint64_t end; + +BUG_ON (length ==0 ); +BUG_ON (offset_in_page (addr|length)); + + end = addr + length; + +if ((addr & mask) != (end & mask)) +return GEN8_PDES_PER_PAGE -i915_pde_index (addr,GEN8_PDE_SHIFT ); + +return i915_pde_index (end,GEN8_PDE_SHIFT ) -i915_pde_index (addr,GEN8_PDE_SHIFT ); +} +int i915_gem_gtt_init (struct drm_device *dev);void i915_gem_init_global_gtt (struct drm_device *dev);void i915_global_gtt_cleanup (struct drm_device *dev);
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH v3 24/25 drm/i915/bdw: Dynamic page table allocations
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
This finishes off the dynamic page tables allocations, in the legacy 3 level style that already exists. Most everything has already been setup to this point, the patch finishes off the enabling by setting the appropriate function pointers.
Zombie tracking:
This could be a separate patch, but I found it helpful for debugging. Since we write page tables asynchronously with respect to the GPU using them, we can't actually free the page tables until we know the GPU won't use them. With this patch, that is always when the context dies.
Scratch unused pages:
The object pages can get freed even if a page table still points to them. Like the zombie fix, we need to make sure we don't let our GPU access arbitrary memory when we've unmapped things.
v2: Update aliasing/true ppgtt allocate/teardown/clear functions for gen 6 & 7.
v3: Rebase.
v4: Remove BUG() from ppgtt_unbind_vma, but keep checking that either teardown_va_range or clear_range functions exist (Daniel).
--- drivers/gpu/drm/i915/i915_gem_gtt.c | 375 +++++++++++++++++++++++++++++------- drivers/gpu/drm/i915/i915_gem_gtt.h | 15 ++ 2 files changed, 325 insertions(+), 65 deletions(-)
drivers/gpu/drm/i915/i915_gem_gtt.c +310 -65
@@ -604,7 +604,7 @@ static void gen8_ppgtt_insert_entries(struct i915_address_space *vm, } } -static void __gen8_do_map_pt (gen8_ppgtt_pde_t *pde, +static void __gen8_do_map_pt (gen8_ppgtt_pde_t *const pde,struct i915_page_table_entry *pt,struct drm_device *dev) {
@@ -621,7 +621,7 @@ static void gen8_map_pagetable_range(struct i915_page_directory_entry *pd,uint64_t length, struct drm_device *dev) { - gen8_ppgtt_pde_t *page_directory = kmap_atomic(pd->page); + gen8_ppgtt_pde_t * const page_directory = kmap_atomic(pd->page);struct i915_page_table_entry *pt;uint64_t temp, pde;
@@ -634,8 +634,9 @@ static void gen8_map_pagetable_range(struct i915_page_directory_entry *pd,kunmap_atomic (page_directory); } -static void gen8_teardown_va_range (struct i915_address_space *vm, -uint64_t start,uint64_t length) +static void __gen8_teardown_va_range (struct i915_address_space *vm, +uint64_t start,uint64_t length, +bool dead) {struct i915_hw_ppgtt *ppgtt = container_of(vm, struct i915_hw_ppgtt, base);
@@ -657,6 +658,13 @@ static void gen8_teardown_va_range(struct i915_address_space *vm, pdpe, vm);continue ; }else { +if (dead && pd->zombie) { +WARN_ON (test_bit (pdpe, ppgtt->pdp.used_pdpes)); +unmap_and_free_pd (pd, vm->dev); + ppgtt->pdp.page_directory[pdpe] =NULL ; +continue ; + } +WARN (!test_bit (pdpe, ppgtt->pdp.used_pdpes),"PDPE %d not reserved, but is allocated (%p)" , pdpe, vm);
@@ -668,34 +676,64 @@ static void gen8_teardown_va_range(struct i915_address_space *vm,"PDE %d is not allocated, but is reserved (%p)\n" , pde, vm);continue ; - }else + }else { +if (dead && pt->zombie) { +WARN_ON (test_bit (pde, pd->used_pdes)); +unmap_and_free_pt (pt, vm->dev); + pd->page_tables[pde] =NULL ; +continue ; + }WARN (!test_bit (pde, pd->used_pdes),"PDE %d not reserved, but is allocated (%p)" , pde, vm); + }bitmap_clear (pt->used_ptes,gen8_pte_index (pd_start),gen8_pte_count (pd_start, pd_len));if (bitmap_empty (pt->used_ptes,GEN8_PTES_PER_PAGE )) { +WARN_ON (!test_and_clear_bit (pde, pd->used_pdes)); +if (!dead) { + pt->zombie =1 ; +continue ; + }unmap_and_free_pt (pt, vm->dev); pd->page_tables[pde] =NULL ; -WARN_ON (!test_and_clear_bit (pde, pd->used_pdes)); + } } +gen8_ppgtt_clear_range (vm, pd_start, pd_len, true); +if (bitmap_empty (pd->used_pdes,GEN8_PDES_PER_PAGE )) { +WARN_ON (!test_and_clear_bit (pdpe, ppgtt->pdp.used_pdpes)); +if (!dead) { +/* We've unmapped a possibly live context. Make +* note of it so we can clean it up later. */ + pd->zombie =1 ; +continue ; + }unmap_and_free_pd (pd, vm->dev); ppgtt->pdp.page_directory[pdpe] =NULL ; -WARN_ON (!test_and_clear_bit (pdpe, ppgtt->pdp.used_pdpes)); } } } +static void gen8_teardown_va_range (struct i915_address_space *vm, +uint64_t start,uint64_t length) +{ +__gen8_teardown_va_range (vm, start, length, false); +} +static void gen8_ppgtt_free (struct i915_hw_ppgtt *ppgtt) { -gen8_teardown_va_range (&ppgtt->base, - ppgtt->base.start, ppgtt->base.total); +trace_i915_va_teardown (&ppgtt->base, + ppgtt->base.start, ppgtt->base.total, +VM_TO_TRACE_NAME (&ppgtt->base)); +__gen8_teardown_va_range (&ppgtt->base, + ppgtt->base.start, ppgtt->base.total, + true); }static void gen8_ppgtt_cleanup (struct i915_address_space *vm)
@@ -706,67 +744,177 @@ static void gen8_ppgtt_cleanup(struct i915_address_space *vm)gen8_ppgtt_free (ppgtt); } -static int gen8_ppgtt_alloc_pagetabs (struct i915_page_directory_entry *pd, +/** +* gen8_ppgtt_alloc_pagetabs() - Allocate page tables for VA range. +* @ppgtt: Master ppgtt structure. +* @pd: Page directory for this address range. +* @start: Starting virtual address to begin allocations. +* @length Size of the allocations. +* @new_pts: Bitmap set by function with new allocations. Likely used by the +* caller to free on error. +* +* Allocate the required number of page tables. Extremely similar to +* gen8_ppgtt_alloc_page_directories(). The main difference is here we are limited by +* the page directory boundary (instead of the page directory pointer). That +* boundary is 1GB virtual. Therefore, unlike gen8_ppgtt_alloc_page_directories(), it is +* possible, and likely that the caller will need to use multiple calls of this +* function to achieve the appropriate allocation. +* +* Return: 0 if success; negative error code otherwise. +*/ +static int gen8_ppgtt_alloc_pagetabs (struct i915_hw_ppgtt *ppgtt, +struct i915_page_directory_entry *pd,uint64_t start,uint64_t length, -struct drm_device *dev) +unsigned long *new_pts) { -struct i915_page_table_entry *unused; +struct i915_page_table_entry *pt;uint64_t temp;uint32_t pde; -gen8_for_each_pde (unused, pd, start, length, temp, pde) { -BUG_ON (unused); - pd->page_tables[pde] =alloc_pt_single (dev); -if (IS_ERR (pd->page_tables[pde])) +gen8_for_each_pde (pt, pd, start, length, temp, pde) { +/* Don't reallocate page tables */ +if (pt) { +/* Scratch is never allocated this way */ +WARN_ON (pt->scratch); +/* If there is a zombie, we can reuse it and save time +* on the allocation. If we clear the zombie status and +* the caller somehow fails, we'll probably hit some +* assertions, so it's up to them to fix up the bitmaps. +*/ +continue ; + } + + pt =alloc_pt_single (ppgtt->base.dev); +if (IS_ERR (pt)) goto unwind_out; + + pd->page_tables[pde] = pt; +set_bit (pde, new_pts); }return 0 ; unwind_out: -while (pde--) -unmap_and_free_pt (pd->page_tables[pde], dev); +for_each_set_bit (pde, new_pts,GEN8_PDES_PER_PAGE ) +unmap_and_free_pt (pd->page_tables[pde], ppgtt->base.dev);return -ENOMEM ; } -/* bitmap of new page_directories */ -static int gen8_ppgtt_alloc_page_directories (struct i915_page_directory_pointer_entry *pdp, +/** +* gen8_ppgtt_alloc_page_directories() - Allocate page directories for VA range. +* @ppgtt: Master ppgtt structure. +* @pdp: Page directory pointer for this address range. +* @start: Starting virtual address to begin allocations. +* @length Size of the allocations. +* @new_pds Bitmap set by function with new allocations. Likely used by the +* caller to free on error. +* +* Allocate the required number of page directories starting at the pde index of +* @start, and ending at the pde index @start + @length. This function will skip +* over already allocated page directories within the range, and only allocate +* new ones, setting the appropriate pointer within the pdp as well as the +* correct position in the bitmap @new_pds. +* +* The function will only allocate the pages within the range for a give page +* directory pointer. In other words, if @start + @length straddles a virtually +* addressed PDP boundary (512GB for 4k pages), there will be more allocations +* required by the caller, This is not currently possible, and the BUG in the +* code will prevent it. +* +* Return: 0 if success; negative error code otherwise. +*/ +static int gen8_ppgtt_alloc_page_directories (struct i915_hw_ppgtt *ppgtt, +struct i915_page_directory_pointer_entry *pdp,uint64_t start,uint64_t length, -struct drm_device *dev) +unsigned long *new_pds) { -struct i915_page_directory_entry *unused; +struct i915_page_directory_entry *pd;uint64_t temp;uint32_t pdpe; +BUG_ON (!bitmap_empty (new_pds,GEN8_LEGACY_PDPES )); +/* FIXME: PPGTT container_of won't work for 64b */ BUG_ON ((start + length) >0x800000000ULL ); -gen8_for_each_pdpe (unused, pdp, start, length, temp, pdpe) { -BUG_ON (unused); - pdp->page_directory[pdpe] =alloc_pd_single (dev); +gen8_for_each_pdpe (pd, pdp, start, length, temp, pdpe) { +if (pd) +continue ; -if (IS_ERR (pdp->page_directory[pdpe])) + pd =alloc_pd_single (ppgtt->base.dev); +if (IS_ERR (pd)) goto unwind_out; + + pdp->page_directory[pdpe] = pd; +set_bit (pdpe, new_pds); }return 0 ; unwind_out: -while (pdpe--) -unmap_and_free_pd (pdp->page_directory[pdpe], dev); +for_each_set_bit (pdpe, new_pds,GEN8_LEGACY_PDPES ) +unmap_and_free_pd (pdp->page_directory[pdpe], ppgtt->base.dev);return -ENOMEM ; } +static inline void +free_gen8_temp_bitmaps (unsigned long *new_pds,unsigned long **new_pts) +{ +int i; +for (i =0 ; i <GEN8_LEGACY_PDPES ; i++) +kfree (new_pts[i]); +kfree (new_pts); +kfree (new_pds); +} + +/* Fills in the page directory bitmap, ant the array of page tables bitmap. Both +* of these are based on the number of PDPEs in the system. +*/ +int __must_check alloc_gen8_temp_bitmaps (unsigned long **new_pds, +unsigned long ***new_pts) +{ +int i; +unsigned long *pds; +unsigned long **pts; + + pds =kcalloc (BITS_TO_LONGS (GEN8_LEGACY_PDPES ),sizeof (unsigned long ),GFP_KERNEL ); +if (!pds) +return -ENOMEM ; + + pts =kcalloc (GEN8_PDES_PER_PAGE ,sizeof (unsigned long *),GFP_KERNEL ); +if (!pts) { +kfree (pds); +return -ENOMEM ; + } + +for (i =0 ; i <GEN8_LEGACY_PDPES ; i++) { + pts[i] =kcalloc (BITS_TO_LONGS (GEN8_PDES_PER_PAGE ), +sizeof (unsigned long ),GFP_KERNEL ); +if (!pts[i]) + goto err_out; + } + + *new_pds = pds; + *new_pts = (unsigned long **)pts; + +return 0 ; + +err_out: +free_gen8_temp_bitmaps (pds, pts); +return -ENOMEM ; +} +static int gen8_alloc_va_range (struct i915_address_space *vm,uint64_t start,uint64_t length) {struct i915_hw_ppgtt *ppgtt = container_of(vm, struct i915_hw_ppgtt, base); +unsigned long *new_page_dirs, **new_page_tables;struct i915_page_directory_entry *pd;const uint64_t orig_start = start;const uint64_t orig_length = length;
@@ -774,43 +922,103 @@ static int gen8_alloc_va_range(struct i915_address_space *vm,uint32_t pdpe;int ret; -/* Do the allocations first so we can easily bail out */ - ret =gen8_ppgtt_alloc_page_directories (&ppgtt->pdp, start, length, - ppgtt->base.dev); +#ifndef CONFIG_64BIT +/* Disallow 64b address on 32b platforms. Nothing is wrong with doing +* this in hardware, but a lot of the drm code is not prepared to handle +* 64b offset on 32b platforms. */ +if (start + length >0x100000000ULL ) +return -E2BIG ; +#endif + +/* Wrap is never okay since we can only represent 48b, and we don't +* actually use the other side of the canonical address space. +*/ +if (WARN_ON (start + length < start)) +return -ERANGE ; + + ret =alloc_gen8_temp_bitmaps (&new_page_dirs, &new_page_tables);if (ret)return ret; +/* Do the allocations first so we can easily bail out */ + ret =gen8_ppgtt_alloc_page_directories (ppgtt, &ppgtt->pdp, start, length, + new_page_dirs); +if (ret) { +free_gen8_temp_bitmaps (new_page_dirs, new_page_tables); +return ret; + } + +/* For every page directory referenced, allocate page tables */ gen8_for_each_pdpe (pd, &ppgtt->pdp, start, length, temp, pdpe) { - ret =gen8_ppgtt_alloc_pagetabs (pd, start, length, - ppgtt->base.dev); +bitmap_zero (new_page_tables[pdpe],GEN8_PDES_PER_PAGE ); + ret =gen8_ppgtt_alloc_pagetabs (ppgtt, pd, start, length, + new_page_tables[pdpe]);if (ret) goto err_out; } -/* Now mark everything we've touched as used. This doesn't allow for -* robust error checking, but it makes the code a hell of a lot simpler. -*/ start = orig_start; length = orig_length; +/* Allocations have completed successfully, so set the bitmaps, and do +* the mappings. */ gen8_for_each_pdpe (pd, &ppgtt->pdp, start, length, temp, pdpe) { +gen8_ppgtt_pde_t *const page_directory =kmap_atomic (pd->page);struct i915_page_table_entry *pt;uint64_t pd_len =gen8_clamp_pd (start, length);uint64_t pd_start = start;uint32_t pde; -gen8_for_each_pde (pt, &ppgtt->pd, pd_start, pd_len, temp, pde) { -bitmap_set (pd->page_tables[pde]->used_ptes, -gen8_pte_index (start), -gen8_pte_count (start, length)); + +/* Every pd should be allocated, we just did that above. */ +BUG_ON (!pd); + +gen8_for_each_pde (pt, pd, pd_start, pd_len, temp, pde) { +/* Same reasoning as pd */ +BUG_ON (!pt); +BUG_ON (!pd_len); +BUG_ON (!gen8_pte_count (pd_start, pd_len)); + +/* Set our used ptes within the page table */ +bitmap_set (pt->used_ptes, +gen8_pte_index (pd_start), +gen8_pte_count (pd_start, pd_len)); + +/* Our pde is now pointing to the pagetable, pt */ set_bit (pde, pd->used_pdes); + +/* Map the PDE to the page table */ +__gen8_do_map_pt (page_directory + pde, pt, vm->dev); + +/* NB: We haven't yet mapped ptes to pages. At this +* point we're still relying on insert_entries() */ + +/* No longer possible this page table is a zombie */ + pt->zombie =0 ; } + +if (!HAS_LLC (vm->dev)) +drm_clflush_virt_range (page_directory,PAGE_SIZE ); + +kunmap_atomic (page_directory); +set_bit (pdpe, ppgtt->pdp.used_pdpes); +/* This pd is officially not a zombie either */ + ppgtt->pdp.page_directory[pdpe]->zombie =0 ; } +free_gen8_temp_bitmaps (new_page_dirs, new_page_tables);return 0 ; err_out: -gen8_teardown_va_range (vm, orig_start, start); +while (pdpe--) { +for_each_set_bit (temp, new_page_tables[pdpe],GEN8_PDES_PER_PAGE ) +unmap_and_free_pt (pd->page_tables[temp], vm->dev); + } + +for_each_set_bit (pdpe, new_page_dirs,GEN8_LEGACY_PDPES ) +unmap_and_free_pd (ppgtt->pdp.page_directory[pdpe], vm->dev); + +free_gen8_temp_bitmaps (new_page_dirs, new_page_tables);return ret; }
@@ -821,37 +1029,68 @@ err_out:* space. * */ -static int gen8_ppgtt_init (struct i915_hw_ppgtt *ppgtt,uint64_t size) +static int gen8_ppgtt_init_common (struct i915_hw_ppgtt *ppgtt,uint64_t size) { -struct i915_page_directory_entry *pd; -uint64_t temp, start =0 ; -const uint64_t orig_length = size; -uint32_t pdpe; -int ret; + ppgtt->scratch_pd =alloc_pt_scratch (ppgtt->base.dev); +if (IS_ERR (ppgtt->scratch_pd)) +return PTR_ERR (ppgtt->scratch_pd); ppgtt->base.start =0 ; ppgtt->base.total = size; - ppgtt->base.clear_range = gen8_ppgtt_clear_range; - ppgtt->base.insert_entries = gen8_ppgtt_insert_entries; ppgtt->base.cleanup = gen8_ppgtt_cleanup; + ppgtt->base.insert_entries = gen8_ppgtt_insert_entries; + ppgtt->switch_mm = gen8_mm_switch; - ppgtt->scratch_pd =alloc_pt_scratch (ppgtt->base.dev); -if (IS_ERR (ppgtt->scratch_pd)) -return PTR_ERR (ppgtt->scratch_pd); +return 0 ; +} + +static int gen8_aliasing_ppgtt_init (struct i915_hw_ppgtt *ppgtt) +{ +struct drm_device *dev = ppgtt->base.dev; +struct drm_i915_private *dev_priv = dev->dev_private; +struct i915_page_directory_entry *pd; +uint64_t temp, start =0 , size = dev_priv->gtt.base.total; +uint32_t pdpe; +int ret; + + ret =gen8_ppgtt_init_common (ppgtt, dev_priv->gtt.base.total); +if (ret) +return ret; +/* Aliasing PPGTT has to always work and be mapped because of the way we +* use RESTORE_INHIBIT in the context switch. This will be fixed +* eventually. */ ret =gen8_alloc_va_range (&ppgtt->base, start, size);if (ret) {unmap_and_free_pt (ppgtt->scratch_pd, ppgtt->base.dev);return ret; } - start =0 ; - size = orig_length; -gen8_for_each_pdpe (pd, &ppgtt->pdp, start, size, temp, pdpe)gen8_map_pagetable_range (pd, start, size, ppgtt->base.dev); + ppgtt->base.allocate_va_range =NULL ; + ppgtt->base.teardown_va_range =NULL ; + ppgtt->base.clear_range = gen8_ppgtt_clear_range; + +return 0 ; +} + +static int gen8_ppgtt_init (struct i915_hw_ppgtt *ppgtt) +{ +struct drm_device *dev = ppgtt->base.dev; +struct drm_i915_private *dev_priv = dev->dev_private; +int ret; + + ret =gen8_ppgtt_init_common (ppgtt, dev_priv->gtt.base.total); +if (ret) +return ret; + + ppgtt->base.allocate_va_range = gen8_alloc_va_range; + ppgtt->base.teardown_va_range = gen8_teardown_va_range; + ppgtt->base.clear_range =NULL ; +return 0 ; }
@@ -1407,9 +1646,9 @@ static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt, bool aliasing) } } - ppgtt->base.allocate_va_range = gen6_alloc_va_range; - ppgtt->base.teardown_va_range = gen6_teardown_va_range; - ppgtt->base.clear_range = gen6_ppgtt_clear_range; + ppgtt->base.allocate_va_range = aliasing ?NULL : gen6_alloc_va_range; + ppgtt->base.teardown_va_range = aliasing ?NULL : gen6_teardown_va_range; + ppgtt->base.clear_range = aliasing ? gen6_ppgtt_clear_range :NULL ; ppgtt->base.insert_entries = gen6_ppgtt_insert_entries; ppgtt->base.cleanup = gen6_ppgtt_cleanup; ppgtt->base.start =0 ;
@@ -1447,8 +1686,10 @@ static int __hw_ppgtt_init(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt,if (INTEL_INFO (dev)->gen <8 )return gen6_ppgtt_init (ppgtt, aliasing); +else if (aliasing) +return gen8_aliasing_ppgtt_init (ppgtt);else -return gen8_ppgtt_init (ppgtt, dev_priv->gtt.base.total); +return gen8_ppgtt_init (ppgtt); }int i915_ppgtt_init (struct drm_device *dev,struct i915_hw_ppgtt *ppgtt) {
@@ -1460,8 +1701,9 @@ int i915_ppgtt_init(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt)kref_init (&ppgtt->ref);drm_mm_init (&ppgtt->base.mm, ppgtt->base.start, ppgtt->base.total); - ppgtt->base.clear_range (&ppgtt->base,0 , - ppgtt->base.total, true); +if (ppgtt->base.clear_range) + ppgtt->base.clear_range (&ppgtt->base,0 , + ppgtt->base.total, true);i915_init_vm (dev_priv, &ppgtt->base); }
@@ -1559,10 +1801,8 @@ ppgtt_bind_vma(struct i915_vma *vma,static void ppgtt_unbind_vma (struct i915_vma *vma) { - vma->vm->clear_range (vma->vm, - vma->node.start, - vma->obj->base.size, - true); +WARN_ON (vma->vm->teardown_va_range && vma->vm->clear_range); +WARN_ON (!vma->vm->teardown_va_range && !vma->vm->clear_range);if (vma->vm->teardown_va_range) { trace_i915_va_teardown(vma->vm, vma->node.start, vma->node.size,
@@ -1570,6 +1810,11 @@ static void ppgtt_unbind_vma(struct i915_vma *vma) vma->vm->teardown_va_range (vma->vm, vma->node.start, vma->node.size); + }else if (vma->vm->clear_range) { + vma->vm->clear_range (vma->vm, + vma->node.start, + vma->obj->base.size, + true); } }
drivers/gpu/drm/i915/i915_gem_gtt.h +15 -0
@@ -190,12 +190,26 @@ struct i915_vma { u32 flags); }; +/* Zombies. We write page tables with the CPU, and hardware switches them with +* the GPU. As such, the only time we can safely remove a page table is when we +* know the context is idle. Since we have no good way to do this, we use the +* zombie. +* +* Under memory pressure, if the system is idle, zombies may be reaped. +* +* There are 3 states a page table can be in (not including scratch) +* bitmap = 0, zombie = 0: unallocated +* bitmap = 1, zombie = 0: allocated +* bitmap = 0, zombie = 1: zombie +* bitmap = 1, zombie = 1: invalid +*/ struct i915_page_table_entry {struct page *page;dma_addr_t daddr;unsigned long *used_ptes;unsigned int scratch:1 ; +unsigned zombie:1 ; };struct i915_page_directory_entry {
@@ -207,6 +221,7 @@ struct i915_page_directory_entry {unsigned long *used_pdes;struct i915_page_table_entry *page_tables[GEN6_PPGTT_PD_ENTRIES ];/* PDEs */ + unsigned zombie:1; };struct i915_page_directory_pointer_entry {
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH v3 25/25 drm/i915/bdw: Support dynamic pdp updates in lrc mode
In reply to Michel ThierryLogic ring contexts need to know the PDPs when they are populated. With dynamic page table allocations, these PDPs may not exist yet.
Check if PDPs have been allocated and use the scratch page if they do not exist yet.
Before submission, update the PDPs in the logic ring context as PDPs have been allocated.
v2: Renamed commit title (Daniel)
--- drivers/gpu/drm/i915/intel_lrc.c | 80 +++++++++++++++++++++++++++++++++++----- 1 file changed, 70 insertions(+), 10 deletions(-)
drivers/gpu/drm/i915/intel_lrc.c +70 -10
@@ -358,6 +358,7 @@ static void execlists_elsp_write(struct intel_engine_cs *ring,static int execlists_update_context (struct drm_i915_gem_object *ctx_obj,struct drm_i915_gem_object *ring_obj, +struct i915_hw_ppgtt *ppgtt,u32 tail) {struct page *page;
@@ -369,6 +370,40 @@ static int execlists_update_context(struct drm_i915_gem_object *ctx_obj, reg_state[CTX_RING_TAIL +1 ] = tail; reg_state[CTX_RING_BUFFER_START +1 ] =i915_gem_obj_ggtt_offset (ring_obj); +/* True PPGTT with dynamic page allocation: update PDP registers and +* point the unallocated PDPs to the scratch page +*/ +if (ppgtt) { +if (test_bit (3 , ppgtt->pdp.used_pdpes)) { + reg_state[CTX_PDP3_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[3 ]->daddr); + reg_state[CTX_PDP3_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[3 ]->daddr); + }else { + reg_state[CTX_PDP3_UDW +1 ] =upper_32_bits (ppgtt->scratch_pd->daddr); + reg_state[CTX_PDP3_LDW +1 ] =lower_32_bits (ppgtt->scratch_pd->daddr); + } +if (test_bit (2 , ppgtt->pdp.used_pdpes)) { + reg_state[CTX_PDP2_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[2 ]->daddr); + reg_state[CTX_PDP2_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[2 ]->daddr); + }else { + reg_state[CTX_PDP2_UDW +1 ] =upper_32_bits (ppgtt->scratch_pd->daddr); + reg_state[CTX_PDP2_LDW +1 ] =lower_32_bits (ppgtt->scratch_pd->daddr); + } +if (test_bit (1 , ppgtt->pdp.used_pdpes)) { + reg_state[CTX_PDP1_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[1 ]->daddr); + reg_state[CTX_PDP1_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[1 ]->daddr); + }else { + reg_state[CTX_PDP1_UDW +1 ] =upper_32_bits (ppgtt->scratch_pd->daddr); + reg_state[CTX_PDP1_LDW +1 ] =lower_32_bits (ppgtt->scratch_pd->daddr); + } +if (test_bit (0 , ppgtt->pdp.used_pdpes)) { + reg_state[CTX_PDP0_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[0 ]->daddr); + reg_state[CTX_PDP0_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[0 ]->daddr); + }else { + reg_state[CTX_PDP0_UDW +1 ] =upper_32_bits (ppgtt->scratch_pd->daddr); + reg_state[CTX_PDP0_LDW +1 ] =lower_32_bits (ppgtt->scratch_pd->daddr); + } + } +kunmap_atomic (reg_state);return 0 ;
@@ -387,7 +422,7 @@ static void execlists_submit_contexts(struct intel_engine_cs *ring,WARN_ON (!i915_gem_obj_is_pinned (ctx_obj0));WARN_ON (!i915_gem_obj_is_pinned (ringbuf0->obj)); -execlists_update_context (ctx_obj0, ringbuf0->obj, tail0); +execlists_update_context (ctx_obj0, ringbuf0->obj, to0->ppgtt, tail0);if (to1) { ringbuf1 = to1->engine[ring->id].ringbuf;
@@ -396,7 +431,7 @@ static void execlists_submit_contexts(struct intel_engine_cs *ring,WARN_ON (!i915_gem_obj_is_pinned (ctx_obj1));WARN_ON (!i915_gem_obj_is_pinned (ringbuf1->obj)); -execlists_update_context (ctx_obj1, ringbuf1->obj, tail1); +execlists_update_context (ctx_obj1, ringbuf1->obj, to1->ppgtt, tail1); }execlists_elsp_write (ring, ctx_obj0, ctx_obj1);
@@ -1731,14 +1766,39 @@ populate_lr_context(struct intel_context *ctx, struct drm_i915_gem_object *ctx_o reg_state[CTX_PDP1_LDW ] =GEN8_RING_PDP_LDW (ring,1 ); reg_state[CTX_PDP0_UDW ] =GEN8_RING_PDP_UDW (ring,0 ); reg_state[CTX_PDP0_LDW ] =GEN8_RING_PDP_LDW (ring,0 ); - reg_state[CTX_PDP3_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[3 ]->daddr); - reg_state[CTX_PDP3_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[3 ]->daddr); - reg_state[CTX_PDP2_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[2 ]->daddr); - reg_state[CTX_PDP2_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[2 ]->daddr); - reg_state[CTX_PDP1_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[1 ]->daddr); - reg_state[CTX_PDP1_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[1 ]->daddr); - reg_state[CTX_PDP0_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[0 ]->daddr); - reg_state[CTX_PDP0_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[0 ]->daddr); + +/* With dynamic page allocation, PDPs may not be allocated at this point, +* Point the unallocated PDPs to the scratch page +*/ +if (test_bit (3 , ppgtt->pdp.used_pdpes)) { + reg_state[CTX_PDP3_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[3 ]->daddr); + reg_state[CTX_PDP3_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[3 ]->daddr); + }else { + reg_state[CTX_PDP3_UDW +1 ] =upper_32_bits (ppgtt->scratch_pd->daddr); + reg_state[CTX_PDP3_LDW +1 ] =lower_32_bits (ppgtt->scratch_pd->daddr); + } +if (test_bit (2 , ppgtt->pdp.used_pdpes)) { + reg_state[CTX_PDP2_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[2 ]->daddr); + reg_state[CTX_PDP2_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[2 ]->daddr); + }else { + reg_state[CTX_PDP2_UDW +1 ] =upper_32_bits (ppgtt->scratch_pd->daddr); + reg_state[CTX_PDP2_LDW +1 ] =lower_32_bits (ppgtt->scratch_pd->daddr); + } +if (test_bit (1 , ppgtt->pdp.used_pdpes)) { + reg_state[CTX_PDP1_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[1 ]->daddr); + reg_state[CTX_PDP1_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[1 ]->daddr); + }else { + reg_state[CTX_PDP1_UDW +1 ] =upper_32_bits (ppgtt->scratch_pd->daddr); + reg_state[CTX_PDP1_LDW +1 ] =lower_32_bits (ppgtt->scratch_pd->daddr); + } +if (test_bit (0 , ppgtt->pdp.used_pdpes)) { + reg_state[CTX_PDP0_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[0 ]->daddr); + reg_state[CTX_PDP0_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[0 ]->daddr); + }else { + reg_state[CTX_PDP0_UDW +1 ] =upper_32_bits (ppgtt->scratch_pd->daddr); + reg_state[CTX_PDP0_LDW +1 ] =lower_32_bits (ppgtt->scratch_pd->daddr); + } +if (ring->id ==RCS ) { reg_state[CTX_LRI_HEADER_2 ] =MI_LOAD_REGISTER_IMM (1 ); reg_state[CTX_R_PWR_CLK_STATE ] =0x20c8 ;
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
v4 cover PPGTT dynamic page allocations
In reply to Michel ThierryThis patchset continues addressing the comments from v2. In particular, it no longer teardowns pagetables dynamically, now happening until the vm is free.
For GEN8, it has also been extended to work in logical ring submission (lrc) mode, as it will be the preferred mode of operation.
I also tried to update the lrc code at the same time the ppgtt refactoring occurred, leaving only one patch that is exclusively for lrc.
This list can be seen in 3 parts:
[01-08] Include code rework for PPGTT (all GENs). [09-14] Adds page table allocation for GEN6/GEN7 [15-24] Enables dynamic allocation in GEN8,for both legacy and execlist submission modes.
Ben Widawsky (21):
drm/i915/trace: Fix offsets for 64b
drm/i915: Rename to GEN8_LEGACY_PDPES
drm/i915: Setup less PPGTT on failed page_directory drm/i915/gen8: Un-hardcode number of page directories drm/i915: page table abstractions
drm/i915: Complete page table structures
drm/i915: Create page table allocators
drm/i915: Track GEN6 page table usage
drm/i915: Extract context switch skip and pd load logic drm/i915: Track page table reload need
drm/i915: Initialize all contexts
drm/i915: Finish gen6/7 dynamic page table allocation drm/i915/bdw: Use dynamic allocation idioms on free drm/i915/bdw: page directories rework allocation drm/i915/bdw: pagetable allocation rework
drm/i915/bdw: Update pdp switch and point unused PDPs to scratch page drm/i915: num_pd_pages/num_pd_entries isn't useful drm/i915: Extract PPGTT param from page_directory alloc drm/i915/bdw: Split out mappings
drm/i915/bdw: begin bitmap tracking
drm/i915/bdw: Dynamic page table allocations
Michel Thierry (3):
drm/i915: Plumb drm_device through page tables operations drm/i915: Add dynamic page trace events drm/i915/bdw: Support dynamic pdp updates in lrc mode
drivers/gpu/drm/i915/i915_debugfs.c | 7 +- drivers/gpu/drm/i915/i915_gem.c | 11 + drivers/gpu/drm/i915/i915_gem_context.c | 64 +- drivers/gpu/drm/i915/i915_gem_execbuffer.c | 11 + drivers/gpu/drm/i915/i915_gem_gtt.c | 1064 ++++++++++++++++++++-------- drivers/gpu/drm/i915/i915_gem_gtt.h | 191 ++++- drivers/gpu/drm/i915/i915_trace.h | 107 ++- drivers/gpu/drm/i915/intel_lrc.c | 80 ++- 8 files changed, 1183 insertions(+), 352 deletions(-)
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH v4 1/24 drm/i915/trace: Fix offsets for 64b
From: Ben Widawsky <[email protected]>
--- drivers/gpu/drm/i915/i915_trace.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-)
drivers/gpu/drm/i915/i915_trace.h +4 -4
@@ -115,7 +115,7 @@ TRACE_EVENT(i915_vma_bind,TP_STRUCT__entry (__field (struct drm_i915_gem_object *,obj ) __field(struct i915_address_space *, vm) -__field (u32, offset) +__field (u64, offset)__field (u32, size)__field (unsigned, flags) ),
@@ -128,7 +128,7 @@ TRACE_EVENT(i915_vma_bind, __entry->flags = flags; ), -TP_printk ("obj=%p, offset=%08x size=%x%s vm=%p" , +TP_printk ("obj=%p, offset=%016llx size=%x%s vm=%p" , __entry->obj, __entry->offset, __entry->size, __entry->flags &PIN_MAPPABLE ?", mappable" :"" , __entry->vm)
@@ -141,7 +141,7 @@ TRACE_EVENT(i915_vma_unbind,TP_STRUCT__entry (__field (struct drm_i915_gem_object *,obj ) __field(struct i915_address_space *, vm) -__field (u32, offset) +__field (u64, offset)__field (u32, size) ),
@@ -152,7 +152,7 @@ TRACE_EVENT(i915_vma_unbind, __entry->size = vma->node.size; ), -TP_printk ("obj=%p, offset=%08x size=%x vm=%p" , +TP_printk ("obj=%p, offset=%016llx size=%x vm=%p" , __entry->obj, __entry->offset, __entry->size, __entry->vm) );
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
Re: [PATCH v4 01/24] drm/i915/trace: Fix offsets for 64b
Quoted 13 lines · Michel Thierry
From: Ben Widawsky <[email protected]>
Signed-off-by Ben Widawsky <[email protected]>Signed-off-by Michel Thierry <[email protected]>--- drivers/gpu/drm/i915/i915_trace.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-)drivers/gpu/drm/i915/i915_trace.h +0
-0
drivers/gpu/drm/i915/i915_trace.h TRACE_EVENT(i915_vma_bind, · line 120
Quoted 7 lines
drivers/gpu/drm/i915/i915_trace.h +1
-1@@ -115,7 +115,7 @@ TRACE_EVENT(i915_vma_bind,TP_STRUCT__entry (__field (struct drm_i915_gem_object *,obj ) __field(struct i915_address_space *, vm) -__field (u32, offset) +__field (u64, offset)__field (u32, size) __field(unsigned, flags)
We seem to use uint64_t for flags in caller. Not that we are using past a few currently so we good for now.
Quoted 35 lines
drivers/gpu/drm/i915/i915_trace.h +3
-3),@@ -128,7 +128,7 @@ TRACE_EVENT(i915_vma_bind, __entry->flags = flags; ), -TP_printk ("obj=%p, offset=%08x size=%x%s vm=%p" , +TP_printk ("obj=%p, offset=%016llx size=%x%s vm=%p" , __entry->obj, __entry->offset, __entry->size, __entry->flags &PIN_MAPPABLE ?", mappable" :"" , __entry->vm)@@ -141,7 +141,7 @@ TRACE_EVENT(i915_vma_unbind,TP_STRUCT__entry (__field (struct drm_i915_gem_object *,obj ) __field(struct i915_address_space *, vm) -__field (u32, offset) +__field (u64, offset)__field (u32, size) ),@@ -152,7 +152,7 @@ TRACE_EVENT(i915_vma_unbind, __entry->size = vma->node.size; ), -TP_printk ("obj=%p, offset=%08x size=%x vm=%p" , +TP_printk ("obj=%p, offset=%016llx size=%x vm=%p" , __entry->obj, __entry->offset, __entry->size, __entry->vm) );Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
_______________________________________________
Intel-gfx mailing list
[email protected]
http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH v4 2/24 drm/i915: Rename to GEN8_LEGACY_PDPES
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
In gen8, 32b PPGTT has always had one "pdp" (it doesn't actually have one, but it resembles having one). The #define was confusing as is, and using "PDPE" is a much better description.
sed -i 's/GEN8_LEGACY_PDPS/GEN8_LEGACY_PDPES/' drivers/gpu/drm/i915/*.[ch]
It also matches the x86 pagetable terminology:
PTE = Page Table Entry - pagetable level 1 page PDE = Page Directory Entry - pagetable level 2 page PDPE = Page Directory Pointer Entry - pagetable level 3 page
And in the near future (for 48b addressing):
PML4E = Page Map Level 4 Entry
v2: Expanded information about Page Directory/Table nomenclature.
--- drivers/gpu/drm/i915/i915_gem_gtt.c | 6 +++--- drivers/gpu/drm/i915/i915_gem_gtt.h | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-)
drivers/gpu/drm/i915/i915_gem_gtt.c +3 -3
@@ -375,7 +375,7 @@ static void gen8_ppgtt_insert_entries(struct i915_address_space *vm, pt_vaddr =NULL ;for_each_sg_page (pages->sgl, &sg_iter, pages->nents,0 ) { -if (WARN_ON (pdpe >=GEN8_LEGACY_PDPS )) +if (WARN_ON (pdpe >=GEN8_LEGACY_PDPES ))break ;if (pt_vaddr ==NULL )
@@ -486,7 +486,7 @@ bail:static int gen8_ppgtt_allocate_page_tables (struct i915_hw_ppgtt *ppgtt,const int max_pdp) { -struct page **pt_pages[GEN8_LEGACY_PDPS ]; +struct page **pt_pages[GEN8_LEGACY_PDPES ];int i, ret;for (i =0 ; i < max_pdp; i++) {
@@ -537,7 +537,7 @@ static int gen8_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt,return -ENOMEM ; ppgtt->num_pd_pages =1 <<get_order (max_pdp <<PAGE_SHIFT ); -BUG_ON (ppgtt->num_pd_pages >GEN8_LEGACY_PDPS ); +BUG_ON (ppgtt->num_pd_pages >GEN8_LEGACY_PDPES );return 0 ; }
drivers/gpu/drm/i915/i915_gem_gtt.h +3 -3
@@ -88,7 +88,7 @@ typedef gen8_gtt_pte_t gen8_ppgtt_pde_t;#define GEN8_PDE_MASK 0x1ff#define GEN8_PTE_SHIFT 12#define GEN8_PTE_MASK 0x1ff -#define GEN8_LEGACY_PDPS 4 +#define GEN8_LEGACY_PDPES 4#define GEN8_PTES_PER_PAGE (PAGE_SIZE / sizeof(gen8_gtt_pte_t))#define GEN8_PDES_PER_PAGE (PAGE_SIZE / sizeof(gen8_ppgtt_pde_t))
@@ -273,12 +273,12 @@ struct i915_hw_ppgtt {unsigned num_pd_pages;/* gen8+ */ union {struct page **pt_pages; -struct page **gen8_pt_pages[GEN8_LEGACY_PDPS ]; +struct page **gen8_pt_pages[GEN8_LEGACY_PDPES ]; };struct page *pd_pages;union {uint32_t pd_offset; -dma_addr_t pd_dma_addr[GEN8_LEGACY_PDPS ]; +dma_addr_t pd_dma_addr[GEN8_LEGACY_PDPES ]; };union {dma_addr_t *pt_dma_addr;
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
Re: [PATCH v4 02/24] drm/i915: Rename to GEN8_LEGACY_PDPES
Quoted 16 lines · Michel Thierry
From: Ben Widawsky <[email protected]>
In gen8, 32b PPGTT has always had one "pdp" (it doesn't actually have one, but it resembles having one). The #define was confusing as is, and using "PDPE" is a much better description.
sed -i 's/GEN8_LEGACY_PDPS/GEN8_LEGACY_PDPES/' drivers/gpu/drm/i915/*.[ch]
It also matches the x86 pagetable terminology:
PTE = Page Table Entry - pagetable level 1 page PDE = Page Directory Entry - pagetable level 2 page PDPE = Page Directory Pointer Entry - pagetable level 3 pageAnd in the near future (for 48b addressing):
PML4E = Page Map Level 4 Entry
Quoted 6 lines
v2: Expanded information about Page Directory/Table nomenclature.
Cc Daniel Vetter <[email protected]>CC Dave Gordon <[email protected]>Signed-off-by Ben Widawsky <[email protected]>Signed-off-by Michel Thierry <[email protected]> (v2)
Quoted 71 lines
--- drivers/gpu/drm/i915/i915_gem_gtt.c | 6 +++--- drivers/gpu/drm/i915/i915_gem_gtt.h | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-)drivers/gpu/drm/i915/i915_gem_gtt.c +3
-3@@ -375,7 +375,7 @@ static void gen8_ppgtt_insert_entries(struct i915_address_space *vm, pt_vaddr =NULL ;for_each_sg_page (pages->sgl, &sg_iter, pages->nents,0 ) { -if (WARN_ON (pdpe >=GEN8_LEGACY_PDPS )) +if (WARN_ON (pdpe >=GEN8_LEGACY_PDPES ))break ;if (pt_vaddr ==NULL )@@ -486,7 +486,7 @@ bail:static int gen8_ppgtt_allocate_page_tables (struct i915_hw_ppgtt *ppgtt,const int max_pdp) { -struct page **pt_pages[GEN8_LEGACY_PDPS ]; +struct page **pt_pages[GEN8_LEGACY_PDPES ];int i, ret;for (i =0 ; i < max_pdp; i++) {@@ -537,7 +537,7 @@ static int gen8_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt,return -ENOMEM ; ppgtt->num_pd_pages =1 <<get_order (max_pdp <<PAGE_SHIFT ); -BUG_ON (ppgtt->num_pd_pages >GEN8_LEGACY_PDPS ); +BUG_ON (ppgtt->num_pd_pages >GEN8_LEGACY_PDPES );return 0 ; }drivers/gpu/drm/i915/i915_gem_gtt.h +3
-3@@ -88,7 +88,7 @@ typedef gen8_gtt_pte_t gen8_ppgtt_pde_t;#define GEN8_PDE_MASK 0x1ff#define GEN8_PTE_SHIFT 12#define GEN8_PTE_MASK 0x1ff -#define GEN8_LEGACY_PDPS 4 +#define GEN8_LEGACY_PDPES 4#define GEN8_PTES_PER_PAGE (PAGE_SIZE / sizeof(gen8_gtt_pte_t))#define GEN8_PDES_PER_PAGE (PAGE_SIZE / sizeof(gen8_ppgtt_pde_t))@@ -273,12 +273,12 @@ struct i915_hw_ppgtt {unsigned num_pd_pages;/* gen8+ */ union {struct page **pt_pages; -struct page **gen8_pt_pages[GEN8_LEGACY_PDPS ]; +struct page **gen8_pt_pages[GEN8_LEGACY_PDPES ]; };struct page *pd_pages;union {uint32_t pd_offset; -dma_addr_t pd_dma_addr[GEN8_LEGACY_PDPS ]; +dma_addr_t pd_dma_addr[GEN8_LEGACY_PDPES ]; };union {dma_addr_t *pt_dma_addr;Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
_______________________________________________
Intel-gfx mailing list
[email protected]
http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH v4 3/24 drm/i915: Setup less PPGTT on failed page_directory
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
The current code will both potentially print a WARN, and setup part of the PPGTT structure. Neither of these harm the current code, it is simply for clarity, and to perhaps prevent later bugs, or weird debug messages.
--- drivers/gpu/drm/i915/i915_gem_gtt.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-)
drivers/gpu/drm/i915/i915_gem_gtt.c +4 -1
@@ -1032,11 +1032,14 @@ alloc: goto alloc; } +if (ret) +return ret; +if (ppgtt->node.start < dev_priv->gtt.mappable_end)DRM_DEBUG ("Forced to use aperture for PDEs\n" ); ppgtt->num_pd_entries =GEN6_PPGTT_PD_ENTRIES ; -return ret; +return 0 ; }static int gen6_ppgtt_allocate_page_tables (struct i915_hw_ppgtt *ppgtt)
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
Re: [PATCH v4 03/24] drm/i915: Setup less PPGTT on failed page_directory
Quoted 10 lines · Michel Thierry
From: Ben Widawsky <[email protected]>
The current code will both potentially print a WARN, and setup part of the PPGTT structure. Neither of these harm the current code, it is simply for clarity, and to perhaps prevent later bugs, or weird debug messages.
Signed-off-by Ben Widawsky <[email protected]>Signed-off-by Michel Thierry <[email protected]>---
Quoted 30 lines
drivers/gpu/drm/i915/i915_gem_gtt.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-)drivers/gpu/drm/i915/i915_gem_gtt.c +4
-1@@ -1032,11 +1032,14 @@ alloc: goto alloc; } +if (ret) +return ret; +if (ppgtt->node.start < dev_priv->gtt.mappable_end)DRM_DEBUG ("Forced to use aperture for PDEs\n" ); ppgtt->num_pd_entries =GEN6_PPGTT_PD_ENTRIES ; -return ret; +return 0 ; }static int gen6_ppgtt_allocate_page_tables (struct i915_hw_ppgtt *ppgtt)Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
_______________________________________________
Intel-gfx mailing list
[email protected]
http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH v4 4/24 drm/i915/gen8: Un-hardcode number of page directories
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
--- drivers/gpu/drm/i915/i915_gem_gtt.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
drivers/gpu/drm/i915/i915_gem_gtt.h +1 -1
@@ -282,7 +282,7 @@ struct i915_hw_ppgtt { };union {dma_addr_t *pt_dma_addr; -dma_addr_t *gen8_pt_dma_addr[4 ]; +dma_addr_t *gen8_pt_dma_addr[GEN8_LEGACY_PDPES ]; };struct drm_i915_file_private *file_priv;
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
Re: [PATCH v4 04/24] drm/i915/gen8: Un-hardcode number of page directories
Quoted 4 lines · Michel Thierry
From: Ben Widawsky <[email protected]>
Signed-off-by Ben Widawsky <[email protected]>Signed-off-by Michel Thierry <[email protected]>
Quoted 24 lines
--- drivers/gpu/drm/i915/i915_gem_gtt.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)drivers/gpu/drm/i915/i915_gem_gtt.h +1
-1@@ -282,7 +282,7 @@ struct i915_hw_ppgtt { };union {dma_addr_t *pt_dma_addr; -dma_addr_t *gen8_pt_dma_addr[4 ]; +dma_addr_t *gen8_pt_dma_addr[GEN8_LEGACY_PDPES ]; };struct drm_i915_file_private *file_priv;Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
_______________________________________________
Intel-gfx mailing list
[email protected]
http://lists.freedesktop.org/mailman/listinfo/intel-gfx
Re: [PATCH v4 04/24] drm/i915/gen8: Un-hardcode number of page directories
Quoted 8 lines · Mika Kuoppala
Quoted 4 lines · Michel Thierry
From: Ben Widawsky <[email protected]>
Signed-off-by Ben Widawsky <[email protected]>Signed-off-by Michel Thierry <[email protected]>Reviewed-by Mika Kuoppala <[email protected]>
Merged up to this one, thanks for patches&review.
-Daniel
Quoted 29 lines
Quoted 25 lines
--- drivers/gpu/drm/i915/i915_gem_gtt.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)drivers/gpu/drm/i915/i915_gem_gtt.h +1
-1@@ -282,7 +282,7 @@ struct i915_hw_ppgtt { };union {dma_addr_t *pt_dma_addr; -dma_addr_t *gen8_pt_dma_addr[4 ]; +dma_addr_t *gen8_pt_dma_addr[GEN8_LEGACY_PDPES ]; };struct drm_i915_file_private *file_priv;Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx_______________________________________________
Intel-gfx mailing list
[email protected]
http://lists.freedesktop.org/mailman/listinfo/intel-gfx
Signature · 7 lines
Daniel Vetter Software Engineer, Intel Corporation +41 (0) 79 365 57 48 - http://blog.ffwll.ch _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH v4 5/24 drm/i915: page table abstractions
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
When we move to dynamic page allocation, keeping page_directory and pagetabs as separate structures will help to break actions into simpler tasks.
To help transition the code nicely there is some wasted space in gen6/7. This will be ameliorated shortly.
Following the x86 pagetable terminology:
PDPE = struct i915_page_directory_pointer_entry. PDE = struct i915_page_directory_entry [page_directory]. PTE = struct i915_page_table_entry [page_tables].
v2: fixed mismatches after clean-up/rebase.
v3: Clarify the names of the multiple levels of page tables (Daniel)
--- drivers/gpu/drm/i915/i915_gem_gtt.c | 177 ++++++++++++++++++------------------ drivers/gpu/drm/i915/i915_gem_gtt.h | 23 ++++- 2 files changed, 107 insertions(+), 93 deletions(-)
drivers/gpu/drm/i915/i915_gem_gtt.c +89 -88
@@ -334,7 +334,8 @@ static void gen8_ppgtt_clear_range(struct i915_address_space *vm, I915_CACHE_LLC, use_scratch);while (num_entries) { -struct page *page_table = ppgtt->gen8_pt_pages[pdpe][pde]; +struct i915_page_directory_entry *pd = &ppgtt->pdp.page_directory[pdpe]; +struct page *page_table = pd->page_tables[pde].page; last_pte = pte + num_entries;if (last_pte >GEN8_PTES_PER_PAGE )
@@ -378,8 +379,12 @@ static void gen8_ppgtt_insert_entries(struct i915_address_space *vm,if (WARN_ON (pdpe >=GEN8_LEGACY_PDPES ))break ; -if (pt_vaddr ==NULL ) - pt_vaddr =kmap_atomic (ppgtt->gen8_pt_pages[pdpe][pde]); +if (pt_vaddr ==NULL ) { +struct i915_page_directory_entry *pd = &ppgtt->pdp.page_directory[pdpe]; +struct page *page_table = pd->page_tables[pde].page; + + pt_vaddr =kmap_atomic (page_table); + } pt_vaddr[pte] = gen8_pte_encode(sg_page_iter_dma_address(&sg_iter),
@@ -403,29 +408,33 @@ static void gen8_ppgtt_insert_entries(struct i915_address_space *vm, } } -static void gen8_free_page_tables (struct page **pt_pages) +static void gen8_free_page_tables (struct i915_page_directory_entry *pd) {int i; -if (pt_pages ==NULL ) +if (pd->page_tables ==NULL )return ;for (i =0 ; i <GEN8_PDES_PER_PAGE ; i++) -if (pt_pages[i]) -__free_pages (pt_pages[i],0 ); +if (pd->page_tables[i].page) +__free_page (pd->page_tables[i].page); } -static void gen8_ppgtt_free (const struct i915_hw_ppgtt *ppgtt) +static void gen8_free_page_directories (struct i915_page_directory_entry *pd) +{ +kfree (pd->page_tables); +__free_page (pd->page); +} + +static void gen8_ppgtt_free (struct i915_hw_ppgtt *ppgtt) {int i;for (i =0 ; i < ppgtt->num_pd_pages; i++) { -gen8_free_page_tables (ppgtt->gen8_pt_pages[i]); -kfree (ppgtt->gen8_pt_pages[i]); +gen8_free_page_tables (&ppgtt->pdp.page_directory[i]); +gen8_free_page_directories (&ppgtt->pdp.page_directory[i]);kfree (ppgtt->gen8_pt_dma_addr[i]); } - -__free_pages (ppgtt->pd_pages,get_order (ppgtt->num_pd_pages <<PAGE_SHIFT )); }static void gen8_ppgtt_unmap_pages (struct i915_hw_ppgtt *ppgtt)
@@ -460,86 +469,75 @@ static void gen8_ppgtt_cleanup(struct i915_address_space *vm)gen8_ppgtt_free (ppgtt); } -static struct page **__gen8_alloc_page_tables (void ) +static int gen8_ppgtt_allocate_dma (struct i915_hw_ppgtt *ppgtt) { -struct page **pt_pages;int i; - pt_pages =kcalloc (GEN8_PDES_PER_PAGE ,sizeof (struct page *),GFP_KERNEL ); -if (!pt_pages) -return ERR_PTR (-ENOMEM ); - -for (i =0 ; i <GEN8_PDES_PER_PAGE ; i++) { - pt_pages[i] =alloc_page (GFP_KERNEL ); -if (!pt_pages[i]) - goto bail; +for (i =0 ; i < ppgtt->num_pd_pages; i++) { + ppgtt->gen8_pt_dma_addr[i] =kcalloc (GEN8_PDES_PER_PAGE , +sizeof (dma_addr_t), +GFP_KERNEL ); +if (!ppgtt->gen8_pt_dma_addr[i]) +return -ENOMEM ; } -return pt_pages; - -bail: -gen8_free_page_tables (pt_pages); -kfree (pt_pages); -return ERR_PTR (-ENOMEM ); +return 0 ; } -static int gen8_ppgtt_allocate_page_tables (struct i915_hw_ppgtt *ppgtt, -const int max_pdp) +static int gen8_ppgtt_allocate_page_tables (struct i915_hw_ppgtt *ppgtt) { -struct page **pt_pages[GEN8_LEGACY_PDPES ]; -int i, ret; +int i, j; -for (i =0 ; i < max_pdp; i++) { - pt_pages[i] =__gen8_alloc_page_tables (); -if (IS_ERR (pt_pages[i])) { - ret =PTR_ERR (pt_pages[i]); - goto unwind_out; +for (i =0 ; i < ppgtt->num_pd_pages; i++) { +for (j =0 ; j <GEN8_PDES_PER_PAGE ; j++) { +struct i915_page_table_entry *pt = &ppgtt->pdp.page_directory[i].page_tables[j]; + + pt->page =alloc_page (GFP_KERNEL | __GFP_ZERO); +if (!pt->page) + goto unwind_out; } } -/* NB: Avoid touching gen8_pt_pages until last to keep the allocation, -* "atomic" - for cleanup purposes. -*/ -for (i =0 ; i < max_pdp; i++) - ppgtt->gen8_pt_pages[i] = pt_pages[i]; -return 0 ; unwind_out: -while (i--) { -gen8_free_page_tables (pt_pages[i]); -kfree (pt_pages[i]); - } +while (i--) +gen8_free_page_tables (&ppgtt->pdp.page_directory[i]); -return ret; +return -ENOMEM ; } -static int gen8_ppgtt_allocate_dma (struct i915_hw_ppgtt *ppgtt) +static int gen8_ppgtt_allocate_page_directories (struct i915_hw_ppgtt *ppgtt, +const int max_pdp) {int i; -for (i =0 ; i < ppgtt->num_pd_pages; i++) { - ppgtt->gen8_pt_dma_addr[i] =kcalloc (GEN8_PDES_PER_PAGE , -sizeof (dma_addr_t), -GFP_KERNEL ); -if (!ppgtt->gen8_pt_dma_addr[i]) -return -ENOMEM ; - } +for (i =0 ; i < max_pdp; i++) { +struct i915_page_table_entry *pt; -return 0 ; -} + pt =kcalloc (GEN8_PDES_PER_PAGE ,sizeof (*pt),GFP_KERNEL ); +if (!pt) + goto unwind_out; -static int gen8_ppgtt_allocate_page_directories (struct i915_hw_ppgtt *ppgtt, -const int max_pdp) -{ - ppgtt->pd_pages =alloc_pages (GFP_KERNEL ,get_order (max_pdp <<PAGE_SHIFT )); -if (!ppgtt->pd_pages) -return -ENOMEM ; + ppgtt->pdp.page_directory[i].page =alloc_page (GFP_KERNEL ); +if (!ppgtt->pdp.page_directory[i].page) + goto unwind_out; + + ppgtt->pdp.page_directory[i].page_tables = pt; + } - ppgtt->num_pd_pages =1 <<get_order (max_pdp <<PAGE_SHIFT ); + ppgtt->num_pd_pages = max_pdp;BUG_ON (ppgtt->num_pd_pages >GEN8_LEGACY_PDPES );return 0 ; + +unwind_out: +while (i--) { +kfree (ppgtt->pdp.page_directory[i].page_tables); +__free_page (ppgtt->pdp.page_directory[i].page); + } + +return -ENOMEM ; } static int gen8_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt,
@@ -551,18 +549,19 @@ static int gen8_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt,if (ret)return ret; - ret =gen8_ppgtt_allocate_page_tables (ppgtt, max_pdp); -if (ret) { -__free_pages (ppgtt->pd_pages,get_order (max_pdp <<PAGE_SHIFT )); -return ret; - } + ret =gen8_ppgtt_allocate_page_tables (ppgtt); +if (ret) + goto err_out; ppgtt->num_pd_entries = max_pdp *GEN8_PDES_PER_PAGE ; ret =gen8_ppgtt_allocate_dma (ppgtt); -if (ret) -gen8_ppgtt_free (ppgtt); +if (!ret) +return ret; +/* TODO: Check this for all cases */ +err_out: +gen8_ppgtt_free (ppgtt);return ret; }
@@ -573,7 +572,7 @@ static int gen8_ppgtt_setup_page_directories(struct i915_hw_ppgtt *ppgtt,int ret; pd_addr =pci_map_page (ppgtt->base.dev->pdev, - &ppgtt->pd_pages[pd],0 , + ppgtt->pdp.page_directory[pd].page,0 ,PAGE_SIZE ,PCI_DMA_BIDIRECTIONAL ); ret =pci_dma_mapping_error (ppgtt->base.dev->pdev, pd_addr);
@@ -593,7 +592,7 @@ static int gen8_ppgtt_setup_page_tables(struct i915_hw_ppgtt *ppgtt,struct page *p;int ret; - p = ppgtt->gen8_pt_pages[pd][pt]; + p = ppgtt->pdp.page_directory[pd].page_tables[pt].page; pt_addr =pci_map_page (ppgtt->base.dev->pdev, p,0 ,PAGE_SIZE ,PCI_DMA_BIDIRECTIONAL ); ret =pci_dma_mapping_error (ppgtt->base.dev->pdev, pt_addr);
@@ -654,7 +653,7 @@ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size)*/ for (i =0 ; i < max_pdp; i++) {gen8_ppgtt_pde_t *pd_vaddr; - pd_vaddr =kmap_atomic (&ppgtt->pd_pages[i]); + pd_vaddr =kmap_atomic (ppgtt->pdp.page_directory[i].page);for (j =0 ; j <GEN8_PDES_PER_PAGE ; j++) {dma_addr_t addr = ppgtt->gen8_pt_dma_addr[i][j]; pd_vaddr[j] = gen8_pde_encode(ppgtt->base.dev, addr,
@@ -717,7 +716,7 @@ static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m) expected);seq_printf (m,"\tPDE: %x\n" , pd_entry); - pt_vaddr =kmap_atomic (ppgtt->pt_pages[pde]); + pt_vaddr =kmap_atomic (ppgtt->pd.page_tables[pde].page);for (pte =0 ; pte <I915_PPGTT_PT_ENTRIES ; pte+=4 ) {unsigned long va = (pde * PAGE_SIZE * I915_PPGTT_PT_ENTRIES) +
@@ -922,7 +921,7 @@ static void gen6_ppgtt_clear_range(struct i915_address_space *vm,if (last_pte >I915_PPGTT_PT_ENTRIES ) last_pte =I915_PPGTT_PT_ENTRIES ; - pt_vaddr =kmap_atomic (ppgtt->pt_pages[act_pt]); + pt_vaddr =kmap_atomic (ppgtt->pd.page_tables[act_pt].page);for (i = first_pte; i < last_pte; i++) pt_vaddr[i] = scratch_pte;
@@ -951,7 +950,7 @@ static void gen6_ppgtt_insert_entries(struct i915_address_space *vm, pt_vaddr =NULL ;for_each_sg_page (pages->sgl, &sg_iter, pages->nents,0 ) {if (pt_vaddr ==NULL ) - pt_vaddr =kmap_atomic (ppgtt->pt_pages[act_pt]); + pt_vaddr =kmap_atomic (ppgtt->pd.page_tables[act_pt].page); pt_vaddr[act_pte] = vm->pte_encode(sg_page_iter_dma_address(&sg_iter),
@@ -986,8 +985,8 @@ static void gen6_ppgtt_free(struct i915_hw_ppgtt *ppgtt)kfree (ppgtt->pt_dma_addr);for (i =0 ; i < ppgtt->num_pd_entries; i++) -__free_page (ppgtt->pt_pages[i]); -kfree (ppgtt->pt_pages); +__free_page (ppgtt->pd.page_tables[i].page); +kfree (ppgtt->pd.page_tables); }static void gen6_ppgtt_cleanup (struct i915_address_space *vm)
@@ -1044,22 +1043,22 @@ alloc:static int gen6_ppgtt_allocate_page_tables (struct i915_hw_ppgtt *ppgtt) { +struct i915_page_table_entry *pt;int i; - ppgtt->pt_pages =kcalloc (ppgtt->num_pd_entries,sizeof (struct page *), -GFP_KERNEL ); - -if (!ppgtt->pt_pages) + pt =kcalloc (ppgtt->num_pd_entries,sizeof (*pt),GFP_KERNEL ); +if (!pt)return -ENOMEM ;for (i =0 ; i < ppgtt->num_pd_entries; i++) { - ppgtt->pt_pages[i] =alloc_page (GFP_KERNEL ); -if (!ppgtt->pt_pages[i]) { + pt[i].page =alloc_page (GFP_KERNEL ); +if (!pt->page) {gen6_ppgtt_free (ppgtt);return -ENOMEM ; } } + ppgtt->pd.page_tables = pt;return 0 ; }
@@ -1094,9 +1093,11 @@ static int gen6_ppgtt_setup_page_tables(struct i915_hw_ppgtt *ppgtt)int i;for (i =0 ; i < ppgtt->num_pd_entries; i++) { +struct page *page;dma_addr_t pt_addr; - pt_addr =pci_map_page (dev->pdev, ppgtt->pt_pages[i],0 ,4096 , + page = ppgtt->pd.page_tables[i].page; + pt_addr =pci_map_page (dev->pdev, page,0 ,4096 ,PCI_DMA_BIDIRECTIONAL );if (pci_dma_mapping_error (dev->pdev, pt_addr)) {
@@ -1140,7 +1141,7 @@ static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt) ppgtt->base.insert_entries = gen6_ppgtt_insert_entries; ppgtt->base.cleanup = gen6_ppgtt_cleanup; ppgtt->base.start =0 ; - ppgtt->base.total = ppgtt->num_pd_entries *I915_PPGTT_PT_ENTRIES *PAGE_SIZE ; + ppgtt->base.total = ppgtt->num_pd_entries *I915_PPGTT_PT_ENTRIES *PAGE_SIZE ; ppgtt->debug_dump = gen6_dump_ppgtt; ppgtt->pd_offset =
drivers/gpu/drm/i915/i915_gem_gtt.h +18 -5
@@ -187,6 +187,20 @@ struct i915_vma { u32 flags); }; +struct i915_page_table_entry { +struct page *page; +}; + +struct i915_page_directory_entry { +struct page *page;/* NULL for GEN6-GEN7 */ +struct i915_page_table_entry *page_tables; +}; + +struct i915_page_directory_pointer_entry { +/* struct page *page; */ +struct i915_page_directory_entry page_directory[GEN8_LEGACY_PDPES ]; +}; +struct i915_address_space {struct drm_mm mm;struct drm_device *dev;
@@ -272,11 +286,6 @@ struct i915_hw_ppgtt {unsigned num_pd_entries;unsigned num_pd_pages;/* gen8+ */ union { -struct page **pt_pages; -struct page **gen8_pt_pages[GEN8_LEGACY_PDPES ]; - }; -struct page *pd_pages; -union {uint32_t pd_offset;dma_addr_t pd_dma_addr[GEN8_LEGACY_PDPES ]; };
@@ -284,6 +293,10 @@ struct i915_hw_ppgtt {dma_addr_t *pt_dma_addr;dma_addr_t *gen8_pt_dma_addr[GEN8_LEGACY_PDPES ]; }; +union { +struct i915_page_directory_pointer_entry pdp; +struct i915_page_directory_entry pd; + };struct drm_i915_file_private *file_priv;
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
Re: [PATCH v4 05/24] drm/i915: page table abstractions
Quoted 65 lines · Michel Thierry
From: Ben Widawsky <[email protected]>
When we move to dynamic page allocation, keeping
page_directoryand pagetabs as separate structures will help to break actions into simpler tasks.To help transition the code nicely there is some wasted space in gen6/7. This will be ameliorated shortly.
Following the x86 pagetable terminology:
PDPE = structi915_page_directory_pointer_entry. PDE = structi915_page_directory_entry[page_directory]. PTE = structi915_page_table_entry[page_tables].v2: fixed mismatches after clean-up/rebase.
v3: Clarify the names of the multiple levels of page tables (Daniel)
Signed-off-by Ben Widawsky <[email protected]>Signed-off-by Michel Thierry <[email protected]> (v2, v3)--- drivers/gpu/drm/i915/i915_gem_gtt.c | 177 ++++++++++++++++++------------------ drivers/gpu/drm/i915/i915_gem_gtt.h | 23 ++++- 2 files changed, 107 insertions(+), 93 deletions(-)drivers/gpu/drm/i915/i915_gem_gtt.c +10
-5@@ -334,7 +334,8 @@ static void gen8_ppgtt_clear_range(struct i915_address_space *vm, I915_CACHE_LLC, use_scratch);while (num_entries) { -struct page *page_table = ppgtt->gen8_pt_pages[pdpe][pde]; +struct i915_page_directory_entry *pd = &ppgtt->pdp.page_directory[pdpe]; +struct page *page_table = pd->page_tables[pde].page; last_pte = pte + num_entries;if (last_pte >GEN8_PTES_PER_PAGE )@@ -378,8 +379,12 @@ static void gen8_ppgtt_insert_entries(struct i915_address_space *vm,if (WARN_ON (pdpe >=GEN8_LEGACY_PDPES ))break ; -if (pt_vaddr ==NULL ) - pt_vaddr =kmap_atomic (ppgtt->gen8_pt_pages[pdpe][pde]); +if (pt_vaddr ==NULL ) { +struct i915_page_directory_entry *pd = &ppgtt->pdp.page_directory[pdpe]; +struct page *page_table = pd->page_tables[pde].page; + + pt_vaddr =kmap_atomic (page_table); + } pt_vaddr[pte] = gen8_pte_encode(sg_page_iter_dma_address(&sg_iter),@@ -403,29 +408,33 @@ static void gen8_ppgtt_insert_entries(struct i915_address_space *vm, } } -static void gen8_free_page_tables (struct page **pt_pages) +static void gen8_free_page_tables (struct i915_page_directory_entry *pd) {int i; -if (pt_pages ==NULL ) +if (pd->page_tables ==NULL )return ;
drivers/gpu/drm/i915/i915_gem_gtt.c static void gen8_ppgtt_insert_entries(struct i915_address_space *vm, · line 423
Quoted 10 lines
drivers/gpu/drm/i915/i915_gem_gtt.c +3
-3for (i =0 ; i <GEN8_PDES_PER_PAGE ; i++) -if (pt_pages[i]) -__free_pages (pt_pages[i],0 ); +if (pd->page_tables[i].page) +__free_page (pd->page_tables[i].page); } -static void gen8_ppgtt_free (const struct i915_hw_ppgtt *ppgtt) +static void gen8_free_page_directories (struct i915_page_directory_entry *pd)
^
You only free one directory so why plural here?
drivers/gpu/drm/i915/i915_gem_gtt.c static void gen8_ppgtt_insert_entries(struct i915_address_space *vm, · line 424
Quoted 1 line
drivers/gpu/drm/i915/i915_gem_gtt.c +1
-0+{
If you free the page tables for the directory here..
drivers/gpu/drm/i915/i915_gem_gtt.c static void gen8_ppgtt_insert_entries(struct i915_address_space *vm, · line 434
Quoted 12 lines
drivers/gpu/drm/i915/i915_gem_gtt.c +6
-2+kfree (pd->page_tables); +__free_page (pd->page); +} + +static void gen8_ppgtt_free (struct i915_hw_ppgtt *ppgtt) {int i;for (i =0 ; i < ppgtt->num_pd_pages; i++) { -gen8_free_page_tables (ppgtt->gen8_pt_pages[i]); -kfree (ppgtt->gen8_pt_pages[i]); +gen8_free_page_tables (&ppgtt->pdp.page_directory[i]);
...this loop will be cleaner.
Also consider renaming 'num_pd_pages' to 'num_pd'. But if it does cause a lot of rebase burden dont worry about it.
Quoted 107 lines
drivers/gpu/drm/i915/i915_gem_gtt.c +28
-48+gen8_free_page_directories (&ppgtt->pdp.page_directory[i]);kfree (ppgtt->gen8_pt_dma_addr[i]); } - -__free_pages (ppgtt->pd_pages,get_order (ppgtt->num_pd_pages <<PAGE_SHIFT )); }static void gen8_ppgtt_unmap_pages (struct i915_hw_ppgtt *ppgtt)@@ -460,86 +469,75 @@ static void gen8_ppgtt_cleanup(struct i915_address_space *vm)gen8_ppgtt_free (ppgtt); } -static struct page **__gen8_alloc_page_tables (void ) +static int gen8_ppgtt_allocate_dma (struct i915_hw_ppgtt *ppgtt) { -struct page **pt_pages;int i; - pt_pages =kcalloc (GEN8_PDES_PER_PAGE ,sizeof (struct page *),GFP_KERNEL ); -if (!pt_pages) -return ERR_PTR (-ENOMEM ); - -for (i =0 ; i <GEN8_PDES_PER_PAGE ; i++) { - pt_pages[i] =alloc_page (GFP_KERNEL ); -if (!pt_pages[i]) - goto bail; +for (i =0 ; i < ppgtt->num_pd_pages; i++) { + ppgtt->gen8_pt_dma_addr[i] =kcalloc (GEN8_PDES_PER_PAGE , +sizeof (dma_addr_t), +GFP_KERNEL ); +if (!ppgtt->gen8_pt_dma_addr[i]) +return -ENOMEM ; } -return pt_pages; - -bail: -gen8_free_page_tables (pt_pages); -kfree (pt_pages); -return ERR_PTR (-ENOMEM ); +return 0 ; } -static int gen8_ppgtt_allocate_page_tables (struct i915_hw_ppgtt *ppgtt, -const int max_pdp) +static int gen8_ppgtt_allocate_page_tables (struct i915_hw_ppgtt *ppgtt) { -struct page **pt_pages[GEN8_LEGACY_PDPES ]; -int i, ret; +int i, j; -for (i =0 ; i < max_pdp; i++) { - pt_pages[i] =__gen8_alloc_page_tables (); -if (IS_ERR (pt_pages[i])) { - ret =PTR_ERR (pt_pages[i]); - goto unwind_out; +for (i =0 ; i < ppgtt->num_pd_pages; i++) { +for (j =0 ; j <GEN8_PDES_PER_PAGE ; j++) { +struct i915_page_table_entry *pt = &ppgtt->pdp.page_directory[i].page_tables[j]; + + pt->page =alloc_page (GFP_KERNEL | __GFP_ZERO); +if (!pt->page) + goto unwind_out; } } -/* NB: Avoid touching gen8_pt_pages until last to keep the allocation, -* "atomic" - for cleanup purposes. -*/ -for (i =0 ; i < max_pdp; i++) - ppgtt->gen8_pt_pages[i] = pt_pages[i]; -return 0 ; unwind_out: -while (i--) { -gen8_free_page_tables (pt_pages[i]); -kfree (pt_pages[i]); - } +while (i--) +gen8_free_page_tables (&ppgtt->pdp.page_directory[i]); -return ret; +return -ENOMEM ; } -static int gen8_ppgtt_allocate_dma (struct i915_hw_ppgtt *ppgtt) +static int gen8_ppgtt_allocate_page_directories (struct i915_hw_ppgtt *ppgtt, +const int max_pdp) {int i; -for (i =0 ; i < ppgtt->num_pd_pages; i++) { - ppgtt->gen8_pt_dma_addr[i] =kcalloc (GEN8_PDES_PER_PAGE , -sizeof (dma_addr_t), -GFP_KERNEL ); -if (!ppgtt->gen8_pt_dma_addr[i]) -return -ENOMEM ; - } +for (i =0 ; i < max_pdp; i++) { +struct i915_page_table_entry *pt; -return 0 ; -} + pt =kcalloc (GEN8_PDES_PER_PAGE ,sizeof (*pt),GFP_KERNEL ); +if (!pt) + goto unwind_out;
drivers/gpu/drm/i915/i915_gem_gtt.c static void gen8_ppgtt_cleanup(struct i915_address_space *vm) · line 524
Quoted 10 lines
drivers/gpu/drm/i915/i915_gem_gtt.c +3
-6-static int gen8_ppgtt_allocate_page_directories (struct i915_hw_ppgtt *ppgtt, -const int max_pdp) -{ - ppgtt->pd_pages =alloc_pages (GFP_KERNEL ,get_order (max_pdp <<PAGE_SHIFT )); -if (!ppgtt->pd_pages) -return -ENOMEM ; + ppgtt->pdp.page_directory[i].page =alloc_page (GFP_KERNEL ); +if (!ppgtt->pdp.page_directory[i].page) + goto unwind_out;
If you end up having alloc error here you will leak the previously allocated pt above.
Also consider that if you do gen8_ppgtt_allocate_page_directory() and add null check for pd->page in gen8_free_page_directory you should be able to avoid the unwinding below completely.
Quoted 32 lines
drivers/gpu/drm/i915/i915_gem_gtt.c +15
-6+ + ppgtt->pdp.page_directory[i].page_tables = pt; + } - ppgtt->num_pd_pages =1 <<get_order (max_pdp <<PAGE_SHIFT ); + ppgtt->num_pd_pages = max_pdp;BUG_ON (ppgtt->num_pd_pages >GEN8_LEGACY_PDPES );return 0 ; + +unwind_out: +while (i--) { +kfree (ppgtt->pdp.page_directory[i].page_tables); +__free_page (ppgtt->pdp.page_directory[i].page); + } + +return -ENOMEM ; } static int gen8_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt,@@ -551,18 +549,19 @@ static int gen8_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt,if (ret)return ret; - ret =gen8_ppgtt_allocate_page_tables (ppgtt, max_pdp); -if (ret) { -__free_pages (ppgtt->pd_pages,get_order (max_pdp <<PAGE_SHIFT )); -return ret; - } + ret =gen8_ppgtt_allocate_page_tables (ppgtt); +if (ret) + goto err_out;
drivers/gpu/drm/i915/i915_gem_gtt.c static int gen8_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt, · line 562
Quoted 10 lines
drivers/gpu/drm/i915/i915_gem_gtt.c +3
-2ppgtt->num_pd_entries = max_pdp *GEN8_PDES_PER_PAGE ; ret =gen8_ppgtt_allocate_dma (ppgtt); -if (ret) -gen8_ppgtt_free (ppgtt); +if (!ret) +return ret; +/* TODO: Check this for all cases */
The check for zero return and then returning it with the comment is confusing. Why not just do the same pattern as in above?
if (ret)
goto err_out;
return 0;
-Mika
Quoted 176 lines
drivers/gpu/drm/i915/i915_gem_gtt.c +20
-16+err_out: +gen8_ppgtt_free (ppgtt);return ret; }@@ -573,7 +572,7 @@ static int gen8_ppgtt_setup_page_directories(struct i915_hw_ppgtt *ppgtt,int ret; pd_addr =pci_map_page (ppgtt->base.dev->pdev, - &ppgtt->pd_pages[pd],0 , + ppgtt->pdp.page_directory[pd].page,0 ,PAGE_SIZE ,PCI_DMA_BIDIRECTIONAL ); ret =pci_dma_mapping_error (ppgtt->base.dev->pdev, pd_addr);@@ -593,7 +592,7 @@ static int gen8_ppgtt_setup_page_tables(struct i915_hw_ppgtt *ppgtt,struct page *p;int ret; - p = ppgtt->gen8_pt_pages[pd][pt]; + p = ppgtt->pdp.page_directory[pd].page_tables[pt].page; pt_addr =pci_map_page (ppgtt->base.dev->pdev, p,0 ,PAGE_SIZE ,PCI_DMA_BIDIRECTIONAL ); ret =pci_dma_mapping_error (ppgtt->base.dev->pdev, pt_addr);@@ -654,7 +653,7 @@ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size)*/ for (i =0 ; i < max_pdp; i++) {gen8_ppgtt_pde_t *pd_vaddr; - pd_vaddr =kmap_atomic (&ppgtt->pd_pages[i]); + pd_vaddr =kmap_atomic (ppgtt->pdp.page_directory[i].page);for (j =0 ; j <GEN8_PDES_PER_PAGE ; j++) {dma_addr_t addr = ppgtt->gen8_pt_dma_addr[i][j]; pd_vaddr[j] = gen8_pde_encode(ppgtt->base.dev, addr,@@ -717,7 +716,7 @@ static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m) expected);seq_printf (m,"\tPDE: %x\n" , pd_entry); - pt_vaddr =kmap_atomic (ppgtt->pt_pages[pde]); + pt_vaddr =kmap_atomic (ppgtt->pd.page_tables[pde].page);for (pte =0 ; pte <I915_PPGTT_PT_ENTRIES ; pte+=4 ) {unsigned long va = (pde * PAGE_SIZE * I915_PPGTT_PT_ENTRIES) +@@ -922,7 +921,7 @@ static void gen6_ppgtt_clear_range(struct i915_address_space *vm,if (last_pte >I915_PPGTT_PT_ENTRIES ) last_pte =I915_PPGTT_PT_ENTRIES ; - pt_vaddr =kmap_atomic (ppgtt->pt_pages[act_pt]); + pt_vaddr =kmap_atomic (ppgtt->pd.page_tables[act_pt].page);for (i = first_pte; i < last_pte; i++) pt_vaddr[i] = scratch_pte;@@ -951,7 +950,7 @@ static void gen6_ppgtt_insert_entries(struct i915_address_space *vm, pt_vaddr =NULL ;for_each_sg_page (pages->sgl, &sg_iter, pages->nents,0 ) {if (pt_vaddr ==NULL ) - pt_vaddr =kmap_atomic (ppgtt->pt_pages[act_pt]); + pt_vaddr =kmap_atomic (ppgtt->pd.page_tables[act_pt].page); pt_vaddr[act_pte] = vm->pte_encode(sg_page_iter_dma_address(&sg_iter),@@ -986,8 +985,8 @@ static void gen6_ppgtt_free(struct i915_hw_ppgtt *ppgtt)kfree (ppgtt->pt_dma_addr);for (i =0 ; i < ppgtt->num_pd_entries; i++) -__free_page (ppgtt->pt_pages[i]); -kfree (ppgtt->pt_pages); +__free_page (ppgtt->pd.page_tables[i].page); +kfree (ppgtt->pd.page_tables); }static void gen6_ppgtt_cleanup (struct i915_address_space *vm)@@ -1044,22 +1043,22 @@ alloc:static int gen6_ppgtt_allocate_page_tables (struct i915_hw_ppgtt *ppgtt) { +struct i915_page_table_entry *pt;int i; - ppgtt->pt_pages =kcalloc (ppgtt->num_pd_entries,sizeof (struct page *), -GFP_KERNEL ); - -if (!ppgtt->pt_pages) + pt =kcalloc (ppgtt->num_pd_entries,sizeof (*pt),GFP_KERNEL ); +if (!pt)return -ENOMEM ;for (i =0 ; i < ppgtt->num_pd_entries; i++) { - ppgtt->pt_pages[i] =alloc_page (GFP_KERNEL ); -if (!ppgtt->pt_pages[i]) { + pt[i].page =alloc_page (GFP_KERNEL ); +if (!pt->page) {gen6_ppgtt_free (ppgtt);return -ENOMEM ; } } + ppgtt->pd.page_tables = pt;return 0 ; }@@ -1094,9 +1093,11 @@ static int gen6_ppgtt_setup_page_tables(struct i915_hw_ppgtt *ppgtt)int i;for (i =0 ; i < ppgtt->num_pd_entries; i++) { +struct page *page;dma_addr_t pt_addr; - pt_addr =pci_map_page (dev->pdev, ppgtt->pt_pages[i],0 ,4096 , + page = ppgtt->pd.page_tables[i].page; + pt_addr =pci_map_page (dev->pdev, page,0 ,4096 ,PCI_DMA_BIDIRECTIONAL );if (pci_dma_mapping_error (dev->pdev, pt_addr)) {@@ -1140,7 +1141,7 @@ static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt) ppgtt->base.insert_entries = gen6_ppgtt_insert_entries; ppgtt->base.cleanup = gen6_ppgtt_cleanup; ppgtt->base.start =0 ; - ppgtt->base.total = ppgtt->num_pd_entries *I915_PPGTT_PT_ENTRIES *PAGE_SIZE ; + ppgtt->base.total = ppgtt->num_pd_entries *I915_PPGTT_PT_ENTRIES *PAGE_SIZE ; ppgtt->debug_dump = gen6_dump_ppgtt; ppgtt->pd_offset =drivers/gpu/drm/i915/i915_gem_gtt.h +18
-5@@ -187,6 +187,20 @@ struct i915_vma { u32 flags); }; +struct i915_page_table_entry { +struct page *page; +}; + +struct i915_page_directory_entry { +struct page *page;/* NULL for GEN6-GEN7 */ +struct i915_page_table_entry *page_tables; +}; + +struct i915_page_directory_pointer_entry { +/* struct page *page; */ +struct i915_page_directory_entry page_directory[GEN8_LEGACY_PDPES ]; +}; +struct i915_address_space {struct drm_mm mm;struct drm_device *dev;@@ -272,11 +286,6 @@ struct i915_hw_ppgtt {unsigned num_pd_entries;unsigned num_pd_pages;/* gen8+ */ union { -struct page **pt_pages; -struct page **gen8_pt_pages[GEN8_LEGACY_PDPES ]; - }; -struct page *pd_pages; -union {uint32_t pd_offset;dma_addr_t pd_dma_addr[GEN8_LEGACY_PDPES ]; };@@ -284,6 +293,10 @@ struct i915_hw_ppgtt {dma_addr_t *pt_dma_addr;dma_addr_t *gen8_pt_dma_addr[GEN8_LEGACY_PDPES ]; }; +union { +struct i915_page_directory_pointer_entry pdp; +struct i915_page_directory_entry pd; + };struct drm_i915_file_private *file_priv;Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
_______________________________________________
Intel-gfx mailing list
[email protected]
http://lists.freedesktop.org/mailman/listinfo/intel-gfx
Re: [PATCH v4 05/24] drm/i915: page table abstractions
Quoted 91 lines · Mika Kuoppala
Quoted 65 lines · Michel Thierry
From: Ben Widawsky <[email protected]>
When we move to dynamic page allocation, keeping
page_directoryand pagetabs as separate structures will help to break actions into simpler tasks.To help transition the code nicely there is some wasted space in gen6/7. This will be ameliorated shortly.
Following the x86 pagetable terminology:
PDPE = structi915_page_directory_pointer_entry. PDE = structi915_page_directory_entry[page_directory]. PTE = structi915_page_table_entry[page_tables].v2: fixed mismatches after clean-up/rebase.
v3: Clarify the names of the multiple levels of page tables (Daniel)
Signed-off-by Ben Widawsky <[email protected]>Signed-off-by Michel Thierry <[email protected]> (v2, v3)--- drivers/gpu/drm/i915/i915_gem_gtt.c | 177 ++++++++++++++++++------------------ drivers/gpu/drm/i915/i915_gem_gtt.h | 23 ++++- 2 files changed, 107 insertions(+), 93 deletions(-)drivers/gpu/drm/i915/i915_gem_gtt.c +10
-5@@ -334,7 +334,8 @@ static void gen8_ppgtt_clear_range(struct i915_address_space *vm, I915_CACHE_LLC, use_scratch);while (num_entries) { -struct page *page_table = ppgtt->gen8_pt_pages[pdpe][pde]; +struct i915_page_directory_entry *pd = &ppgtt->pdp.page_directory[pdpe]; +struct page *page_table = pd->page_tables[pde].page; last_pte = pte + num_entries;if (last_pte >GEN8_PTES_PER_PAGE )@@ -378,8 +379,12 @@ static void gen8_ppgtt_insert_entries(struct i915_address_space *vm,if (WARN_ON (pdpe >=GEN8_LEGACY_PDPES ))break ; -if (pt_vaddr ==NULL ) - pt_vaddr =kmap_atomic (ppgtt->gen8_pt_pages[pdpe][pde]); +if (pt_vaddr ==NULL ) { +struct i915_page_directory_entry *pd = &ppgtt->pdp.page_directory[pdpe]; +struct page *page_table = pd->page_tables[pde].page; + + pt_vaddr =kmap_atomic (page_table); + } pt_vaddr[pte] = gen8_pte_encode(sg_page_iter_dma_address(&sg_iter),@@ -403,29 +408,33 @@ static void gen8_ppgtt_insert_entries(struct i915_address_space *vm, } } -static void gen8_free_page_tables (struct page **pt_pages) +static void gen8_free_page_tables (struct i915_page_directory_entry *pd) {int i; -if (pt_pages ==NULL ) +if (pd->page_tables ==NULL )return ;drivers/gpu/drm/i915/i915_gem_gtt.c static void gen8_ppgtt_insert_entries(struct i915_address_space *vm, · line 423
Quoted 10 lines
drivers/gpu/drm/i915/i915_gem_gtt.c +3
-3for (i =0 ; i <GEN8_PDES_PER_PAGE ; i++) -if (pt_pages[i]) -__free_pages (pt_pages[i],0 ); +if (pd->page_tables[i].page) +__free_page (pd->page_tables[i].page); } -static void gen8_ppgtt_free (const struct i915_hw_ppgtt *ppgtt) +static void gen8_free_page_directories (struct i915_page_directory_entry *pd)^You only free one directory so why plural here?
drivers/gpu/drm/i915/i915_gem_gtt.c static void gen8_ppgtt_insert_entries(struct i915_address_space *vm, · line 424
Quoted 1 line
drivers/gpu/drm/i915/i915_gem_gtt.c +1
-0+{If you free the page tables for the directory here..
Quoted 8 lines
drivers/gpu/drm/i915/i915_gem_gtt.c +5
-0+kfree (pd->page_tables); +__free_page (pd->page); +} + +static void gen8_ppgtt_free (struct i915_hw_ppgtt *ppgtt) {int i;
Quoted 8 lines
drivers/gpu/drm/i915/i915_gem_gtt.c static void gen8_ppgtt_insert_entries(struct i915_address_space *vm, · line 434
Quoted 4 lines
drivers/gpu/drm/i915/i915_gem_gtt.c +1
-2for (i =0 ; i < ppgtt->num_pd_pages; i++) { -gen8_free_page_tables (ppgtt->gen8_pt_pages[i]); -kfree (ppgtt->gen8_pt_pages[i]); +gen8_free_page_tables (&ppgtt->pdp.page_directory[i]);...this loop will be cleaner.
Also consider renaming '
num_pd_pages' to 'num_pd'. But if it does cause a lot of rebase burden dont worry about it.
num_pd_pages will go away in patch #19, so I rather not change that. All other comments addressed in v4.
Thanks,
-Michel
Quoted 345 lines
Quoted 107 lines
drivers/gpu/drm/i915/i915_gem_gtt.c +28
-48+gen8_free_page_directories (&ppgtt->pdp.page_directory[i]);kfree (ppgtt->gen8_pt_dma_addr[i]); } - -__free_pages (ppgtt->pd_pages,get_order (ppgtt->num_pd_pages <<PAGE_SHIFT )); }static void gen8_ppgtt_unmap_pages (struct i915_hw_ppgtt *ppgtt)@@ -460,86 +469,75 @@ static void gen8_ppgtt_cleanup(struct i915_address_space *vm)gen8_ppgtt_free (ppgtt); } -static struct page **__gen8_alloc_page_tables (void ) +static int gen8_ppgtt_allocate_dma (struct i915_hw_ppgtt *ppgtt) { -struct page **pt_pages;int i; - pt_pages =kcalloc (GEN8_PDES_PER_PAGE ,sizeof (struct page *),GFP_KERNEL ); -if (!pt_pages) -return ERR_PTR (-ENOMEM ); - -for (i =0 ; i <GEN8_PDES_PER_PAGE ; i++) { - pt_pages[i] =alloc_page (GFP_KERNEL ); -if (!pt_pages[i]) - goto bail; +for (i =0 ; i < ppgtt->num_pd_pages; i++) { + ppgtt->gen8_pt_dma_addr[i] =kcalloc (GEN8_PDES_PER_PAGE , +sizeof (dma_addr_t), +GFP_KERNEL ); +if (!ppgtt->gen8_pt_dma_addr[i]) +return -ENOMEM ; } -return pt_pages; - -bail: -gen8_free_page_tables (pt_pages); -kfree (pt_pages); -return ERR_PTR (-ENOMEM ); +return 0 ; } -static int gen8_ppgtt_allocate_page_tables (struct i915_hw_ppgtt *ppgtt, -const int max_pdp) +static int gen8_ppgtt_allocate_page_tables (struct i915_hw_ppgtt *ppgtt) { -struct page **pt_pages[GEN8_LEGACY_PDPES ]; -int i, ret; +int i, j; -for (i =0 ; i < max_pdp; i++) { - pt_pages[i] =__gen8_alloc_page_tables (); -if (IS_ERR (pt_pages[i])) { - ret =PTR_ERR (pt_pages[i]); - goto unwind_out; +for (i =0 ; i < ppgtt->num_pd_pages; i++) { +for (j =0 ; j <GEN8_PDES_PER_PAGE ; j++) { +struct i915_page_table_entry *pt = &ppgtt->pdp.page_directory[i].page_tables[j]; + + pt->page =alloc_page (GFP_KERNEL | __GFP_ZERO); +if (!pt->page) + goto unwind_out; } } -/* NB: Avoid touching gen8_pt_pages until last to keep the allocation, -* "atomic" - for cleanup purposes. -*/ -for (i =0 ; i < max_pdp; i++) - ppgtt->gen8_pt_pages[i] = pt_pages[i]; -return 0 ; unwind_out: -while (i--) { -gen8_free_page_tables (pt_pages[i]); -kfree (pt_pages[i]); - } +while (i--) +gen8_free_page_tables (&ppgtt->pdp.page_directory[i]); -return ret; +return -ENOMEM ; } -static int gen8_ppgtt_allocate_dma (struct i915_hw_ppgtt *ppgtt) +static int gen8_ppgtt_allocate_page_directories (struct i915_hw_ppgtt *ppgtt, +const int max_pdp) {int i; -for (i =0 ; i < ppgtt->num_pd_pages; i++) { - ppgtt->gen8_pt_dma_addr[i] =kcalloc (GEN8_PDES_PER_PAGE , -sizeof (dma_addr_t), -GFP_KERNEL ); -if (!ppgtt->gen8_pt_dma_addr[i]) -return -ENOMEM ; - } +for (i =0 ; i < max_pdp; i++) { +struct i915_page_table_entry *pt; -return 0 ; -} + pt =kcalloc (GEN8_PDES_PER_PAGE ,sizeof (*pt),GFP_KERNEL ); +if (!pt) + goto unwind_out;drivers/gpu/drm/i915/i915_gem_gtt.c static void gen8_ppgtt_cleanup(struct i915_address_space *vm) · line 524
Quoted 10 lines
drivers/gpu/drm/i915/i915_gem_gtt.c +3
-6-static int gen8_ppgtt_allocate_page_directories (struct i915_hw_ppgtt *ppgtt, -const int max_pdp) -{ - ppgtt->pd_pages =alloc_pages (GFP_KERNEL ,get_order (max_pdp <<PAGE_SHIFT )); -if (!ppgtt->pd_pages) -return -ENOMEM ; + ppgtt->pdp.page_directory[i].page =alloc_page (GFP_KERNEL ); +if (!ppgtt->pdp.page_directory[i].page) + goto unwind_out;If you end up having alloc error here you will leak the previously allocated pt above.
Also consider that if you do
gen8_ppgtt_allocate_page_directory()and add null check forpd->pageingen8_free_page_directoryyou should be able to avoid the unwinding below completely.Quoted 32 lines
drivers/gpu/drm/i915/i915_gem_gtt.c +15
-6+ + ppgtt->pdp.page_directory[i].page_tables = pt; + } - ppgtt->num_pd_pages =1 <<get_order (max_pdp <<PAGE_SHIFT ); + ppgtt->num_pd_pages = max_pdp;BUG_ON (ppgtt->num_pd_pages >GEN8_LEGACY_PDPES );return 0 ; + +unwind_out: +while (i--) { +kfree (ppgtt->pdp.page_directory[i].page_tables); +__free_page (ppgtt->pdp.page_directory[i].page); + } + +return -ENOMEM ; } static int gen8_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt,@@ -551,18 +549,19 @@ static int gen8_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt,if (ret)return ret; - ret =gen8_ppgtt_allocate_page_tables (ppgtt, max_pdp); -if (ret) { -__free_pages (ppgtt->pd_pages,get_order (max_pdp <<PAGE_SHIFT )); -return ret; - } + ret =gen8_ppgtt_allocate_page_tables (ppgtt); +if (ret) + goto err_out;drivers/gpu/drm/i915/i915_gem_gtt.c static int gen8_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt, · line 562
Quoted 10 lines
drivers/gpu/drm/i915/i915_gem_gtt.c +3
-2ppgtt->num_pd_entries = max_pdp *GEN8_PDES_PER_PAGE ; ret =gen8_ppgtt_allocate_dma (ppgtt); -if (ret) -gen8_ppgtt_free (ppgtt); +if (!ret) +return ret; +/* TODO: Check this for all cases */ The check for zero return and then returning it with the comment is confusing. Why not just do the same pattern as in above?
if (ret)
goto err_out;return 0;
-Mika
Quoted 169 lines
drivers/gpu/drm/i915/i915_gem_gtt.c +20
-16+err_out: +gen8_ppgtt_free (ppgtt);return ret; }@@ -573,7 +572,7 @@ static int gen8_ppgtt_setup_page_directories(struct i915_hw_ppgtt *ppgtt,int ret; pd_addr =pci_map_page (ppgtt->base.dev->pdev, - &ppgtt->pd_pages[pd],0 , + ppgtt->pdp.page_directory[pd].page,0 ,PAGE_SIZE ,PCI_DMA_BIDIRECTIONAL ); ret =pci_dma_mapping_error (ppgtt->base.dev->pdev, pd_addr);@@ -593,7 +592,7 @@ static int gen8_ppgtt_setup_page_tables(struct i915_hw_ppgtt *ppgtt,struct page *p;int ret; - p = ppgtt->gen8_pt_pages[pd][pt]; + p = ppgtt->pdp.page_directory[pd].page_tables[pt].page; pt_addr =pci_map_page (ppgtt->base.dev->pdev, p,0 ,PAGE_SIZE ,PCI_DMA_BIDIRECTIONAL ); ret =pci_dma_mapping_error (ppgtt->base.dev->pdev, pt_addr);@@ -654,7 +653,7 @@ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size)*/ for (i =0 ; i < max_pdp; i++) {gen8_ppgtt_pde_t *pd_vaddr; - pd_vaddr =kmap_atomic (&ppgtt->pd_pages[i]); + pd_vaddr =kmap_atomic (ppgtt->pdp.page_directory[i].page);for (j =0 ; j <GEN8_PDES_PER_PAGE ; j++) {dma_addr_t addr = ppgtt->gen8_pt_dma_addr[i][j]; pd_vaddr[j] = gen8_pde_encode(ppgtt->base.dev, addr,@@ -717,7 +716,7 @@ static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m) expected);seq_printf (m,"\tPDE: %x\n" , pd_entry); - pt_vaddr =kmap_atomic (ppgtt->pt_pages[pde]); + pt_vaddr =kmap_atomic (ppgtt->pd.page_tables[pde].page);for (pte =0 ; pte <I915_PPGTT_PT_ENTRIES ; pte+=4 ) {unsigned long va = (pde * PAGE_SIZE * I915_PPGTT_PT_ENTRIES) +@@ -922,7 +921,7 @@ static void gen6_ppgtt_clear_range(struct i915_address_space *vm,if (last_pte >I915_PPGTT_PT_ENTRIES ) last_pte =I915_PPGTT_PT_ENTRIES ; - pt_vaddr =kmap_atomic (ppgtt->pt_pages[act_pt]); + pt_vaddr =kmap_atomic (ppgtt->pd.page_tables[act_pt].page);for (i = first_pte; i < last_pte; i++) pt_vaddr[i] = scratch_pte;@@ -951,7 +950,7 @@ static void gen6_ppgtt_insert_entries(struct i915_address_space *vm, pt_vaddr =NULL ;for_each_sg_page (pages->sgl, &sg_iter, pages->nents,0 ) {if (pt_vaddr ==NULL ) - pt_vaddr =kmap_atomic (ppgtt->pt_pages[act_pt]); + pt_vaddr =kmap_atomic (ppgtt->pd.page_tables[act_pt].page); pt_vaddr[act_pte] = vm->pte_encode(sg_page_iter_dma_address(&sg_iter),@@ -986,8 +985,8 @@ static void gen6_ppgtt_free(struct i915_hw_ppgtt *ppgtt)kfree (ppgtt->pt_dma_addr);for (i =0 ; i < ppgtt->num_pd_entries; i++) -__free_page (ppgtt->pt_pages[i]); -kfree (ppgtt->pt_pages); +__free_page (ppgtt->pd.page_tables[i].page); +kfree (ppgtt->pd.page_tables); }static void gen6_ppgtt_cleanup (struct i915_address_space *vm)@@ -1044,22 +1043,22 @@ alloc:static int gen6_ppgtt_allocate_page_tables (struct i915_hw_ppgtt *ppgtt) { +struct i915_page_table_entry *pt;int i; - ppgtt->pt_pages =kcalloc (ppgtt->num_pd_entries,sizeof (struct page *), -GFP_KERNEL ); - -if (!ppgtt->pt_pages) + pt =kcalloc (ppgtt->num_pd_entries,sizeof (*pt),GFP_KERNEL ); +if (!pt)return -ENOMEM ;for (i =0 ; i < ppgtt->num_pd_entries; i++) { - ppgtt->pt_pages[i] =alloc_page (GFP_KERNEL ); -if (!ppgtt->pt_pages[i]) { + pt[i].page =alloc_page (GFP_KERNEL ); +if (!pt->page) {gen6_ppgtt_free (ppgtt);return -ENOMEM ; } } + ppgtt->pd.page_tables = pt;return 0 ; }@@ -1094,9 +1093,11 @@ static int gen6_ppgtt_setup_page_tables(struct i915_hw_ppgtt *ppgtt)int i;for (i =0 ; i < ppgtt->num_pd_entries; i++) { +struct page *page;dma_addr_t pt_addr; - pt_addr =pci_map_page (dev->pdev, ppgtt->pt_pages[i],0 ,4096 , + page = ppgtt->pd.page_tables[i].page; + pt_addr =pci_map_page (dev->pdev, page,0 ,4096 ,PCI_DMA_BIDIRECTIONAL );if (pci_dma_mapping_error (dev->pdev, pt_addr)) {@@ -1140,7 +1141,7 @@ static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt) ppgtt->base.insert_entries = gen6_ppgtt_insert_entries; ppgtt->base.cleanup = gen6_ppgtt_cleanup; ppgtt->base.start =0 ; - ppgtt->base.total = ppgtt->num_pd_entries *I915_PPGTT_PT_ENTRIES *PAGE_SIZE ; + ppgtt->base.total = ppgtt->num_pd_entries *I915_PPGTT_PT_ENTRIES *PAGE_SIZE ; ppgtt->debug_dump = gen6_dump_ppgtt; ppgtt->pd_offset =drivers/gpu/drm/i915/i915_gem_gtt.h +18
-5@@ -187,6 +187,20 @@ struct i915_vma { u32 flags); }; +struct i915_page_table_entry { +struct page *page; +}; + +struct i915_page_directory_entry { +struct page *page;/* NULL for GEN6-GEN7 */ +struct i915_page_table_entry *page_tables; +}; + +struct i915_page_directory_pointer_entry { +/* struct page *page; */ +struct i915_page_directory_entry page_directory[GEN8_LEGACY_PDPES ]; +}; +struct i915_address_space {struct drm_mm mm;struct drm_device *dev;@@ -272,11 +286,6 @@ struct i915_hw_ppgtt {unsigned num_pd_entries;unsigned num_pd_pages;/* gen8+ */ union { -struct page **pt_pages; -struct page **gen8_pt_pages[GEN8_LEGACY_PDPES ]; - }; -struct page *pd_pages; -union {uint32_t pd_offset;dma_addr_t pd_dma_addr[GEN8_LEGACY_PDPES ]; };@@ -284,6 +293,10 @@ struct i915_hw_ppgtt {dma_addr_t *pt_dma_addr;dma_addr_t *gen8_pt_dma_addr[GEN8_LEGACY_PDPES ]; }; +union { +struct i915_page_directory_pointer_entry pdp; +struct i915_page_directory_entry pd; + };struct drm_i915_file_private *file_priv;
Quoted 7 lines
Quoted 7 lines
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
_______________________________________________
Intel-gfx mailing list
[email protected]
http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH v4 6/24 drm/i915: Complete page table structures
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
Move the remaining members over to the new page table structures.
This can be squashed with the previous commit if desire. The reasoning is the same as that patch. I simply felt it is easier to review if split.
v2: In lrc: s/ppgtt->pd_dma_addr[i]/ppgtt->pdp.page_directory[i].daddr/
v3: Rebase.
--- drivers/gpu/drm/i915/i915_debugfs.c | 2 +- drivers/gpu/drm/i915/i915_gem_gtt.c | 85 +++++++++++++------------------------ drivers/gpu/drm/i915/i915_gem_gtt.h | 14 +++--- drivers/gpu/drm/i915/intel_lrc.c | 16 +++---- 4 files changed, 45 insertions(+), 72 deletions(-)
drivers/gpu/drm/i915/i915_debugfs.c +1 -1
@@ -2153,7 +2153,7 @@ static void gen6_ppgtt_info(struct seq_file *m, struct drm_device *dev)struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;seq_puts (m,"aliasing PPGTT:\n" ); -seq_printf (m,"pd gtt offset: 0x%08x\n" , ppgtt->pd_offset); +seq_printf (m,"pd gtt offset: 0x%08x\n" , ppgtt->pd.pd_offset); ppgtt->debug_dump (ppgtt, m); }
drivers/gpu/drm/i915/i915_gem_gtt.c +30 -55
220 lines not on this page · raw message
drivers/gpu/drm/i915/i915_gem_gtt.h +6 -8
@@ -189,10 +189,16 @@ struct i915_vma {struct i915_page_table_entry {struct page *page; +dma_addr_t daddr; };struct i915_page_directory_entry {struct page *page;/* NULL for GEN6-GEN7 */ +union { +uint32_t pd_offset; +dma_addr_t daddr; + }; +struct i915_page_table_entry *page_tables; };
@@ -286,14 +292,6 @@ struct i915_hw_ppgtt {unsigned num_pd_entries;unsigned num_pd_pages;/* gen8+ */ union { -uint32_t pd_offset; -dma_addr_t pd_dma_addr[GEN8_LEGACY_PDPES ]; - }; -union { -dma_addr_t *pt_dma_addr; -dma_addr_t *gen8_pt_dma_addr[GEN8_LEGACY_PDPES ]; - }; -union {struct i915_page_directory_pointer_entry pdp;struct i915_page_directory_entry pd; };
drivers/gpu/drm/i915/intel_lrc.c +8 -8
@@ -1731,14 +1731,14 @@ populate_lr_context(struct intel_context *ctx, struct drm_i915_gem_object *ctx_o reg_state[CTX_PDP1_LDW ] =GEN8_RING_PDP_LDW (ring,1 ); reg_state[CTX_PDP0_UDW ] =GEN8_RING_PDP_UDW (ring,0 ); reg_state[CTX_PDP0_LDW ] =GEN8_RING_PDP_LDW (ring,0 ); - reg_state[CTX_PDP3_UDW +1 ] =upper_32_bits (ppgtt->pd_dma_addr[3 ]); - reg_state[CTX_PDP3_LDW +1 ] =lower_32_bits (ppgtt->pd_dma_addr[3 ]); - reg_state[CTX_PDP2_UDW +1 ] =upper_32_bits (ppgtt->pd_dma_addr[2 ]); - reg_state[CTX_PDP2_LDW +1 ] =lower_32_bits (ppgtt->pd_dma_addr[2 ]); - reg_state[CTX_PDP1_UDW +1 ] =upper_32_bits (ppgtt->pd_dma_addr[1 ]); - reg_state[CTX_PDP1_LDW +1 ] =lower_32_bits (ppgtt->pd_dma_addr[1 ]); - reg_state[CTX_PDP0_UDW +1 ] =upper_32_bits (ppgtt->pd_dma_addr[0 ]); - reg_state[CTX_PDP0_LDW +1 ] =lower_32_bits (ppgtt->pd_dma_addr[0 ]); + reg_state[CTX_PDP3_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[3 ].daddr); + reg_state[CTX_PDP3_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[3 ].daddr); + reg_state[CTX_PDP2_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[2 ].daddr); + reg_state[CTX_PDP2_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[2 ].daddr); + reg_state[CTX_PDP1_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[1 ].daddr); + reg_state[CTX_PDP1_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[1 ].daddr); + reg_state[CTX_PDP0_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[0 ].daddr); + reg_state[CTX_PDP0_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[0 ].daddr);if (ring->id ==RCS ) { reg_state[CTX_LRI_HEADER_2 ] =MI_LOAD_REGISTER_IMM (1 ); reg_state[CTX_R_PWR_CLK_STATE ] =0x20c8 ;
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH v4 7/24 drm/i915: Create page table allocators
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
As we move toward dynamic page table allocation, it becomes much easier to manage our data structures if break do things less coarsely by breaking up all of our actions into individual tasks. This makes the code easier to write, read, and verify.
Aside from the dissection of the allocation functions, the patch statically allocates the page table structures without a page directory. This remains the same for all platforms,
The patch itself should not have much functional difference. The primary noticeable difference is the fact that page tables are no longer allocated, but rather statically declared as part of the page directory. This has non-zero overhead, but things gain non-trivial complexity as a result.
This patch exists for a few reasons:
1. Splitting out the functions allows easily combining GEN6 and GEN8 code. Page tables have no difference based on GEN8. As we'll see in a future patch when we add the DMA mappings to the allocations, it requires only one small change to make work, and error handling should just fall into place.
2. Unless we always want to allocate all page tables under a given PDE, we'll have to eventually break this up into an array of pointers (or pointer to pointer).
3. Having the discrete functions is easier to review, and understand. All allocations and frees now take place in just a couple of locations. Reviewing, and catching leaks should be easy.
4. Less important: the GFP flags are confined to one location, which makes playing around with such things trivial.
v2: Updated commit message to explain why this patch exists
v3: For lrc, s/pdp.page_directory[i].daddr/pdp.page_directory[i]->daddr/
v4: Renamed free_pt/pd_single functions to unmap_and_free_pt/pd (Daniel)
v5: Added additional safety checks in gen8 clear/free/unmap.
--- drivers/gpu/drm/i915/i915_gem_gtt.c | 251 ++++++++++++++++++++++++------------ drivers/gpu/drm/i915/i915_gem_gtt.h | 4 +- drivers/gpu/drm/i915/intel_lrc.c | 16 +-- 3 files changed, 179 insertions(+), 92 deletions(-)
drivers/gpu/drm/i915/i915_gem_gtt.c +169 -82
@@ -275,6 +275,99 @@ static gen6_gtt_pte_t iris_pte_encode(dma_addr_t addr,return pte; } +static void unmap_and_free_pt (struct i915_page_table_entry *pt) +{ +if (WARN_ON (!pt->page)) +return ; +__free_page (pt->page); +kfree (pt); +} + +static struct i915_page_table_entry *alloc_pt_single (void ) +{ +struct i915_page_table_entry *pt; + + pt =kzalloc (sizeof (*pt),GFP_KERNEL ); +if (!pt) +return ERR_PTR (-ENOMEM ); + + pt->page =alloc_page (GFP_KERNEL | __GFP_ZERO); +if (!pt->page) { +kfree (pt); +return ERR_PTR (-ENOMEM ); + } + +return pt; +} + +/** +* alloc_pt_range() - Allocate a multiple page tables +* @pd: The page directory which will have at least @count entries +* available to point to the allocated page tables. +* @pde: First page directory entry for which we are allocating. +* @count: Number of pages to allocate. +* +* Allocates multiple page table pages and sets the appropriate entries in the +* page table structure within the page directory. Function cleans up after +* itself on any failures. +* +* Return: 0 if allocation succeeded. +*/ +static int alloc_pt_range (struct i915_page_directory_entry *pd,uint16_t pde,size_t count) +{ +int i, ret; + +/* 512 is the max page tables per page_directory on any platform. +* TODO: make WARN after patch series is done +*/ +BUG_ON (pde + count >GEN6_PPGTT_PD_ENTRIES ); + +for (i = pde; i < pde + count; i++) { +struct i915_page_table_entry *pt =alloc_pt_single (); + +if (IS_ERR (pt)) { + ret =PTR_ERR (pt); + goto err_out; + } +WARN (pd->page_tables[i], +"Leaking page directory entry %d (%pa)\n" , + i, pd->page_tables[i]); + pd->page_tables[i] = pt; + } + +return 0 ; + +err_out: +while (i--) +unmap_and_free_pt (pd->page_tables[i]); +return ret; +} + +static void unmap_and_free_pd (struct i915_page_directory_entry *pd) +{ +if (pd->page) { +__free_page (pd->page); +kfree (pd); + } +} + +static struct i915_page_directory_entry *alloc_pd_single (void ) +{ +struct i915_page_directory_entry *pd; + + pd =kzalloc (sizeof (*pd),GFP_KERNEL ); +if (!pd) +return ERR_PTR (-ENOMEM ); + + pd->page =alloc_page (GFP_KERNEL | __GFP_ZERO); +if (!pd->page) { +kfree (pd); +return ERR_PTR (-ENOMEM ); + } + +return pd; +} +/* Broadwell Page Directory Pointer Descriptors */ static int gen8_write_pdp (struct intel_engine_cs *ring,unsigned entry,uint64_t val)
@@ -307,7 +400,7 @@ static int gen8_mm_switch(struct i915_hw_ppgtt *ppgtt,int used_pd = ppgtt->num_pd_entries /GEN8_PDES_PER_PAGE ;for (i = used_pd -1 ; i >=0 ; i--) { -dma_addr_t addr = ppgtt->pdp.page_directory[i].daddr; +dma_addr_t addr = ppgtt->pdp.page_directory[i]->daddr; ret =gen8_write_pdp (ring, i, addr);if (ret)return ret;
@@ -334,8 +427,24 @@ static void gen8_ppgtt_clear_range(struct i915_address_space *vm, I915_CACHE_LLC, use_scratch);while (num_entries) { -struct i915_page_directory_entry *pd = &ppgtt->pdp.page_directory[pdpe]; -struct page *page_table = pd->page_tables[pde].page; +struct i915_page_directory_entry *pd; +struct i915_page_table_entry *pt; +struct page *page_table; + +if (WARN_ON (!ppgtt->pdp.page_directory[pdpe])) +continue ; + + pd = ppgtt->pdp.page_directory[pdpe]; + +if (WARN_ON (!pd->page_tables[pde])) +continue ; + + pt = pd->page_tables[pde]; + +if (WARN_ON (!pt->page)) +continue ; + + page_table = pt->page; last_pte = pte + num_entries;if (last_pte >GEN8_PTES_PER_PAGE )
@@ -380,8 +489,9 @@ static void gen8_ppgtt_insert_entries(struct i915_address_space *vm,break ;if (pt_vaddr ==NULL ) { -struct i915_page_directory_entry *pd = &ppgtt->pdp.page_directory[pdpe]; -struct page *page_table = pd->page_tables[pde].page; +struct i915_page_directory_entry *pd = ppgtt->pdp.page_directory[pdpe]; +struct i915_page_table_entry *pt = pd->page_tables[pde]; +struct page *page_table = pt->page; pt_vaddr =kmap_atomic (page_table); }
@@ -412,18 +522,16 @@ static void gen8_free_page_tables(struct i915_page_directory_entry *pd) {int i; -if (pd->page_tables ==NULL ) +if (!pd->page)return ; -for (i =0 ; i <GEN8_PDES_PER_PAGE ; i++) -if (pd->page_tables[i].page) -__free_page (pd->page_tables[i].page); -} +for (i =0 ; i <GEN8_PDES_PER_PAGE ; i++) { +if (WARN_ON (!pd->page_tables[i])) +continue ; -static void gen8_free_page_directories (struct i915_page_directory_entry *pd) -{ -kfree (pd->page_tables); -__free_page (pd->page); +unmap_and_free_pt (pd->page_tables[i]); + pd->page_tables[i] =NULL ; + } }static void gen8_ppgtt_free (struct i915_hw_ppgtt *ppgtt)
@@ -431,8 +539,11 @@ static void gen8_ppgtt_free(struct i915_hw_ppgtt *ppgtt)int i;for (i =0 ; i < ppgtt->num_pd_pages; i++) { -gen8_free_page_tables (&ppgtt->pdp.page_directory[i]); -gen8_free_page_directories (&ppgtt->pdp.page_directory[i]); +if (WARN_ON (!ppgtt->pdp.page_directory[i])) +continue ; + +gen8_free_page_tables (ppgtt->pdp.page_directory[i]); +unmap_and_free_pd (ppgtt->pdp.page_directory[i]); } }
@@ -444,14 +555,23 @@ static void gen8_ppgtt_unmap_pages(struct i915_hw_ppgtt *ppgtt)for (i =0 ; i < ppgtt->num_pd_pages; i++) {/* TODO: In the future we'll support sparse mappings, so this * will have to change. */ -if (!ppgtt->pdp.page_directory[i].daddr) +if (!ppgtt->pdp.page_directory[i]->daddr)continue ; -pci_unmap_page (hwdev, ppgtt->pdp.page_directory[i].daddr,PAGE_SIZE , +pci_unmap_page (hwdev, ppgtt->pdp.page_directory[i]->daddr,PAGE_SIZE ,PCI_DMA_BIDIRECTIONAL );for (j =0 ; j <GEN8_PDES_PER_PAGE ; j++) { -dma_addr_t addr = ppgtt->pdp.page_directory[i].page_tables[j].daddr; +struct i915_page_directory_entry *pd = ppgtt->pdp.page_directory[i]; +struct i915_page_table_entry *pt; +dma_addr_t addr; + +if (WARN_ON (!pd->page_tables[j])) +continue ; + + pt = pd->page_tables[j]; + addr = pt->daddr; +if (addr)pci_unmap_page (hwdev, addr,PAGE_SIZE ,PCI_DMA_BIDIRECTIONAL );
@@ -470,25 +590,20 @@ static void gen8_ppgtt_cleanup(struct i915_address_space *vm)static int gen8_ppgtt_allocate_page_tables (struct i915_hw_ppgtt *ppgtt) { -int i, j; +int i, ret;for (i =0 ; i < ppgtt->num_pd_pages; i++) { -struct i915_page_directory_entry *pd = &ppgtt->pdp.page_directory[i]; -for (j =0 ; j <GEN8_PDES_PER_PAGE ; j++) { -struct i915_page_table_entry *pt = &pd->page_tables[j]; - - pt->page =alloc_page (GFP_KERNEL | __GFP_ZERO); -if (!pt->page) - goto unwind_out; - - } + ret =alloc_pt_range (ppgtt->pdp.page_directory[i], +0 ,GEN8_PDES_PER_PAGE ); +if (ret) + goto unwind_out; }return 0 ; unwind_out:while (i--) -gen8_free_page_tables (&ppgtt->pdp.page_directory[i]); +gen8_free_page_tables (ppgtt->pdp.page_directory[i]);return -ENOMEM ; }
@@ -499,17 +614,9 @@ static int gen8_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt,int i;for (i =0 ; i < max_pdp; i++) { -struct i915_page_table_entry *pt; - - pt =kcalloc (GEN8_PDES_PER_PAGE ,sizeof (*pt),GFP_KERNEL ); -if (!pt) + ppgtt->pdp.page_directory[i] =alloc_pd_single (); +if (IS_ERR (ppgtt->pdp.page_directory[i])) goto unwind_out; - - ppgtt->pdp.page_directory[i].page =alloc_page (GFP_KERNEL ); -if (!ppgtt->pdp.page_directory[i].page) - goto unwind_out; - - ppgtt->pdp.page_directory[i].page_tables = pt; } ppgtt->num_pd_pages = max_pdp;
@@ -518,10 +625,8 @@ static int gen8_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt,return 0 ; unwind_out: -while (i--) { -kfree (ppgtt->pdp.page_directory[i].page_tables); -__free_page (ppgtt->pdp.page_directory[i].page); - } +while (i--) +unmap_and_free_pd (ppgtt->pdp.page_directory[i]);return -ENOMEM ; }
@@ -556,14 +661,14 @@ static int gen8_ppgtt_setup_page_directories(struct i915_hw_ppgtt *ppgtt,int ret; pd_addr =pci_map_page (ppgtt->base.dev->pdev, - ppgtt->pdp.page_directory[pd].page,0 , + ppgtt->pdp.page_directory[pd]->page,0 ,PAGE_SIZE ,PCI_DMA_BIDIRECTIONAL ); ret =pci_dma_mapping_error (ppgtt->base.dev->pdev, pd_addr);if (ret)return ret; - ppgtt->pdp.page_directory[pd].daddr = pd_addr; + ppgtt->pdp.page_directory[pd]->daddr = pd_addr;return 0 ; }
@@ -573,8 +678,8 @@ static int gen8_ppgtt_setup_page_tables(struct i915_hw_ppgtt *ppgtt, const int pt) {dma_addr_t pt_addr; -struct i915_page_directory_entry *pdir = &ppgtt->pdp.page_directory[pd]; -struct i915_page_table_entry *ptab = &pdir->page_tables[pt]; +struct i915_page_directory_entry *pdir = ppgtt->pdp.page_directory[pd]; +struct i915_page_table_entry *ptab = pdir->page_tables[pt];struct page *p = ptab->page;int ret;
@@ -637,10 +742,12 @@ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size)* will never need to touch the PDEs again. */ for (i =0 ; i < max_pdp; i++) { +struct i915_page_directory_entry *pd = ppgtt->pdp.page_directory[i];gen8_ppgtt_pde_t *pd_vaddr; - pd_vaddr =kmap_atomic (ppgtt->pdp.page_directory[i].page); + pd_vaddr =kmap_atomic (ppgtt->pdp.page_directory[i]->page);for (j =0 ; j <GEN8_PDES_PER_PAGE ; j++) { -dma_addr_t addr = ppgtt->pdp.page_directory[i].page_tables[j].daddr; +struct i915_page_table_entry *pt = pd->page_tables[j]; +dma_addr_t addr = pt->daddr; pd_vaddr[j] =gen8_pde_encode (ppgtt->base.dev, addr,I915_CACHE_LLC ); }
@@ -691,7 +798,7 @@ static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m)for (pde =0 ; pde < ppgtt->num_pd_entries; pde++) {u32 expected;gen6_gtt_pte_t *pt_vaddr; -dma_addr_t pt_addr = ppgtt->pd.page_tables[pde].daddr; +dma_addr_t pt_addr = ppgtt->pd.page_tables[pde]->daddr; pd_entry =readl (pd_addr + pde); expected = (GEN6_PDE_ADDR_ENCODE (pt_addr) |GEN6_PDE_VALID );
@@ -702,7 +809,7 @@ static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m) expected);seq_printf (m,"\tPDE: %x\n" , pd_entry); - pt_vaddr =kmap_atomic (ppgtt->pd.page_tables[pde].page); + pt_vaddr =kmap_atomic (ppgtt->pd.page_tables[pde]->page);for (pte =0 ; pte <I915_PPGTT_PT_ENTRIES ; pte+=4 ) {unsigned long va = (pde * PAGE_SIZE * I915_PPGTT_PT_ENTRIES) +
@@ -741,7 +848,7 @@ static void gen6_write_pdes(struct i915_hw_ppgtt *ppgtt)for (i =0 ; i < ppgtt->num_pd_entries; i++) {dma_addr_t pt_addr; - pt_addr = ppgtt->pd.page_tables[i].daddr; + pt_addr = ppgtt->pd.page_tables[i]->daddr; pd_entry =GEN6_PDE_ADDR_ENCODE (pt_addr); pd_entry |=GEN6_PDE_VALID ;
@@ -907,7 +1014,7 @@ static void gen6_ppgtt_clear_range(struct i915_address_space *vm,if (last_pte >I915_PPGTT_PT_ENTRIES ) last_pte =I915_PPGTT_PT_ENTRIES ; - pt_vaddr =kmap_atomic (ppgtt->pd.page_tables[act_pt].page); + pt_vaddr =kmap_atomic (ppgtt->pd.page_tables[act_pt]->page);for (i = first_pte; i < last_pte; i++) pt_vaddr[i] = scratch_pte;
@@ -936,7 +1043,7 @@ static void gen6_ppgtt_insert_entries(struct i915_address_space *vm, pt_vaddr =NULL ;for_each_sg_page (pages->sgl, &sg_iter, pages->nents,0 ) {if (pt_vaddr ==NULL ) - pt_vaddr =kmap_atomic (ppgtt->pd.page_tables[act_pt].page); + pt_vaddr =kmap_atomic (ppgtt->pd.page_tables[act_pt]->page); pt_vaddr[act_pte] = vm->pte_encode(sg_page_iter_dma_address(&sg_iter),
@@ -959,7 +1066,7 @@ static void gen6_ppgtt_unmap_pages(struct i915_hw_ppgtt *ppgtt)for (i =0 ; i < ppgtt->num_pd_entries; i++)pci_unmap_page (ppgtt->base.dev->pdev, - ppgtt->pd.page_tables[i].daddr, + ppgtt->pd.page_tables[i]->daddr,4096 ,PCI_DMA_BIDIRECTIONAL ); }
@@ -968,8 +1075,9 @@ static void gen6_ppgtt_free(struct i915_hw_ppgtt *ppgtt)int i;for (i =0 ; i < ppgtt->num_pd_entries; i++) -__free_page (ppgtt->pd.page_tables[i].page); -kfree (ppgtt->pd.page_tables); +unmap_and_free_pt (ppgtt->pd.page_tables[i]); + +unmap_and_free_pd (&ppgtt->pd); }static void gen6_ppgtt_cleanup (struct i915_address_space *vm)
@@ -1024,27 +1132,6 @@ alloc:return 0 ; } -static int gen6_ppgtt_allocate_page_tables (struct i915_hw_ppgtt *ppgtt) -{ -struct i915_page_table_entry *pt; -int i; - - pt =kcalloc (ppgtt->num_pd_entries,sizeof (*pt),GFP_KERNEL ); -if (!pt) -return -ENOMEM ; - -for (i =0 ; i < ppgtt->num_pd_entries; i++) { - pt[i].page =alloc_page (GFP_KERNEL ); -if (!pt->page) { -gen6_ppgtt_free (ppgtt); -return -ENOMEM ; - } - } - - ppgtt->pd.page_tables = pt; -return 0 ; -} -static int gen6_ppgtt_alloc (struct i915_hw_ppgtt *ppgtt) {int ret;
@@ -1053,7 +1140,7 @@ static int gen6_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt)if (ret)return ret; - ret =gen6_ppgtt_allocate_page_tables (ppgtt); + ret =alloc_pt_range (&ppgtt->pd,0 , ppgtt->num_pd_entries);if (ret) {drm_mm_remove_node (&ppgtt->node);return ret;
@@ -1071,7 +1158,7 @@ static int gen6_ppgtt_setup_page_tables(struct i915_hw_ppgtt *ppgtt)struct page *page;dma_addr_t pt_addr; - page = ppgtt->pd.page_tables[i].page; + page = ppgtt->pd.page_tables[i]->page; pt_addr =pci_map_page (dev->pdev, page,0 ,4096 ,PCI_DMA_BIDIRECTIONAL );
@@ -1080,7 +1167,7 @@ static int gen6_ppgtt_setup_page_tables(struct i915_hw_ppgtt *ppgtt)return -EIO ; } - ppgtt->pd.page_tables[i].daddr = pt_addr; + ppgtt->pd.page_tables[i]->daddr = pt_addr; }return 0 ;
drivers/gpu/drm/i915/i915_gem_gtt.h +2 -2
@@ -199,12 +199,12 @@ struct i915_page_directory_entry {dma_addr_t daddr; }; -struct i915_page_table_entry *page_tables; +struct i915_page_table_entry *page_tables[GEN6_PPGTT_PD_ENTRIES ];/* PDEs */ };struct i915_page_directory_pointer_entry {/* struct page *page; */ -struct i915_page_directory_entry page_directory[GEN8_LEGACY_PDPES ]; +struct i915_page_directory_entry *page_directory[GEN8_LEGACY_PDPES ]; };struct i915_address_space {
drivers/gpu/drm/i915/intel_lrc.c +8 -8
@@ -1731,14 +1731,14 @@ populate_lr_context(struct intel_context *ctx, struct drm_i915_gem_object *ctx_o reg_state[CTX_PDP1_LDW ] =GEN8_RING_PDP_LDW (ring,1 ); reg_state[CTX_PDP0_UDW ] =GEN8_RING_PDP_UDW (ring,0 ); reg_state[CTX_PDP0_LDW ] =GEN8_RING_PDP_LDW (ring,0 ); - reg_state[CTX_PDP3_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[3 ].daddr); - reg_state[CTX_PDP3_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[3 ].daddr); - reg_state[CTX_PDP2_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[2 ].daddr); - reg_state[CTX_PDP2_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[2 ].daddr); - reg_state[CTX_PDP1_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[1 ].daddr); - reg_state[CTX_PDP1_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[1 ].daddr); - reg_state[CTX_PDP0_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[0 ].daddr); - reg_state[CTX_PDP0_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[0 ].daddr); + reg_state[CTX_PDP3_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[3 ]->daddr); + reg_state[CTX_PDP3_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[3 ]->daddr); + reg_state[CTX_PDP2_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[2 ]->daddr); + reg_state[CTX_PDP2_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[2 ]->daddr); + reg_state[CTX_PDP1_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[1 ]->daddr); + reg_state[CTX_PDP1_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[1 ]->daddr); + reg_state[CTX_PDP0_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[0 ]->daddr); + reg_state[CTX_PDP0_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[0 ]->daddr);if (ring->id ==RCS ) { reg_state[CTX_LRI_HEADER_2 ] =MI_LOAD_REGISTER_IMM (1 ); reg_state[CTX_R_PWR_CLK_STATE ] =0x20c8 ;
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
Re: [PATCH v4 07/24] drm/i915: Create page table allocators
Quoted 96 lines · Michel Thierry
From: Ben Widawsky <[email protected]>
As we move toward dynamic page table allocation, it becomes much easier to manage our data structures if break do things less coarsely by breaking up all of our actions into individual tasks. This makes the code easier to write, read, and verify.
Aside from the dissection of the allocation functions, the patch statically allocates the page table structures without a page directory. This remains the same for all platforms,
The patch itself should not have much functional difference. The primary noticeable difference is the fact that page tables are no longer allocated, but rather statically declared as part of the page directory. This has non-zero overhead, but things gain non-trivial complexity as a result.
This patch exists for a few reasons:
1. Splitting out the functions allows easily combining GEN6 and GEN8 code. Page tables have no difference based on GEN8. As we'll see in a future patch when we add the DMA mappings to the allocations, it requires only one small change to make work, and error handling should just fall into place.2. Unless we always want to allocate all page tables under a given PDE, we'll have to eventually break this up into an array of pointers (or pointer to pointer).
3. Having the discrete functions is easier to review, and understand. All allocations and frees now take place in just a couple of locations. Reviewing, and catching leaks should be easy.
4. Less important: the GFP flags are confined to one location, which makes playing around with such things trivial.
v2: Updated commit message to explain why this patch exists
v3: For lrc, s/pdp.page_directory[i].daddr/pdp.page_directory[i]->daddr/
v4: Renamed free_pt/pd_single functions to unmap_and_free_pt/pd (Daniel)
v5: Added additional safety checks in gen8 clear/free/unmap.
Signed-off-by Ben Widawsky <[email protected]>Signed-off-by Michel Thierry <[email protected]> (v3, v4, v5)--- drivers/gpu/drm/i915/i915_gem_gtt.c | 251 ++++++++++++++++++++++++------------ drivers/gpu/drm/i915/i915_gem_gtt.h | 4 +- drivers/gpu/drm/i915/intel_lrc.c | 16 +-- 3 files changed, 179 insertions(+), 92 deletions(-)drivers/gpu/drm/i915/i915_gem_gtt.c +37
-0@@ -275,6 +275,99 @@ static gen6_gtt_pte_t iris_pte_encode(dma_addr_t addr,return pte; } +static void unmap_and_free_pt (struct i915_page_table_entry *pt) +{ +if (WARN_ON (!pt->page)) +return ; +__free_page (pt->page); +kfree (pt); +} + +static struct i915_page_table_entry *alloc_pt_single (void ) +{ +struct i915_page_table_entry *pt; + + pt =kzalloc (sizeof (*pt),GFP_KERNEL ); +if (!pt) +return ERR_PTR (-ENOMEM ); + + pt->page =alloc_page (GFP_KERNEL | __GFP_ZERO); +if (!pt->page) { +kfree (pt); +return ERR_PTR (-ENOMEM ); + } + +return pt; +} + +/** +* alloc_pt_range() - Allocate a multiple page tables +* @pd: The page directory which will have at least @count entries +* available to point to the allocated page tables. +* @pde: First page directory entry for which we are allocating. +* @count: Number of pages to allocate. +* +* Allocates multiple page table pages and sets the appropriate entries in the +* page table structure within the page directory. Function cleans up after +* itself on any failures. +* +* Return: 0 if allocation succeeded.
drivers/gpu/drm/i915/i915_gem_gtt.c static gen6_gtt_pte_t iris_pte_encode(dma_addr_t addr, · line 324
Quoted 10 lines
drivers/gpu/drm/i915/i915_gem_gtt.c +10
-0+*/ +static int alloc_pt_range (struct i915_page_directory_entry *pd,uint16_t pde,size_t count) +{ +int i, ret; + +/* 512 is the max page tables per page_directory on any platform. +* TODO: make WARN after patch series is done +*/ +BUG_ON (pde + count >GEN6_PPGTT_PD_ENTRIES ); +
WARN_ON in here and return -EINVAL.
-Mika
Quoted 446 lines
drivers/gpu/drm/i915/i915_gem_gtt.c +122
-82+for (i = pde; i < pde + count; i++) { +struct i915_page_table_entry *pt =alloc_pt_single (); + +if (IS_ERR (pt)) { + ret =PTR_ERR (pt); + goto err_out; + } +WARN (pd->page_tables[i], +"Leaking page directory entry %d (%pa)\n" , + i, pd->page_tables[i]); + pd->page_tables[i] = pt; + } + +return 0 ; + +err_out: +while (i--) +unmap_and_free_pt (pd->page_tables[i]); +return ret; +} + +static void unmap_and_free_pd (struct i915_page_directory_entry *pd) +{ +if (pd->page) { +__free_page (pd->page); +kfree (pd); + } +} + +static struct i915_page_directory_entry *alloc_pd_single (void ) +{ +struct i915_page_directory_entry *pd; + + pd =kzalloc (sizeof (*pd),GFP_KERNEL ); +if (!pd) +return ERR_PTR (-ENOMEM ); + + pd->page =alloc_page (GFP_KERNEL | __GFP_ZERO); +if (!pd->page) { +kfree (pd); +return ERR_PTR (-ENOMEM ); + } + +return pd; +} +/* Broadwell Page Directory Pointer Descriptors */ static int gen8_write_pdp (struct intel_engine_cs *ring,unsigned entry,uint64_t val)@@ -307,7 +400,7 @@ static int gen8_mm_switch(struct i915_hw_ppgtt *ppgtt,int used_pd = ppgtt->num_pd_entries /GEN8_PDES_PER_PAGE ;for (i = used_pd -1 ; i >=0 ; i--) { -dma_addr_t addr = ppgtt->pdp.page_directory[i].daddr; +dma_addr_t addr = ppgtt->pdp.page_directory[i]->daddr; ret =gen8_write_pdp (ring, i, addr);if (ret)return ret;@@ -334,8 +427,24 @@ static void gen8_ppgtt_clear_range(struct i915_address_space *vm, I915_CACHE_LLC, use_scratch);while (num_entries) { -struct i915_page_directory_entry *pd = &ppgtt->pdp.page_directory[pdpe]; -struct page *page_table = pd->page_tables[pde].page; +struct i915_page_directory_entry *pd; +struct i915_page_table_entry *pt; +struct page *page_table; + +if (WARN_ON (!ppgtt->pdp.page_directory[pdpe])) +continue ; + + pd = ppgtt->pdp.page_directory[pdpe]; + +if (WARN_ON (!pd->page_tables[pde])) +continue ; + + pt = pd->page_tables[pde]; + +if (WARN_ON (!pt->page)) +continue ; + + page_table = pt->page; last_pte = pte + num_entries;if (last_pte >GEN8_PTES_PER_PAGE )@@ -380,8 +489,9 @@ static void gen8_ppgtt_insert_entries(struct i915_address_space *vm,break ;if (pt_vaddr ==NULL ) { -struct i915_page_directory_entry *pd = &ppgtt->pdp.page_directory[pdpe]; -struct page *page_table = pd->page_tables[pde].page; +struct i915_page_directory_entry *pd = ppgtt->pdp.page_directory[pdpe]; +struct i915_page_table_entry *pt = pd->page_tables[pde]; +struct page *page_table = pt->page; pt_vaddr =kmap_atomic (page_table); }@@ -412,18 +522,16 @@ static void gen8_free_page_tables(struct i915_page_directory_entry *pd) {int i; -if (pd->page_tables ==NULL ) +if (!pd->page)return ; -for (i =0 ; i <GEN8_PDES_PER_PAGE ; i++) -if (pd->page_tables[i].page) -__free_page (pd->page_tables[i].page); -} +for (i =0 ; i <GEN8_PDES_PER_PAGE ; i++) { +if (WARN_ON (!pd->page_tables[i])) +continue ; -static void gen8_free_page_directories (struct i915_page_directory_entry *pd) -{ -kfree (pd->page_tables); -__free_page (pd->page); +unmap_and_free_pt (pd->page_tables[i]); + pd->page_tables[i] =NULL ; + } }static void gen8_ppgtt_free (struct i915_hw_ppgtt *ppgtt)@@ -431,8 +539,11 @@ static void gen8_ppgtt_free(struct i915_hw_ppgtt *ppgtt)int i;for (i =0 ; i < ppgtt->num_pd_pages; i++) { -gen8_free_page_tables (&ppgtt->pdp.page_directory[i]); -gen8_free_page_directories (&ppgtt->pdp.page_directory[i]); +if (WARN_ON (!ppgtt->pdp.page_directory[i])) +continue ; + +gen8_free_page_tables (ppgtt->pdp.page_directory[i]); +unmap_and_free_pd (ppgtt->pdp.page_directory[i]); } }@@ -444,14 +555,23 @@ static void gen8_ppgtt_unmap_pages(struct i915_hw_ppgtt *ppgtt)for (i =0 ; i < ppgtt->num_pd_pages; i++) {/* TODO: In the future we'll support sparse mappings, so this * will have to change. */ -if (!ppgtt->pdp.page_directory[i].daddr) +if (!ppgtt->pdp.page_directory[i]->daddr)continue ; -pci_unmap_page (hwdev, ppgtt->pdp.page_directory[i].daddr,PAGE_SIZE , +pci_unmap_page (hwdev, ppgtt->pdp.page_directory[i]->daddr,PAGE_SIZE ,PCI_DMA_BIDIRECTIONAL );for (j =0 ; j <GEN8_PDES_PER_PAGE ; j++) { -dma_addr_t addr = ppgtt->pdp.page_directory[i].page_tables[j].daddr; +struct i915_page_directory_entry *pd = ppgtt->pdp.page_directory[i]; +struct i915_page_table_entry *pt; +dma_addr_t addr; + +if (WARN_ON (!pd->page_tables[j])) +continue ; + + pt = pd->page_tables[j]; + addr = pt->daddr; +if (addr)pci_unmap_page (hwdev, addr,PAGE_SIZE ,PCI_DMA_BIDIRECTIONAL );@@ -470,25 +590,20 @@ static void gen8_ppgtt_cleanup(struct i915_address_space *vm)static int gen8_ppgtt_allocate_page_tables (struct i915_hw_ppgtt *ppgtt) { -int i, j; +int i, ret;for (i =0 ; i < ppgtt->num_pd_pages; i++) { -struct i915_page_directory_entry *pd = &ppgtt->pdp.page_directory[i]; -for (j =0 ; j <GEN8_PDES_PER_PAGE ; j++) { -struct i915_page_table_entry *pt = &pd->page_tables[j]; - - pt->page =alloc_page (GFP_KERNEL | __GFP_ZERO); -if (!pt->page) - goto unwind_out; - - } + ret =alloc_pt_range (ppgtt->pdp.page_directory[i], +0 ,GEN8_PDES_PER_PAGE ); +if (ret) + goto unwind_out; }return 0 ; unwind_out:while (i--) -gen8_free_page_tables (&ppgtt->pdp.page_directory[i]); +gen8_free_page_tables (ppgtt->pdp.page_directory[i]);return -ENOMEM ; }@@ -499,17 +614,9 @@ static int gen8_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt,int i;for (i =0 ; i < max_pdp; i++) { -struct i915_page_table_entry *pt; - - pt =kcalloc (GEN8_PDES_PER_PAGE ,sizeof (*pt),GFP_KERNEL ); -if (!pt) + ppgtt->pdp.page_directory[i] =alloc_pd_single (); +if (IS_ERR (ppgtt->pdp.page_directory[i])) goto unwind_out; - - ppgtt->pdp.page_directory[i].page =alloc_page (GFP_KERNEL ); -if (!ppgtt->pdp.page_directory[i].page) - goto unwind_out; - - ppgtt->pdp.page_directory[i].page_tables = pt; } ppgtt->num_pd_pages = max_pdp;@@ -518,10 +625,8 @@ static int gen8_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt,return 0 ; unwind_out: -while (i--) { -kfree (ppgtt->pdp.page_directory[i].page_tables); -__free_page (ppgtt->pdp.page_directory[i].page); - } +while (i--) +unmap_and_free_pd (ppgtt->pdp.page_directory[i]);return -ENOMEM ; }@@ -556,14 +661,14 @@ static int gen8_ppgtt_setup_page_directories(struct i915_hw_ppgtt *ppgtt,int ret; pd_addr =pci_map_page (ppgtt->base.dev->pdev, - ppgtt->pdp.page_directory[pd].page,0 , + ppgtt->pdp.page_directory[pd]->page,0 ,PAGE_SIZE ,PCI_DMA_BIDIRECTIONAL ); ret =pci_dma_mapping_error (ppgtt->base.dev->pdev, pd_addr);if (ret)return ret; - ppgtt->pdp.page_directory[pd].daddr = pd_addr; + ppgtt->pdp.page_directory[pd]->daddr = pd_addr;return 0 ; }@@ -573,8 +678,8 @@ static int gen8_ppgtt_setup_page_tables(struct i915_hw_ppgtt *ppgtt, const int pt) {dma_addr_t pt_addr; -struct i915_page_directory_entry *pdir = &ppgtt->pdp.page_directory[pd]; -struct i915_page_table_entry *ptab = &pdir->page_tables[pt]; +struct i915_page_directory_entry *pdir = ppgtt->pdp.page_directory[pd]; +struct i915_page_table_entry *ptab = pdir->page_tables[pt];struct page *p = ptab->page;int ret;@@ -637,10 +742,12 @@ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size)* will never need to touch the PDEs again. */ for (i =0 ; i < max_pdp; i++) { +struct i915_page_directory_entry *pd = ppgtt->pdp.page_directory[i];gen8_ppgtt_pde_t *pd_vaddr; - pd_vaddr =kmap_atomic (ppgtt->pdp.page_directory[i].page); + pd_vaddr =kmap_atomic (ppgtt->pdp.page_directory[i]->page);for (j =0 ; j <GEN8_PDES_PER_PAGE ; j++) { -dma_addr_t addr = ppgtt->pdp.page_directory[i].page_tables[j].daddr; +struct i915_page_table_entry *pt = pd->page_tables[j]; +dma_addr_t addr = pt->daddr; pd_vaddr[j] =gen8_pde_encode (ppgtt->base.dev, addr,I915_CACHE_LLC ); }@@ -691,7 +798,7 @@ static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m)for (pde =0 ; pde < ppgtt->num_pd_entries; pde++) {u32 expected;gen6_gtt_pte_t *pt_vaddr; -dma_addr_t pt_addr = ppgtt->pd.page_tables[pde].daddr; +dma_addr_t pt_addr = ppgtt->pd.page_tables[pde]->daddr; pd_entry =readl (pd_addr + pde); expected = (GEN6_PDE_ADDR_ENCODE (pt_addr) |GEN6_PDE_VALID );@@ -702,7 +809,7 @@ static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m) expected);seq_printf (m,"\tPDE: %x\n" , pd_entry); - pt_vaddr =kmap_atomic (ppgtt->pd.page_tables[pde].page); + pt_vaddr =kmap_atomic (ppgtt->pd.page_tables[pde]->page);for (pte =0 ; pte <I915_PPGTT_PT_ENTRIES ; pte+=4 ) {unsigned long va = (pde * PAGE_SIZE * I915_PPGTT_PT_ENTRIES) +@@ -741,7 +848,7 @@ static void gen6_write_pdes(struct i915_hw_ppgtt *ppgtt)for (i =0 ; i < ppgtt->num_pd_entries; i++) {dma_addr_t pt_addr; - pt_addr = ppgtt->pd.page_tables[i].daddr; + pt_addr = ppgtt->pd.page_tables[i]->daddr; pd_entry =GEN6_PDE_ADDR_ENCODE (pt_addr); pd_entry |=GEN6_PDE_VALID ;@@ -907,7 +1014,7 @@ static void gen6_ppgtt_clear_range(struct i915_address_space *vm,if (last_pte >I915_PPGTT_PT_ENTRIES ) last_pte =I915_PPGTT_PT_ENTRIES ; - pt_vaddr =kmap_atomic (ppgtt->pd.page_tables[act_pt].page); + pt_vaddr =kmap_atomic (ppgtt->pd.page_tables[act_pt]->page);for (i = first_pte; i < last_pte; i++) pt_vaddr[i] = scratch_pte;@@ -936,7 +1043,7 @@ static void gen6_ppgtt_insert_entries(struct i915_address_space *vm, pt_vaddr =NULL ;for_each_sg_page (pages->sgl, &sg_iter, pages->nents,0 ) {if (pt_vaddr ==NULL ) - pt_vaddr =kmap_atomic (ppgtt->pd.page_tables[act_pt].page); + pt_vaddr =kmap_atomic (ppgtt->pd.page_tables[act_pt]->page); pt_vaddr[act_pte] = vm->pte_encode(sg_page_iter_dma_address(&sg_iter),@@ -959,7 +1066,7 @@ static void gen6_ppgtt_unmap_pages(struct i915_hw_ppgtt *ppgtt)for (i =0 ; i < ppgtt->num_pd_entries; i++)pci_unmap_page (ppgtt->base.dev->pdev, - ppgtt->pd.page_tables[i].daddr, + ppgtt->pd.page_tables[i]->daddr,4096 ,PCI_DMA_BIDIRECTIONAL ); }@@ -968,8 +1075,9 @@ static void gen6_ppgtt_free(struct i915_hw_ppgtt *ppgtt)int i;for (i =0 ; i < ppgtt->num_pd_entries; i++) -__free_page (ppgtt->pd.page_tables[i].page); -kfree (ppgtt->pd.page_tables); +unmap_and_free_pt (ppgtt->pd.page_tables[i]); + +unmap_and_free_pd (&ppgtt->pd); }static void gen6_ppgtt_cleanup (struct i915_address_space *vm)@@ -1024,27 +1132,6 @@ alloc:return 0 ; } -static int gen6_ppgtt_allocate_page_tables (struct i915_hw_ppgtt *ppgtt) -{ -struct i915_page_table_entry *pt; -int i; - - pt =kcalloc (ppgtt->num_pd_entries,sizeof (*pt),GFP_KERNEL ); -if (!pt) -return -ENOMEM ; - -for (i =0 ; i < ppgtt->num_pd_entries; i++) { - pt[i].page =alloc_page (GFP_KERNEL ); -if (!pt->page) { -gen6_ppgtt_free (ppgtt); -return -ENOMEM ; - } - } - - ppgtt->pd.page_tables = pt; -return 0 ; -} -static int gen6_ppgtt_alloc (struct i915_hw_ppgtt *ppgtt) {int ret;@@ -1053,7 +1140,7 @@ static int gen6_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt)if (ret)return ret; - ret =gen6_ppgtt_allocate_page_tables (ppgtt); + ret =alloc_pt_range (&ppgtt->pd,0 , ppgtt->num_pd_entries);if (ret) {drm_mm_remove_node (&ppgtt->node);return ret;@@ -1071,7 +1158,7 @@ static int gen6_ppgtt_setup_page_tables(struct i915_hw_ppgtt *ppgtt)struct page *page;dma_addr_t pt_addr; - page = ppgtt->pd.page_tables[i].page; + page = ppgtt->pd.page_tables[i]->page; pt_addr =pci_map_page (dev->pdev, page,0 ,4096 ,PCI_DMA_BIDIRECTIONAL );@@ -1080,7 +1167,7 @@ static int gen6_ppgtt_setup_page_tables(struct i915_hw_ppgtt *ppgtt)return -EIO ; } - ppgtt->pd.page_tables[i].daddr = pt_addr; + ppgtt->pd.page_tables[i]->daddr = pt_addr; }return 0 ;drivers/gpu/drm/i915/i915_gem_gtt.h +2
-2@@ -199,12 +199,12 @@ struct i915_page_directory_entry {dma_addr_t daddr; }; -struct i915_page_table_entry *page_tables; +struct i915_page_table_entry *page_tables[GEN6_PPGTT_PD_ENTRIES ];/* PDEs */ };struct i915_page_directory_pointer_entry {/* struct page *page; */ -struct i915_page_directory_entry page_directory[GEN8_LEGACY_PDPES ]; +struct i915_page_directory_entry *page_directory[GEN8_LEGACY_PDPES ]; };struct i915_address_space {drivers/gpu/drm/i915/intel_lrc.c +8
-8@@ -1731,14 +1731,14 @@ populate_lr_context(struct intel_context *ctx, struct drm_i915_gem_object *ctx_o reg_state[CTX_PDP1_LDW ] =GEN8_RING_PDP_LDW (ring,1 ); reg_state[CTX_PDP0_UDW ] =GEN8_RING_PDP_UDW (ring,0 ); reg_state[CTX_PDP0_LDW ] =GEN8_RING_PDP_LDW (ring,0 ); - reg_state[CTX_PDP3_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[3 ].daddr); - reg_state[CTX_PDP3_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[3 ].daddr); - reg_state[CTX_PDP2_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[2 ].daddr); - reg_state[CTX_PDP2_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[2 ].daddr); - reg_state[CTX_PDP1_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[1 ].daddr); - reg_state[CTX_PDP1_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[1 ].daddr); - reg_state[CTX_PDP0_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[0 ].daddr); - reg_state[CTX_PDP0_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[0 ].daddr); + reg_state[CTX_PDP3_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[3 ]->daddr); + reg_state[CTX_PDP3_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[3 ]->daddr); + reg_state[CTX_PDP2_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[2 ]->daddr); + reg_state[CTX_PDP2_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[2 ]->daddr); + reg_state[CTX_PDP1_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[1 ]->daddr); + reg_state[CTX_PDP1_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[1 ]->daddr); + reg_state[CTX_PDP0_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[0 ]->daddr); + reg_state[CTX_PDP0_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[0 ]->daddr);if (ring->id ==RCS ) { reg_state[CTX_LRI_HEADER_2 ] =MI_LOAD_REGISTER_IMM (1 ); reg_state[CTX_R_PWR_CLK_STATE ] =0x20c8 ;Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
_______________________________________________
Intel-gfx mailing list
[email protected]
http://lists.freedesktop.org/mailman/listinfo/intel-gfx
Re: [PATCH v4 07/24] drm/i915: Create page table allocators
Quoted 110 lines · Mika Kuoppala
Quoted 96 lines · Michel Thierry
From: Ben Widawsky <[email protected]>
As we move toward dynamic page table allocation, it becomes much easier to manage our data structures if break do things less coarsely by breaking up all of our actions into individual tasks. This makes the code easier to write, read, and verify.
Aside from the dissection of the allocation functions, the patch statically allocates the page table structures without a page directory. This remains the same for all platforms,
The patch itself should not have much functional difference. The primary noticeable difference is the fact that page tables are no longer allocated, but rather statically declared as part of the page directory. This has non-zero overhead, but things gain non-trivial complexity as a result.
This patch exists for a few reasons:
1. Splitting out the functions allows easily combining GEN6 and GEN8 code. Page tables have no difference based on GEN8. As we'll see in a future patch when we add the DMA mappings to the allocations, it requires only one small change to make work, and error handling should just fall into place.2. Unless we always want to allocate all page tables under a given PDE, we'll have to eventually break this up into an array of pointers (or pointer to pointer).
3. Having the discrete functions is easier to review, and understand. All allocations and frees now take place in just a couple of locations. Reviewing, and catching leaks should be easy.
4. Less important: the GFP flags are confined to one location, which makes playing around with such things trivial.
v2: Updated commit message to explain why this patch exists
v3: For lrc, s/pdp.page_directory[i].daddr/pdp.page_directory[i]->daddr/
v4: Renamed free_pt/pd_single functions to unmap_and_free_pt/pd (Daniel)
v5: Added additional safety checks in gen8 clear/free/unmap.
Signed-off-by Ben Widawsky <[email protected]>Signed-off-by Michel Thierry <[email protected]> (v3, v4, v5)--- drivers/gpu/drm/i915/i915_gem_gtt.c | 251 ++++++++++++++++++++++++------------ drivers/gpu/drm/i915/i915_gem_gtt.h | 4 +- drivers/gpu/drm/i915/intel_lrc.c | 16 +-- 3 files changed, 179 insertions(+), 92 deletions(-)drivers/gpu/drm/i915/i915_gem_gtt.c +37
-0@@ -275,6 +275,99 @@ static gen6_gtt_pte_t iris_pte_encode(dma_addr_t addr,return pte; } +static void unmap_and_free_pt (struct i915_page_table_entry *pt) +{ +if (WARN_ON (!pt->page)) +return ; +__free_page (pt->page); +kfree (pt); +} + +static struct i915_page_table_entry *alloc_pt_single (void ) +{ +struct i915_page_table_entry *pt; + + pt =kzalloc (sizeof (*pt),GFP_KERNEL ); +if (!pt) +return ERR_PTR (-ENOMEM ); + + pt->page =alloc_page (GFP_KERNEL | __GFP_ZERO); +if (!pt->page) { +kfree (pt); +return ERR_PTR (-ENOMEM ); + } + +return pt; +} + +/** +* alloc_pt_range() - Allocate a multiple page tables +* @pd: The page directory which will have at least @count entries +* available to point to the allocated page tables. +* @pde: First page directory entry for which we are allocating. +* @count: Number of pages to allocate. +* +* Allocates multiple page table pages and sets the appropriate entries in the +* page table structure within the page directory. Function cleans up after +* itself on any failures. +* +* Return: 0 if allocation succeeded. drivers/gpu/drm/i915/i915_gem_gtt.c static gen6_gtt_pte_t iris_pte_encode(dma_addr_t addr, · line 324
Quoted 10 lines
drivers/gpu/drm/i915/i915_gem_gtt.c +10
-0+*/ +static int alloc_pt_range (struct i915_page_directory_entry *pd,uint16_t pde,size_t count) +{ +int i, ret; + +/* 512 is the max page tables per page_directory on any platform. +* TODO: make WARN after patch series is done +*/ +BUG_ON (pde + count >GEN6_PPGTT_PD_ENTRIES ); +
WARN_ONin here and return-EINVAL.
Quoted 1 line
-Mika
I applied the changes in v6.
Thanks for the review.
-Michel
_______________________________________________
Intel-gfx mailing list
[email protected]
http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH v4 8/24 drm/i915: Plumb drm_device through page tables operations
In reply to Michel ThierryThe next patch in the series will require it for alloc_pt_single.
--- drivers/gpu/drm/i915/i915_gem_gtt.c | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-)
drivers/gpu/drm/i915/i915_gem_gtt.c +16 -13
@@ -138,7 +138,6 @@ static int sanitize_enable_ppgtt(struct drm_device *dev, int enable_ppgtt)return has_aliasing_ppgtt ?1 :0 ; } -static void ppgtt_bind_vma (struct i915_vma *vma,enum i915_cache_level cache_level,u32 flags);
@@ -275,7 +274,7 @@ static gen6_gtt_pte_t iris_pte_encode(dma_addr_t addr,return pte; } -static void unmap_and_free_pt (struct i915_page_table_entry *pt) +static void unmap_and_free_pt (struct i915_page_table_entry *pt,struct drm_device *dev) {if (WARN_ON (!pt->page))return ;
@@ -283,7 +282,7 @@ static void unmap_and_free_pt(struct i915_page_table_entry *pt)kfree (pt); } -static struct i915_page_table_entry *alloc_pt_single (void ) +static struct i915_page_table_entry *alloc_pt_single (struct drm_device *dev) {struct i915_page_table_entry *pt;
@@ -313,7 +312,9 @@ static struct i915_page_table_entry *alloc_pt_single(void)* * Return: 0 if allocation succeeded. */ -static int alloc_pt_range (struct i915_page_directory_entry *pd,uint16_t pde,size_t count) +static int alloc_pt_range (struct i915_page_directory_entry *pd,uint16_t pde,size_t count, +struct drm_device *dev) + {int i, ret;
@@ -323,7 +324,7 @@ static int alloc_pt_range(struct i915_page_directory_entry *pd, uint16_t pde, siBUG_ON (pde + count >GEN6_PPGTT_PD_ENTRIES );for (i = pde; i < pde + count; i++) { -struct i915_page_table_entry *pt =alloc_pt_single (); +struct i915_page_table_entry *pt =alloc_pt_single (dev);if (IS_ERR (pt)) { ret =PTR_ERR (pt);
@@ -339,7 +340,7 @@ static int alloc_pt_range(struct i915_page_directory_entry *pd, uint16_t pde, si err_out:while (i--) -unmap_and_free_pt (pd->page_tables[i]); +unmap_and_free_pt (pd->page_tables[i], dev);return ret; }
@@ -518,7 +519,7 @@ static void gen8_ppgtt_insert_entries(struct i915_address_space *vm, } } -static void gen8_free_page_tables (struct i915_page_directory_entry *pd) +static void gen8_free_page_tables (struct i915_page_directory_entry *pd,struct drm_device *dev) {int i;
@@ -529,7 +530,7 @@ static void gen8_free_page_tables(struct i915_page_directory_entry *pd)if (WARN_ON (!pd->page_tables[i]))continue ; -unmap_and_free_pt (pd->page_tables[i]); +unmap_and_free_pt (pd->page_tables[i], dev); pd->page_tables[i] =NULL ; } }
@@ -542,7 +543,7 @@ static void gen8_ppgtt_free(struct i915_hw_ppgtt *ppgtt)if (WARN_ON (!ppgtt->pdp.page_directory[i]))continue ; -gen8_free_page_tables (ppgtt->pdp.page_directory[i]); +gen8_free_page_tables (ppgtt->pdp.page_directory[i], ppgtt->base.dev);unmap_and_free_pd (ppgtt->pdp.page_directory[i]); } }
@@ -594,7 +595,7 @@ static int gen8_ppgtt_allocate_page_tables(struct i915_hw_ppgtt *ppgtt)for (i =0 ; i < ppgtt->num_pd_pages; i++) { ret =alloc_pt_range (ppgtt->pdp.page_directory[i], -0 ,GEN8_PDES_PER_PAGE ); +0 ,GEN8_PDES_PER_PAGE , ppgtt->base.dev);if (ret) goto unwind_out; }
@@ -603,7 +604,7 @@ static int gen8_ppgtt_allocate_page_tables(struct i915_hw_ppgtt *ppgtt) unwind_out:while (i--) -gen8_free_page_tables (ppgtt->pdp.page_directory[i]); +gen8_free_page_tables (ppgtt->pdp.page_directory[i], ppgtt->base.dev);return -ENOMEM ; }
@@ -1075,7 +1076,7 @@ static void gen6_ppgtt_free(struct i915_hw_ppgtt *ppgtt)int i;for (i =0 ; i < ppgtt->num_pd_entries; i++) -unmap_and_free_pt (ppgtt->pd.page_tables[i]); +unmap_and_free_pt (ppgtt->pd.page_tables[i], ppgtt->base.dev);unmap_and_free_pd (&ppgtt->pd); }
@@ -1140,7 +1141,9 @@ static int gen6_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt)if (ret)return ret; - ret =alloc_pt_range (&ppgtt->pd,0 , ppgtt->num_pd_entries); + ret =alloc_pt_range (&ppgtt->pd,0 , ppgtt->num_pd_entries, + ppgtt->base.dev); +if (ret) {drm_mm_remove_node (&ppgtt->node);return ret;
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH v4 9/24 drm/i915: Track GEN6 page table usage
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
Instead of implementing the full tracking + dynamic allocation, this patch does a bit less than half of the work, by tracking and warning on unexpected conditions. The tracking itself follows which PTEs within a page table are currently being used for objects. The next patch will modify this to actually allocate the page tables only when necessary.
With the current patch there isn't much in the way of making a gen agnostic range allocation function. However, in the next patch we'll add more specificity which makes having separate functions a bit easier to manage.
One important change introduced here is that DMA mappings are created/destroyed at the same page directories/tables are allocated/deallocated.
Notice that aliasing PPGTT is not managed here. The patch which actually begins dynamic allocation/teardown explains the reasoning for this.
v2: s/pdp.page_directory/pdp.page_directorys
Make a scratch page allocation helper
v3: Rebase and expand commit message.
v4: Allocate required pagetables only when it is needed, _bind_to_vm instead of bind_vma (Daniel).
v5: Rebased to remove the unnecessary noise in the diff, also:
- PDE mask is GEN agnostic, renamed GEN6_PDE_MASK to I915_PDE_MASK.
- Removed unnecessary checks in gen6_alloc_va_range.
- Changed map/unmap_px_single macros to use dma functions directly and be part of a static inline function instead.
- Moved drm_device plumbing through page tables operation to its own patch.
- Moved allocate/teardown_va_range calls until they are fully implemented (in subsequent patch).
- Merged pt and scratch_pt unmap_and_free path.
- Moved scratch page allocator helper to the patch that will use it.
v6: Reduce complexity by not tearing down pagetables dynamically, the same can be achieved while freeing empty vms. (Daniel)
--- drivers/gpu/drm/i915/i915_gem_gtt.c | 191 +++++++++++++++++++++++++----------- drivers/gpu/drm/i915/i915_gem_gtt.h | 75 ++++++++++++++ 2 files changed, 206 insertions(+), 60 deletions(-)
drivers/gpu/drm/i915/i915_gem_gtt.c +131 -60
@@ -274,29 +274,88 @@ static gen6_gtt_pte_t iris_pte_encode(dma_addr_t addr,return pte; } -static void unmap_and_free_pt (struct i915_page_table_entry *pt,struct drm_device *dev) +#define i915_dma_unmap_single (px, dev) \ + __i915_dma_unmap_single((px)->daddr, dev) + +static inline void __i915_dma_unmap_single (dma_addr_t daddr, +struct drm_device *dev) +{ +struct device *device = &dev->pdev->dev; + +dma_unmap_page (device, daddr,4096 ,PCI_DMA_BIDIRECTIONAL ); +} + +/** +* i915_dma_map_px_single() - Create a dma mapping for a page table/dir/etc. +* @px: Page table/dir/etc to get a DMA map for +* @dev: drm device +* +* Page table allocations are unified across all gens. They always require a +* single 4k allocation, as well as a DMA mapping. If we keep the structs +* symmetric here, the simple macro covers us for every page table type. +* +* Return: 0 if success. +*/ +#define i915_dma_map_px_single (px, dev) \ + i915_dma_map_page_single((px)->page, (dev), &(px)->daddr) + +static inline int i915_dma_map_page_single (struct page *page, +struct drm_device *dev, +dma_addr_t *daddr) +{ +struct device *device = &dev->pdev->dev; + + *daddr =dma_map_page (device, page,0 ,4096 ,PCI_DMA_BIDIRECTIONAL ); +return dma_mapping_error (device, *daddr); +} + +static void unmap_and_free_pt (struct i915_page_table_entry *pt, +struct drm_device *dev) {if (WARN_ON (!pt->page))return ; + +i915_dma_unmap_single (pt, dev);__free_page (pt->page); +kfree (pt->used_ptes);kfree (pt); }static struct i915_page_table_entry *alloc_pt_single (struct drm_device *dev) {struct i915_page_table_entry *pt; +const size_t count =INTEL_INFO (dev)->gen >=8 ? +GEN8_PTES_PER_PAGE :I915_PPGTT_PT_ENTRIES ; +int ret = -ENOMEM ; pt =kzalloc (sizeof (*pt),GFP_KERNEL );if (!pt)return ERR_PTR (-ENOMEM ); + pt->used_ptes =kcalloc (BITS_TO_LONGS (count),sizeof (*pt->used_ptes), +GFP_KERNEL ); + +if (!pt->used_ptes) + goto fail_bitmap; + pt->page =alloc_page (GFP_KERNEL | __GFP_ZERO); -if (!pt->page) { -kfree (pt); -return ERR_PTR (-ENOMEM ); - } +if (!pt->page) + goto fail_page; + + ret =i915_dma_map_px_single (pt, dev); +if (ret) + goto fail_dma;return pt; + +fail_dma: +__free_page (pt->page); +fail_page: +kfree (pt->used_ptes); +fail_bitmap: +kfree (pt); + +return ERR_PTR (ret); }/**
@@ -836,26 +895,36 @@ static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m) } } -static void gen6_write_pdes (struct i915_hw_ppgtt *ppgtt) +/* Write pde (index) from the page directory @pd to the page table @pt */ +static void gen6_write_pdes (struct i915_page_directory_entry *pd, +const int pde,struct i915_page_table_entry *pt) { -struct drm_i915_private *dev_priv = ppgtt->base.dev->dev_private; -gen6_gtt_pte_t __iomem *pd_addr; -uint32_t pd_entry; -int i; +struct i915_hw_ppgtt *ppgtt = + container_of(pd, struct i915_hw_ppgtt, pd); +u32 pd_entry; -WARN_ON (ppgtt->pd.pd_offset &0x3f ); - pd_addr = (gen6_gtt_pte_t __iomem *)dev_priv->gtt.gsm + - ppgtt->pd.pd_offset /sizeof (gen6_gtt_pte_t); -for (i =0 ; i < ppgtt->num_pd_entries; i++) { -dma_addr_t pt_addr; + pd_entry =GEN6_PDE_ADDR_ENCODE (pt->daddr); + pd_entry |=GEN6_PDE_VALID ; - pt_addr = ppgtt->pd.page_tables[i]->daddr; - pd_entry =GEN6_PDE_ADDR_ENCODE (pt_addr); - pd_entry |=GEN6_PDE_VALID ; +writel (pd_entry, ppgtt->pd_addr + pde); -writel (pd_entry, pd_addr + i); - } -readl (pd_addr); +/* XXX: Caller needs to make sure the write completes if necessary */ +} + +/* Write all the page tables found in the ppgtt structure to incrementing page +* directories. */ +static void gen6_write_page_range (struct drm_i915_private *dev_priv, +struct i915_page_directory_entry *pd,uint32_t start,uint32_t length) +{ +struct i915_page_table_entry *pt; +uint32_t pde, temp; + +gen6_for_each_pde (pt, pd, start, length, temp, pde) +gen6_write_pdes (pd, pde, pt); + +/* Make sure write is complete before other code can use this page +* table. Also require for WC mapped PTEs */ +readl (dev_priv->gtt.gsm); }static uint32_t get_pd_offset (struct i915_hw_ppgtt *ppgtt)
@@ -1071,6 +1140,28 @@ static void gen6_ppgtt_unmap_pages(struct i915_hw_ppgtt *ppgtt) 4096, PCI_DMA_BIDIRECTIONAL); } +static int gen6_alloc_va_range (struct i915_address_space *vm, +uint64_t start,uint64_t length) +{ +struct i915_hw_ppgtt *ppgtt = + container_of(vm, struct i915_hw_ppgtt, base); +struct i915_page_table_entry *pt; +uint32_t pde, temp; + +gen6_for_each_pde (pt, &ppgtt->pd, start, length, temp, pde) { +DECLARE_BITMAP (tmp_bitmap,I915_PPGTT_PT_ENTRIES ); + +bitmap_zero (tmp_bitmap,I915_PPGTT_PT_ENTRIES ); +bitmap_set (tmp_bitmap,gen6_pte_index (start), +gen6_pte_count (start, length)); + +bitmap_or (pt->used_ptes, pt->used_ptes, tmp_bitmap, +I915_PPGTT_PT_ENTRIES ); + } + +return 0 ; +} +static void gen6_ppgtt_free (struct i915_hw_ppgtt *ppgtt) {int i;
@@ -1117,20 +1208,24 @@ alloc:0 , dev_priv->gtt.base.total, 0);if (ret) -return ret; + goto err_out; retried = true; goto alloc; }if (ret) -return ret; + goto err_out; +if (ppgtt->node.start < dev_priv->gtt.mappable_end)DRM_DEBUG ("Forced to use aperture for PDEs\n" ); ppgtt->num_pd_entries =GEN6_PPGTT_PD_ENTRIES ;return 0 ; + +err_out: +return ret; }static int gen6_ppgtt_alloc (struct i915_hw_ppgtt *ppgtt)
@@ -1152,30 +1247,6 @@ static int gen6_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt)return 0 ; } -static int gen6_ppgtt_setup_page_tables (struct i915_hw_ppgtt *ppgtt) -{ -struct drm_device *dev = ppgtt->base.dev; -int i; - -for (i =0 ; i < ppgtt->num_pd_entries; i++) { -struct page *page; -dma_addr_t pt_addr; - - page = ppgtt->pd.page_tables[i]->page; - pt_addr =pci_map_page (dev->pdev, page,0 ,4096 , -PCI_DMA_BIDIRECTIONAL ); - -if (pci_dma_mapping_error (dev->pdev, pt_addr)) { -gen6_ppgtt_unmap_pages (ppgtt); -return -EIO ; - } - - ppgtt->pd.page_tables[i]->daddr = pt_addr; - } - -return 0 ; -} -static int gen6_ppgtt_init (struct i915_hw_ppgtt *ppgtt) {struct drm_device *dev = ppgtt->base.dev;
@@ -1196,12 +1267,7 @@ static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt)if (ret)return ret; - ret =gen6_ppgtt_setup_page_tables (ppgtt); -if (ret) { -gen6_ppgtt_free (ppgtt); -return ret; - } - + ppgtt->base.allocate_va_range = gen6_alloc_va_range; ppgtt->base.clear_range = gen6_ppgtt_clear_range; ppgtt->base.insert_entries = gen6_ppgtt_insert_entries; ppgtt->base.cleanup = gen6_ppgtt_cleanup;
@@ -1212,13 +1278,17 @@ static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt) ppgtt->pd.pd_offset = ppgtt->node.start /PAGE_SIZE *sizeof (gen6_gtt_pte_t); + ppgtt->pd_addr = (gen6_gtt_pte_t __iomem *)dev_priv->gtt.gsm + + ppgtt->pd.pd_offset /sizeof (gen6_gtt_pte_t); + ppgtt->base.clear_range (&ppgtt->base,0 , ppgtt->base.total, true); +gen6_write_page_range (dev_priv, &ppgtt->pd,0 , ppgtt->base.total); +DRM_DEBUG_DRIVER ("Allocated pde space (%ldM) at GTT entry: %lx\n" , ppgtt->node.size >>20 , ppgtt->node.start /PAGE_SIZE ); -gen6_write_pdes (ppgtt);DRM_DEBUG ("Adding PPGTT at offset %x\n" , ppgtt->pd.pd_offset <<10 );
@@ -1491,13 +1561,14 @@ void i915_gem_restore_gtt_mappings(struct drm_device *dev)list_for_each_entry (vm, &dev_priv->vm_list, global_link) {/* TODO: Perhaps it shouldn't be gen6 specific */ -if (i915_is_ggtt (vm)) { -if (dev_priv->mm.aliasing_ppgtt) -gen6_write_pdes (dev_priv->mm.aliasing_ppgtt); -continue ; - } -gen6_write_pdes (container_of(vm ,struct i915_hw_ppgtt ,base )); +struct i915_hw_ppgtt *ppgtt = + container_of(vm, struct i915_hw_ppgtt, base); + +if (i915_is_ggtt (vm)) + ppgtt = dev_priv->mm.aliasing_ppgtt; + +gen6_write_page_range (dev_priv, &ppgtt->pd,0 , ppgtt->num_pd_entries); }i915_ggtt_flush (dev_priv);
drivers/gpu/drm/i915/i915_gem_gtt.h +75 -0
106 lines not on this page · raw message
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
Re: [PATCH v4 09/24] drm/i915: Track GEN6 page table usage
Quoted 75 lines · Michel Thierry
From: Ben Widawsky <[email protected]>
Instead of implementing the full tracking + dynamic allocation, this patch does a bit less than half of the work, by tracking and warning on unexpected conditions. The tracking itself follows which PTEs within a page table are currently being used for objects. The next patch will modify this to actually allocate the page tables only when necessary.
With the current patch there isn't much in the way of making a gen agnostic range allocation function. However, in the next patch we'll add more specificity which makes having separate functions a bit easier to manage.
One important change introduced here is that DMA mappings are created/destroyed at the same page directories/tables are allocated/deallocated.
Notice that aliasing PPGTT is not managed here. The patch which actually begins dynamic allocation/teardown explains the reasoning for this.
v2: s/pdp.page_directory/pdp.page_directorys
Make a scratch page allocation helperv3: Rebase and expand commit message.
v4: Allocate required pagetables only when it is needed,
_bind_to_vminstead ofbind_vma(Daniel).v5: Rebased to remove the unnecessary noise in the diff, also:
- PDE mask is GEN agnostic, renamedGEN6_PDE_MASKtoI915_PDE_MASK.
- Removed unnecessary checks ingen6_alloc_va_range.
- Changed map/unmap_px_single macros to use dma functions directly and be part of a static inline function instead.
- Moveddrm_deviceplumbing through page tables operation to its own patch.
- Moved allocate/teardown_va_range calls until they are fully implemented (in subsequent patch).
- Merged pt andscratch_ptunmap_and_freepath.
- Moved scratch page allocator helper to the patch that will use it.v6: Reduce complexity by not tearing down pagetables dynamically, the same can be achieved while freeing empty vms. (Daniel)
Cc Daniel Vetter <[email protected]>Signed-off-by Ben Widawsky <[email protected]>Signed-off-by Michel Thierry <[email protected]> (v3+)--- drivers/gpu/drm/i915/i915_gem_gtt.c | 191 +++++++++++++++++++++++++----------- drivers/gpu/drm/i915/i915_gem_gtt.h | 75 ++++++++++++++ 2 files changed, 206 insertions(+), 60 deletions(-)drivers/gpu/drm/i915/i915_gem_gtt.c +15
-1@@ -274,29 +274,88 @@ static gen6_gtt_pte_t iris_pte_encode(dma_addr_t addr,return pte; } -static void unmap_and_free_pt (struct i915_page_table_entry *pt,struct drm_device *dev) +#define i915_dma_unmap_single (px, dev) \ + __i915_dma_unmap_single((px)->daddr, dev) + +static inline void __i915_dma_unmap_single (dma_addr_t daddr, +struct drm_device *dev) +{ +struct device *device = &dev->pdev->dev; + +dma_unmap_page (device, daddr,4096 ,PCI_DMA_BIDIRECTIONAL ); +} + +/** +* i915_dma_map_px_single() - Create a dma mapping for a page table/dir/etc. +* @px: Page table/dir/etc to get a DMA map for +* @dev: drm device
drivers/gpu/drm/i915/i915_gem_gtt.c static gen6_gtt_pte_t iris_pte_encode(dma_addr_t addr, · line 301
Quoted 10 lines
drivers/gpu/drm/i915/i915_gem_gtt.c +10
-0+* +* Page table allocations are unified across all gens. They always require a +* single 4k allocation, as well as a DMA mapping. If we keep the structs +* symmetric here, the simple macro covers us for every page table type. +* +* Return: 0 if success. +*/ +#define i915_dma_map_px_single (px, dev) \ + i915_dma_map_page_single((px)->page, (dev), &(px)->daddr) +
If this is symmetrical to i915_dma_unmap_single() is the _px_ needed?
Quoted 65 lines
drivers/gpu/drm/i915/i915_gem_gtt.c +39
-4+static inline int i915_dma_map_page_single (struct page *page, +struct drm_device *dev, +dma_addr_t *daddr) +{ +struct device *device = &dev->pdev->dev; + + *daddr =dma_map_page (device, page,0 ,4096 ,PCI_DMA_BIDIRECTIONAL ); +return dma_mapping_error (device, *daddr); +} + +static void unmap_and_free_pt (struct i915_page_table_entry *pt, +struct drm_device *dev) {if (WARN_ON (!pt->page))return ; + +i915_dma_unmap_single (pt, dev);__free_page (pt->page); +kfree (pt->used_ptes);kfree (pt); }static struct i915_page_table_entry *alloc_pt_single (struct drm_device *dev) {struct i915_page_table_entry *pt; +const size_t count =INTEL_INFO (dev)->gen >=8 ? +GEN8_PTES_PER_PAGE :I915_PPGTT_PT_ENTRIES ; +int ret = -ENOMEM ; pt =kzalloc (sizeof (*pt),GFP_KERNEL );if (!pt)return ERR_PTR (-ENOMEM ); + pt->used_ptes =kcalloc (BITS_TO_LONGS (count),sizeof (*pt->used_ptes), +GFP_KERNEL ); + +if (!pt->used_ptes) + goto fail_bitmap; + pt->page =alloc_page (GFP_KERNEL | __GFP_ZERO); -if (!pt->page) { -kfree (pt); -return ERR_PTR (-ENOMEM ); - } +if (!pt->page) + goto fail_page; + + ret =i915_dma_map_px_single (pt, dev); +if (ret) + goto fail_dma;return pt; + +fail_dma: +__free_page (pt->page); +fail_page: +kfree (pt->used_ptes); +fail_bitmap: +kfree (pt); + +return ERR_PTR (ret); }/**
drivers/gpu/drm/i915/i915_gem_gtt.c static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m) · line 899
Quoted 6 lines
drivers/gpu/drm/i915/i915_gem_gtt.c +2
-1@@ -836,26 +895,36 @@ static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m) } } -static void gen6_write_pdes (struct i915_hw_ppgtt *ppgtt) +/* Write pde (index) from the page directory @pd to the page table @pt */ +static void gen6_write_pdes(struct i915_page_directory_entry *pd,
For me it seems that you will write only one pde entry so s/pdes/pde ?
Quoted 18 lines
drivers/gpu/drm/i915/i915_gem_gtt.c +6
-9+ const int pde, struct i915_page_table_entry *pt) { -struct drm_i915_private *dev_priv = ppgtt->base.dev->dev_private; -gen6_gtt_pte_t __iomem *pd_addr; -uint32_t pd_entry; -int i; + struct i915_hw_ppgtt *ppgtt = + container_of(pd, struct i915_hw_ppgtt, pd); +u32 pd_entry; -WARN_ON (ppgtt->pd.pd_offset &0x3f ); - pd_addr = (gen6_gtt_pte_t __iomem *)dev_priv->gtt.gsm + - ppgtt->pd.pd_offset /sizeof (gen6_gtt_pte_t); -for (i =0 ; i < ppgtt->num_pd_entries; i++) { -dma_addr_t pt_addr; + pd_entry =GEN6_PDE_ADDR_ENCODE (pt->daddr); + pd_entry |=GEN6_PDE_VALID ;
drivers/gpu/drm/i915/i915_gem_gtt.c static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m) · line 912
Quoted 10 lines
drivers/gpu/drm/i915/i915_gem_gtt.c +3
-6- pt_addr = ppgtt->pd.page_tables[i]->daddr; - pd_entry =GEN6_PDE_ADDR_ENCODE (pt_addr); - pd_entry |=GEN6_PDE_VALID ; +writel (pd_entry, ppgtt->pd_addr + pde); -writel (pd_entry, pd_addr + i); - } -readl (pd_addr); +/* XXX: Caller needs to make sure the write completes if necessary */ +}
Move this comment on top of the function and lift the XXX:
Quoted 144 lines
drivers/gpu/drm/i915/i915_gem_gtt.c +49
-35+ +/* Write all the page tables found in the ppgtt structure to incrementing page +* directories. */ +static void gen6_write_page_range (struct drm_i915_private *dev_priv, +struct i915_page_directory_entry *pd,uint32_t start,uint32_t length) +{ +struct i915_page_table_entry *pt; +uint32_t pde, temp; + +gen6_for_each_pde (pt, pd, start, length, temp, pde) +gen6_write_pdes (pd, pde, pt); + +/* Make sure write is complete before other code can use this page +* table. Also require for WC mapped PTEs */ +readl (dev_priv->gtt.gsm); }static uint32_t get_pd_offset (struct i915_hw_ppgtt *ppgtt)@@ -1071,6 +1140,28 @@ static void gen6_ppgtt_unmap_pages(struct i915_hw_ppgtt *ppgtt) 4096, PCI_DMA_BIDIRECTIONAL); } +static int gen6_alloc_va_range (struct i915_address_space *vm, +uint64_t start,uint64_t length) +{ +struct i915_hw_ppgtt *ppgtt = + container_of(vm, struct i915_hw_ppgtt, base); +struct i915_page_table_entry *pt; +uint32_t pde, temp; + +gen6_for_each_pde (pt, &ppgtt->pd, start, length, temp, pde) { +DECLARE_BITMAP (tmp_bitmap,I915_PPGTT_PT_ENTRIES ); + +bitmap_zero (tmp_bitmap,I915_PPGTT_PT_ENTRIES ); +bitmap_set (tmp_bitmap,gen6_pte_index (start), +gen6_pte_count (start, length)); + +bitmap_or (pt->used_ptes, pt->used_ptes, tmp_bitmap, +I915_PPGTT_PT_ENTRIES ); + } + +return 0 ; +} +static void gen6_ppgtt_free (struct i915_hw_ppgtt *ppgtt) {int i;@@ -1117,20 +1208,24 @@ alloc:0 , dev_priv->gtt.base.total, 0);if (ret) -return ret; + goto err_out; retried = true; goto alloc; }if (ret) -return ret; + goto err_out; +if (ppgtt->node.start < dev_priv->gtt.mappable_end)DRM_DEBUG ("Forced to use aperture for PDEs\n" ); ppgtt->num_pd_entries =GEN6_PPGTT_PD_ENTRIES ;return 0 ; + +err_out: +return ret; }static int gen6_ppgtt_alloc (struct i915_hw_ppgtt *ppgtt)@@ -1152,30 +1247,6 @@ static int gen6_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt)return 0 ; } -static int gen6_ppgtt_setup_page_tables (struct i915_hw_ppgtt *ppgtt) -{ -struct drm_device *dev = ppgtt->base.dev; -int i; - -for (i =0 ; i < ppgtt->num_pd_entries; i++) { -struct page *page; -dma_addr_t pt_addr; - - page = ppgtt->pd.page_tables[i]->page; - pt_addr =pci_map_page (dev->pdev, page,0 ,4096 , -PCI_DMA_BIDIRECTIONAL ); - -if (pci_dma_mapping_error (dev->pdev, pt_addr)) { -gen6_ppgtt_unmap_pages (ppgtt); -return -EIO ; - } - - ppgtt->pd.page_tables[i]->daddr = pt_addr; - } - -return 0 ; -} -static int gen6_ppgtt_init (struct i915_hw_ppgtt *ppgtt) {struct drm_device *dev = ppgtt->base.dev;@@ -1196,12 +1267,7 @@ static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt)if (ret)return ret; - ret =gen6_ppgtt_setup_page_tables (ppgtt); -if (ret) { -gen6_ppgtt_free (ppgtt); -return ret; - } - + ppgtt->base.allocate_va_range = gen6_alloc_va_range; ppgtt->base.clear_range = gen6_ppgtt_clear_range; ppgtt->base.insert_entries = gen6_ppgtt_insert_entries; ppgtt->base.cleanup = gen6_ppgtt_cleanup;@@ -1212,13 +1278,17 @@ static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt) ppgtt->pd.pd_offset = ppgtt->node.start /PAGE_SIZE *sizeof (gen6_gtt_pte_t); + ppgtt->pd_addr = (gen6_gtt_pte_t __iomem *)dev_priv->gtt.gsm + + ppgtt->pd.pd_offset /sizeof (gen6_gtt_pte_t); + ppgtt->base.clear_range (&ppgtt->base,0 , ppgtt->base.total, true); +gen6_write_page_range (dev_priv, &ppgtt->pd,0 , ppgtt->base.total); +DRM_DEBUG_DRIVER ("Allocated pde space (%ldM) at GTT entry: %lx\n" , ppgtt->node.size >>20 , ppgtt->node.start /PAGE_SIZE ); -gen6_write_pdes (ppgtt);DRM_DEBUG ("Adding PPGTT at offset %x\n" , ppgtt->pd.pd_offset <<10 );@@ -1491,13 +1561,14 @@ void i915_gem_restore_gtt_mappings(struct drm_device *dev)list_for_each_entry (vm, &dev_priv->vm_list, global_link) {/* TODO: Perhaps it shouldn't be gen6 specific */ -if (i915_is_ggtt (vm)) { -if (dev_priv->mm.aliasing_ppgtt)
drivers/gpu/drm/i915/i915_gem_gtt.c void i915_gem_restore_gtt_mappings(struct drm_device *dev) · line 1569
Quoted 10 lines
drivers/gpu/drm/i915/i915_gem_gtt.c +5
-4-gen6_write_pdes (dev_priv->mm.aliasing_ppgtt); -continue ; - } -gen6_write_pdes (container_of(vm ,struct i915_hw_ppgtt ,base )); +struct i915_hw_ppgtt *ppgtt = + container_of(vm, struct i915_hw_ppgtt, base); + +if (i915_is_ggtt (vm)) + ppgtt = dev_priv->mm.aliasing_ppgtt;
If we have ggtt but aliasing is not enabled we get NULL here and oops over in next function?
-Mika
Quoted 126 lines
drivers/gpu/drm/i915/i915_gem_gtt.c +2
-0+ +gen6_write_page_range (dev_priv, &ppgtt->pd,0 , ppgtt->num_pd_entries); }i915_ggtt_flush (dev_priv);drivers/gpu/drm/i915/i915_gem_gtt.h +75
-0@@ -54,7 +54,10 @@ typedef gen8_gtt_pte_t gen8_ppgtt_pde_t;#define GEN6_PPGTT_PD_ENTRIES 512#define GEN6_PD_SIZE (GEN6_PPGTT_PD_ENTRIES * PAGE_SIZE)#define GEN6_PD_ALIGN (PAGE_SIZE * 16) +#define GEN6_PDE_SHIFT 22#define GEN6_PDE_VALID (1 << 0) +#define I915_PDE_MASK (GEN6_PPGTT_PD_ENTRIES-1) +#define NUM_PTE (pde_shift) (1 << (pde_shift - PAGE_SHIFT))#define GEN7_PTE_CACHE_L3_LLC (3 << 1)@@ -190,6 +193,8 @@ struct i915_vma {struct i915_page_table_entry {struct page *page;dma_addr_t daddr; + +unsigned long *used_ptes; };struct i915_page_directory_entry {@@ -246,6 +251,9 @@ struct i915_address_space {gen6_gtt_pte_t (*pte_encode)(dma_addr_t addr,enum i915_cache_level level,bool valid,u32 flags);/* Create a valid PTE */ +int (*allocate_va_range)(struct i915_address_space *vm, +uint64_t start, +uint64_t length); void (*clear_range)(struct i915_address_space *vm, uint64_t start, uint64_t length,@@ -298,12 +306,79 @@ struct i915_hw_ppgtt {struct drm_i915_file_private *file_priv; +gen6_gtt_pte_t __iomem *pd_addr; +int (*enable)(struct i915_hw_ppgtt *ppgtt);int (*switch_mm)(struct i915_hw_ppgtt *ppgtt,struct intel_engine_cs *ring);void (*debug_dump)(struct i915_hw_ppgtt *ppgtt,struct seq_file *m); }; +/* For each pde iterates over every pde between from start until start + length. +* If start, and start+length are not perfectly divisible, the macro will round +* down, and up as needed. The macro modifies pde, start, and length. Dev is +* only used to differentiate shift values. Temp is temp. On gen6/7, start = 0, +* and length = 2G effectively iterates over every PDE in the system. On gen8+ +* it simply iterates over every page directory entry in a page directory. +* +* XXX: temp is not actually needed, but it saves doing the ALIGN operation. +*/ +#define gen6_for_each_pde (pt, pd, start, length, temp, iter) \ + for (iter = gen6_pde_index(start), pt = (pd)->page_tables[iter]; \ + length > 0 && iter < GEN6_PPGTT_PD_ENTRIES; \ + pt = (pd)->page_tables[++iter], \ + temp = ALIGN(start+1, 1 << GEN6_PDE_SHIFT) - start, \ + temp = min_t(unsigned, temp, length), \ + start += temp, length -= temp) + +static inline uint32_t i915_pte_index (uint64_t address,uint32_t pde_shift) +{ +const uint32_t mask =NUM_PTE (pde_shift) -1 ; + +return (address >>PAGE_SHIFT ) & mask; +} + +/* Helper to counts the number of PTEs within the given length. This count does +* not cross a page table boundary, so the max value would be +* I915_PPGTT_PT_ENTRIES for GEN6, and GEN8_PTES_PER_PAGE for GEN8. +*/ +static inline size_t i915_pte_count (uint64_t addr,size_t length, +uint32_t pde_shift) +{ +const uint64_t mask = ~((1 << pde_shift) -1 ); +uint64_t end; + +BUG_ON (length ==0 ); +BUG_ON (offset_in_page (addr|length)); + + end = addr + length; + +if ((addr & mask) != (end & mask)) +return NUM_PTE (pde_shift) -i915_pte_index (addr, pde_shift); + +return i915_pte_index (end, pde_shift) -i915_pte_index (addr, pde_shift); +} + +static inline uint32_t i915_pde_index (uint64_t addr,uint32_t shift) +{ +return (addr >> shift) &I915_PDE_MASK ; +} + +static inline uint32_t gen6_pte_index (uint32_t addr) +{ +return i915_pte_index (addr,GEN6_PDE_SHIFT ); +} + +static inline size_t gen6_pte_count (uint32_t addr,uint32_t length) +{ +return i915_pte_count (addr, length,GEN6_PDE_SHIFT ); +} + +static inline uint32_t gen6_pde_index (uint32_t addr) +{ +return i915_pde_index (addr,GEN6_PDE_SHIFT ); +} +int i915_gem_gtt_init (struct drm_device *dev);void i915_gem_init_global_gtt (struct drm_device *dev);void i915_global_gtt_cleanup (struct drm_device *dev);Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
_______________________________________________
Intel-gfx mailing list
[email protected]
http://lists.freedesktop.org/mailman/listinfo/intel-gfx
Re: [PATCH v4 09/24] drm/i915: Track GEN6 page table usage
Quoted 349 lines · Mika Kuoppala
Quoted 75 lines · Michel Thierry
From: Ben Widawsky <[email protected]>
Instead of implementing the full tracking + dynamic allocation, this patch does a bit less than half of the work, by tracking and warning on unexpected conditions. The tracking itself follows which PTEs within a page table are currently being used for objects. The next patch will modify this to actually allocate the page tables only when necessary.
With the current patch there isn't much in the way of making a gen agnostic range allocation function. However, in the next patch we'll add more specificity which makes having separate functions a bit easier to manage.
One important change introduced here is that DMA mappings are created/destroyed at the same page directories/tables are allocated/deallocated.
Notice that aliasing PPGTT is not managed here. The patch which actually begins dynamic allocation/teardown explains the reasoning for this.
v2: s/pdp.page_directory/pdp.page_directorys
Make a scratch page allocation helperv3: Rebase and expand commit message.
v4: Allocate required pagetables only when it is needed,
_bind_to_vminstead ofbind_vma(Daniel).v5: Rebased to remove the unnecessary noise in the diff, also:
- PDE mask is GEN agnostic, renamedGEN6_PDE_MASKtoI915_PDE_MASK.
- Removed unnecessary checks ingen6_alloc_va_range.
- Changed map/unmap_px_single macros to use dma functions directly and be part of a static inline function instead.
- Moveddrm_deviceplumbing through page tables operation to its own patch.
- Moved allocate/teardown_va_range calls until they are fully implemented (in subsequent patch).
- Merged pt andscratch_ptunmap_and_freepath.
- Moved scratch page allocator helper to the patch that will use it.v6: Reduce complexity by not tearing down pagetables dynamically, the same can be achieved while freeing empty vms. (Daniel)
Cc Daniel Vetter <[email protected]>Signed-off-by Ben Widawsky <[email protected]>Signed-off-by Michel Thierry <[email protected]> (v3+)--- drivers/gpu/drm/i915/i915_gem_gtt.c | 191 +++++++++++++++++++++++++----------- drivers/gpu/drm/i915/i915_gem_gtt.h | 75 ++++++++++++++ 2 files changed, 206 insertions(+), 60 deletions(-)drivers/gpu/drm/i915/i915_gem_gtt.c +15
-1@@ -274,29 +274,88 @@ static gen6_gtt_pte_t iris_pte_encode(dma_addr_t addr,return pte; } -static void unmap_and_free_pt (struct i915_page_table_entry *pt,struct drm_device *dev) +#define i915_dma_unmap_single (px, dev) \ + __i915_dma_unmap_single((px)->daddr, dev) + +static inline void __i915_dma_unmap_single (dma_addr_t daddr, +struct drm_device *dev) +{ +struct device *device = &dev->pdev->dev; + +dma_unmap_page (device, daddr,4096 ,PCI_DMA_BIDIRECTIONAL ); +} + +/** +* i915_dma_map_px_single() - Create a dma mapping for a page table/dir/etc. +* @px: Page table/dir/etc to get a DMA map for +* @dev: drm device drivers/gpu/drm/i915/i915_gem_gtt.c static gen6_gtt_pte_t iris_pte_encode(dma_addr_t addr, · line 301
Quoted 10 lines
drivers/gpu/drm/i915/i915_gem_gtt.c +10
-0+* +* Page table allocations are unified across all gens. They always require a +* single 4k allocation, as well as a DMA mapping. If we keep the structs +* symmetric here, the simple macro covers us for every page table type. +* +* Return: 0 if success. +*/ +#define i915_dma_map_px_single (px, dev) \ + i915_dma_map_page_single((px)->page, (dev), &(px)->daddr) +If this is symmetrical to
i915_dma_unmap_single()is the _px_ needed?Quoted 65 lines
drivers/gpu/drm/i915/i915_gem_gtt.c +39
-4+static inline int i915_dma_map_page_single (struct page *page, +struct drm_device *dev, +dma_addr_t *daddr) +{ +struct device *device = &dev->pdev->dev; + + *daddr =dma_map_page (device, page,0 ,4096 ,PCI_DMA_BIDIRECTIONAL ); +return dma_mapping_error (device, *daddr); +} + +static void unmap_and_free_pt (struct i915_page_table_entry *pt, +struct drm_device *dev) {if (WARN_ON (!pt->page))return ; + +i915_dma_unmap_single (pt, dev);__free_page (pt->page); +kfree (pt->used_ptes);kfree (pt); }static struct i915_page_table_entry *alloc_pt_single (struct drm_device *dev) {struct i915_page_table_entry *pt; +const size_t count =INTEL_INFO (dev)->gen >=8 ? +GEN8_PTES_PER_PAGE :I915_PPGTT_PT_ENTRIES ; +int ret = -ENOMEM ; pt =kzalloc (sizeof (*pt),GFP_KERNEL );if (!pt)return ERR_PTR (-ENOMEM ); + pt->used_ptes =kcalloc (BITS_TO_LONGS (count),sizeof (*pt->used_ptes), +GFP_KERNEL ); + +if (!pt->used_ptes) + goto fail_bitmap; + pt->page =alloc_page (GFP_KERNEL | __GFP_ZERO); -if (!pt->page) { -kfree (pt); -return ERR_PTR (-ENOMEM ); - } +if (!pt->page) + goto fail_page; + + ret =i915_dma_map_px_single (pt, dev); +if (ret) + goto fail_dma;return pt; + +fail_dma: +__free_page (pt->page); +fail_page: +kfree (pt->used_ptes); +fail_bitmap: +kfree (pt); + +return ERR_PTR (ret); }/** drivers/gpu/drm/i915/i915_gem_gtt.c static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m) · line 899
Quoted 6 lines
drivers/gpu/drm/i915/i915_gem_gtt.c +2
-1@@ -836,26 +895,36 @@ static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m) } } -static void gen6_write_pdes (struct i915_hw_ppgtt *ppgtt) +/* Write pde (index) from the page directory @pd to the page table @pt */ +static void gen6_write_pdes(struct i915_page_directory_entry *pd,For me it seems that you will write only one pde entry so s/pdes/pde ?
Quoted 18 lines
drivers/gpu/drm/i915/i915_gem_gtt.c +6
-9+ const int pde, struct i915_page_table_entry *pt) { -struct drm_i915_private *dev_priv = ppgtt->base.dev->dev_private; -gen6_gtt_pte_t __iomem *pd_addr; -uint32_t pd_entry; -int i; + struct i915_hw_ppgtt *ppgtt = + container_of(pd, struct i915_hw_ppgtt, pd); +u32 pd_entry; -WARN_ON (ppgtt->pd.pd_offset &0x3f ); - pd_addr = (gen6_gtt_pte_t __iomem *)dev_priv->gtt.gsm + - ppgtt->pd.pd_offset /sizeof (gen6_gtt_pte_t); -for (i =0 ; i < ppgtt->num_pd_entries; i++) { -dma_addr_t pt_addr; + pd_entry =GEN6_PDE_ADDR_ENCODE (pt->daddr); + pd_entry |=GEN6_PDE_VALID ;drivers/gpu/drm/i915/i915_gem_gtt.c static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m) · line 912
Quoted 10 lines
drivers/gpu/drm/i915/i915_gem_gtt.c +3
-6- pt_addr = ppgtt->pd.page_tables[i]->daddr; - pd_entry =GEN6_PDE_ADDR_ENCODE (pt_addr); - pd_entry |=GEN6_PDE_VALID ; +writel (pd_entry, ppgtt->pd_addr + pde); -writel (pd_entry, pd_addr + i); - } -readl (pd_addr); +/* XXX: Caller needs to make sure the write completes if necessary */ +}Move this comment on top of the function and lift the XXX:
Quoted 144 lines
drivers/gpu/drm/i915/i915_gem_gtt.c +49
-35+ +/* Write all the page tables found in the ppgtt structure to incrementing page +* directories. */ +static void gen6_write_page_range (struct drm_i915_private *dev_priv, +struct i915_page_directory_entry *pd,uint32_t start,uint32_t length) +{ +struct i915_page_table_entry *pt; +uint32_t pde, temp; + +gen6_for_each_pde (pt, pd, start, length, temp, pde) +gen6_write_pdes (pd, pde, pt); + +/* Make sure write is complete before other code can use this page +* table. Also require for WC mapped PTEs */ +readl (dev_priv->gtt.gsm); }static uint32_t get_pd_offset (struct i915_hw_ppgtt *ppgtt)@@ -1071,6 +1140,28 @@ static void gen6_ppgtt_unmap_pages(struct i915_hw_ppgtt *ppgtt) 4096, PCI_DMA_BIDIRECTIONAL); } +static int gen6_alloc_va_range (struct i915_address_space *vm, +uint64_t start,uint64_t length) +{ +struct i915_hw_ppgtt *ppgtt = + container_of(vm, struct i915_hw_ppgtt, base); +struct i915_page_table_entry *pt; +uint32_t pde, temp; + +gen6_for_each_pde (pt, &ppgtt->pd, start, length, temp, pde) { +DECLARE_BITMAP (tmp_bitmap,I915_PPGTT_PT_ENTRIES ); + +bitmap_zero (tmp_bitmap,I915_PPGTT_PT_ENTRIES ); +bitmap_set (tmp_bitmap,gen6_pte_index (start), +gen6_pte_count (start, length)); + +bitmap_or (pt->used_ptes, pt->used_ptes, tmp_bitmap, +I915_PPGTT_PT_ENTRIES ); + } + +return 0 ; +} +static void gen6_ppgtt_free (struct i915_hw_ppgtt *ppgtt) {int i;@@ -1117,20 +1208,24 @@ alloc:0 , dev_priv->gtt.base.total, 0);if (ret) -return ret; + goto err_out; retried = true; goto alloc; }if (ret) -return ret; + goto err_out; +if (ppgtt->node.start < dev_priv->gtt.mappable_end)DRM_DEBUG ("Forced to use aperture for PDEs\n" ); ppgtt->num_pd_entries =GEN6_PPGTT_PD_ENTRIES ;return 0 ; + +err_out: +return ret; }static int gen6_ppgtt_alloc (struct i915_hw_ppgtt *ppgtt)@@ -1152,30 +1247,6 @@ static int gen6_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt)return 0 ; } -static int gen6_ppgtt_setup_page_tables (struct i915_hw_ppgtt *ppgtt) -{ -struct drm_device *dev = ppgtt->base.dev; -int i; - -for (i =0 ; i < ppgtt->num_pd_entries; i++) { -struct page *page; -dma_addr_t pt_addr; - - page = ppgtt->pd.page_tables[i]->page; - pt_addr =pci_map_page (dev->pdev, page,0 ,4096 , -PCI_DMA_BIDIRECTIONAL ); - -if (pci_dma_mapping_error (dev->pdev, pt_addr)) { -gen6_ppgtt_unmap_pages (ppgtt); -return -EIO ; - } - - ppgtt->pd.page_tables[i]->daddr = pt_addr; - } - -return 0 ; -} -static int gen6_ppgtt_init (struct i915_hw_ppgtt *ppgtt) {struct drm_device *dev = ppgtt->base.dev;@@ -1196,12 +1267,7 @@ static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt)if (ret)return ret; - ret =gen6_ppgtt_setup_page_tables (ppgtt); -if (ret) { -gen6_ppgtt_free (ppgtt); -return ret; - } - + ppgtt->base.allocate_va_range = gen6_alloc_va_range; ppgtt->base.clear_range = gen6_ppgtt_clear_range; ppgtt->base.insert_entries = gen6_ppgtt_insert_entries; ppgtt->base.cleanup = gen6_ppgtt_cleanup;@@ -1212,13 +1278,17 @@ static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt) ppgtt->pd.pd_offset = ppgtt->node.start /PAGE_SIZE *sizeof (gen6_gtt_pte_t); + ppgtt->pd_addr = (gen6_gtt_pte_t __iomem *)dev_priv->gtt.gsm + + ppgtt->pd.pd_offset /sizeof (gen6_gtt_pte_t); + ppgtt->base.clear_range (&ppgtt->base,0 , ppgtt->base.total, true); +gen6_write_page_range (dev_priv, &ppgtt->pd,0 , ppgtt->base.total); +DRM_DEBUG_DRIVER ("Allocated pde space (%ldM) at GTT entry: %lx\n" , ppgtt->node.size >>20 , ppgtt->node.start /PAGE_SIZE ); -gen6_write_pdes (ppgtt);DRM_DEBUG ("Adding PPGTT at offset %x\n" , ppgtt->pd.pd_offset <<10 );@@ -1491,13 +1561,14 @@ void i915_gem_restore_gtt_mappings(struct drm_device *dev)list_for_each_entry (vm, &dev_priv->vm_list, global_link) {/* TODO: Perhaps it shouldn't be gen6 specific */ -if (i915_is_ggtt (vm)) { -if (dev_priv->mm.aliasing_ppgtt)drivers/gpu/drm/i915/i915_gem_gtt.c void i915_gem_restore_gtt_mappings(struct drm_device *dev) · line 1569
Quoted 10 lines
drivers/gpu/drm/i915/i915_gem_gtt.c +5
-4-gen6_write_pdes (dev_priv->mm.aliasing_ppgtt); -continue ; - } -gen6_write_pdes (container_of(vm ,struct i915_hw_ppgtt ,base )); +struct i915_hw_ppgtt *ppgtt = + container_of(vm, struct i915_hw_ppgtt, base); + +if (i915_is_ggtt (vm)) + ppgtt = dev_priv->mm.aliasing_ppgtt;If we have ggtt but aliasing is not enabled we get NULL here and oops over in next function?
Quoted 2 lines
-Mika
I applied the changes in v7.
Thanks for the review.
-Michel
_______________________________________________
Intel-gfx mailing list
[email protected]
http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH v4 10/24 drm/i915: Extract context switch skip and pd load logic
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
We have some fanciness coming up. This patch just breaks out the logic of context switch skip, pd load pre, and pd load post.
v2: Use new functions to replace the logic right away (Daniel)
--- drivers/gpu/drm/i915/i915_gem_context.c | 40 +++++++++++++++++++++++++-------- 1 file changed, 31 insertions(+), 9 deletions(-)
drivers/gpu/drm/i915/i915_gem_context.c +31 -9
@@ -565,6 +565,33 @@ mi_set_context(struct intel_engine_cs *ring,return ret; } +static inline bool should_skip_switch (struct intel_engine_cs *ring, +struct intel_context *from, +struct intel_context *to) +{ +if (from == to && !to->remap_slice) +return true; + +return false; +} + +static bool +needs_pd_load_pre (struct intel_engine_cs *ring,struct intel_context *to) +{ +struct drm_i915_private *dev_priv = ring->dev->dev_private; + +return ((INTEL_INFO (ring->dev)->gen <8 ) || + (ring != &dev_priv->ring[RCS ])) && to->ppgtt; +} + +static bool +needs_pd_load_post (struct intel_engine_cs *ring,struct intel_context *to) +{ +return (!to->legacy_hw_ctx.initialized || +i915_gem_context_is_default (to)) && + to->ppgtt &&IS_GEN8 (ring->dev); +} +static int do_switch (struct intel_engine_cs *ring,struct intel_context *to) {
@@ -573,9 +600,6 @@ static int do_switch(struct intel_engine_cs *ring,u32 hw_flags =0 ;bool uninitialized = false;struct i915_vma *vma; -bool needs_pd_load_pre = ((INTEL_INFO (ring->dev)->gen <8 ) || - (ring != &dev_priv->ring[RCS ])) && to->ppgtt; -bool needs_pd_load_post = false;int ret, i;if (from !=NULL && ring == &dev_priv->ring[RCS ]) {
@@ -583,7 +607,7 @@ static int do_switch(struct intel_engine_cs *ring,BUG_ON (!i915_gem_obj_is_pinned (from->legacy_hw_ctx.rcs_state)); } -if (from == to && !to->remap_slice) +if (should_skip_switch (ring, from, to))return 0 ;/* Trying to pin first makes error handling easier. */
@@ -601,7 +625,7 @@ static int do_switch(struct intel_engine_cs *ring,*/ from = ring->last_context; -if (needs_pd_load_pre) { +if (needs_pd_load_pre (ring, to)) {/* Older GENs and non render rings still want the load first, * "PP_DCLV followed by PP_DIR_BASE register through Load * Register Immediate commands in Ring Buffer before submitting
@@ -646,16 +670,14 @@ static int do_switch(struct intel_engine_cs *ring,* XXX: If we implemented page directory eviction code, this * optimization needs to be removed. */ -if (!to->legacy_hw_ctx.initialized ||i915_gem_context_is_default (to)) { +if (!to->legacy_hw_ctx.initialized ||i915_gem_context_is_default (to)) hw_flags |=MI_RESTORE_INHIBIT ; - needs_pd_load_post = to->ppgtt &&IS_GEN8 (ring->dev); - } ret =mi_set_context (ring, to, hw_flags);if (ret) goto unpin_out; -if (needs_pd_load_post) { +if (needs_pd_load_post (ring, to)) { ret = to->ppgtt->switch_mm (to->ppgtt, ring);/* The hardware context switch is emitted, but we haven't * actually changed the state - so it's probably safe to bail
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH v4 11/24 drm/i915: Track page table reload need
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
This patch was formerly known as, "Force pd restore when PDEs change, gen6-7." I had to change the name because it is needed for GEN8 too.
The real issue this is trying to solve is when a new object is mapped into the current address space. The GPU does not snoop the new mapping so we must do the gen specific action to reload the page tables.
GEN8 and GEN7 do differ in the way they load page tables for the RCS. GEN8 does so with the context restore, while GEN7 requires the proper load commands in the command streamer. Non-render is similar for both.
Caveat for GEN7
The docs say you cannot change the PDEs of a currently running context. We never map new PDEs of a running context, and expect them to be present - so I think this is okay. (We can unmap, but this should also be okay since we only unmap unreferenced objects that the GPU shouldn't be tryingto va->pa xlate.) The MI_SET_CONTEXT command does have a flag to signal that even if the context is the same, force a reload. It's unclear exactly what this does, but I have a hunch it's the right thing to do.
The logic assumes that we always emit a context switch after mapping new PDEs, and before we submit a batch. This is the case today, and has been the case since the inception of hardware contexts. A note in the comment let's the user know.
It's not just for gen8. If the current context has mappings change, we need a context reload to switch
v2: Rebased after ppgtt clean up patches. Split the warning for aliasing and true ppgtt options. And do not break aliasing ppgtt, where to->ppgtt is always null.
v3: Invalidate PPGTT TLBs inside alloc_va_range.
v4: Rename ppgtt_invalidate_tlbs to mark_tlbs_dirty and move pd_dirty_rings from i915_address_space to i915_hw_ppgtt. Fixes when neither ctx->ppgtt and aliasing_ppgtt exist.
v5: Removed references to teardown_va_range.
--- drivers/gpu/drm/i915/i915_gem_context.c | 29 ++++++++++++++++++++++++----- drivers/gpu/drm/i915/i915_gem_execbuffer.c | 11 +++++++++++ drivers/gpu/drm/i915/i915_gem_gtt.c | 11 +++++++++++ drivers/gpu/drm/i915/i915_gem_gtt.h | 1 + 4 files changed, 47 insertions(+), 5 deletions(-)
drivers/gpu/drm/i915/i915_gem_context.c +24 -5
@@ -569,8 +569,20 @@ static inline bool should_skip_switch(struct intel_engine_cs *ring,struct intel_context *from, struct intel_context *to) { -if (from == to && !to->remap_slice) -return true; + struct drm_i915_private *dev_priv = ring->dev->dev_private; + +if (to->remap_slice) +return false; + +if (to->ppgtt) { +if (from == to && !test_bit (ring->id, + &to->ppgtt->pd_dirty_rings)) +return true; + }else if (dev_priv->mm.aliasing_ppgtt) { +if (from == to && !test_bit (ring->id, + &dev_priv->mm.aliasing_ppgtt->pd_dirty_rings)) +return true; + }return false; }
@@ -587,9 +599,8 @@ needs_pd_load_pre(struct intel_engine_cs *ring, struct intel_context *to)static bool needs_pd_load_post (struct intel_engine_cs *ring,struct intel_context *to) { -return (!to->legacy_hw_ctx.initialized || -i915_gem_context_is_default (to)) && - to->ppgtt &&IS_GEN8 (ring->dev); +return IS_GEN8 (ring->dev) && + (to->ppgtt || &to->ppgtt->pd_dirty_rings); } static int do_switch(struct intel_engine_cs *ring,
@@ -634,6 +645,12 @@ static int do_switch(struct intel_engine_cs *ring, ret = to->ppgtt->switch_mm (to->ppgtt, ring);if (ret) goto unpin_out; + +/* Doing a PD load always reloads the page dirs */ +if (to->ppgtt) +clear_bit (ring->id, &to->ppgtt->pd_dirty_rings); +else +clear_bit (ring->id, &dev_priv->mm.aliasing_ppgtt->pd_dirty_rings); }if (ring != &dev_priv->ring[RCS ]) {
@@ -672,6 +689,8 @@ static int do_switch(struct intel_engine_cs *ring,*/ if (!to->legacy_hw_ctx.initialized ||i915_gem_context_is_default (to)) hw_flags |=MI_RESTORE_INHIBIT ; +else if (to->ppgtt &&test_and_clear_bit (ring->id, &to->ppgtt->pd_dirty_rings)) + hw_flags |=MI_FORCE_RESTORE ; ret =mi_set_context (ring, to, hw_flags);if (ret)
drivers/gpu/drm/i915/i915_gem_execbuffer.c +11 -0
@@ -1198,6 +1198,13 @@ i915_gem_ringbuffer_submission(struct drm_device *dev, struct drm_file *file,if (ret) goto error; +if (ctx->ppgtt) +WARN (ctx->ppgtt->pd_dirty_rings & (1 <<ring->id), +"%s didn't clear reload\n" , ring->name); +else if (dev_priv->mm.aliasing_ppgtt) +WARN (dev_priv->mm.aliasing_ppgtt->pd_dirty_rings & + (1 <<ring->id),"%s didn't clear reload\n" , ring->name); + instp_mode = args->flags &I915_EXEC_CONSTANTS_MASK ; instp_mask =I915_EXEC_CONSTANTS_MASK ;switch (instp_mode) {
@@ -1445,6 +1452,10 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,if (ret) goto err; +/* XXX: Reserve has possibly change PDEs which means we must do a +* context switch before we can coherently read some of the reserved +* VMAs. */ +/* The objects are in their final locations, apply the relocations. */ if (need_relocs) ret =i915_gem_execbuffer_relocate (eb);
drivers/gpu/drm/i915/i915_gem_gtt.c +11 -0
@@ -1140,6 +1140,16 @@ static void gen6_ppgtt_unmap_pages(struct i915_hw_ppgtt *ppgtt) 4096, PCI_DMA_BIDIRECTIONAL); } +/* PDE TLBs are a pain invalidate pre GEN8. It requires a context reload. If we +* are switching between contexts with the same LRCA, we also must do a force +* restore. +*/ +static inline void mark_tlbs_dirty (struct i915_hw_ppgtt *ppgtt) +{ +/* If current vm != vm, */ + ppgtt->pd_dirty_rings =INTEL_INFO (ppgtt->base.dev)->ring_mask; +} +static int gen6_alloc_va_range (struct i915_address_space *vm,uint64_t start,uint64_t length) {
@@ -1159,6 +1169,7 @@ static int gen6_alloc_va_range(struct i915_address_space *vm, I915_PPGTT_PT_ENTRIES); } +mark_tlbs_dirty (ppgtt);return 0 ; }
drivers/gpu/drm/i915/i915_gem_gtt.h +1 -0
@@ -297,6 +297,7 @@ struct i915_hw_ppgtt {struct i915_address_space base;struct kref ref;struct drm_mm_node node; +unsigned long pd_dirty_rings;unsigned num_pd_entries;unsigned num_pd_pages;/* gen8+ */ union {
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH v4 12/24 drm/i915: Initialize all contexts
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
The problem is we're going to switch to a new context, which could be the default context. The plan was to use restore inhibit, which would be fine, except if we are using dynamic page tables (which we will). If we use dynamic page tables and we don't load new page tables, the previous page tables might go away, and future operations will fault.
CTXA runs.
switch to default, restore inhibit
CTXA dies and has its address space taken away.
Run CTXB, tries to save using the context A's address space - this fails.
The general solution is to make sure every context has it's own state, and its own address space. For cases when we must restore inhibit, first thing we do is load a valid address space. I thought this would be enough, but apparently there are references within the context itself which will refer to the old address space - therefore, we also must reinitialize.
It was tricky to track this down as we don't have much insight into what happens in a context save.
This is required for the next patch which enables dynamic page tables.
v2: to->ppgtt is only valid in full ppgtt.
--- drivers/gpu/drm/i915/i915_gem_context.c | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-)
drivers/gpu/drm/i915/i915_gem_context.c +11 -14
@@ -596,13 +596,6 @@ needs_pd_load_pre(struct intel_engine_cs *ring, struct intel_context *to) (ring != &dev_priv->ring[RCS])) && to->ppgtt; } -static bool -needs_pd_load_post (struct intel_engine_cs *ring,struct intel_context *to) -{ -return IS_GEN8 (ring->dev) && - (to->ppgtt || &to->ppgtt->pd_dirty_rings); -} -static int do_switch (struct intel_engine_cs *ring,struct intel_context *to) {
@@ -683,20 +676,24 @@ static int do_switch(struct intel_engine_cs *ring,/* GEN8 does *not* require an explicit reload if the PDPs have been * setup, and we do not wish to move them. -* -* XXX: If we implemented page directory eviction code, this -* optimization needs to be removed. */ -if (!to->legacy_hw_ctx.initialized ||i915_gem_context_is_default (to)) +if (!to->legacy_hw_ctx.initialized) { hw_flags |=MI_RESTORE_INHIBIT ; -else if (to->ppgtt &&test_and_clear_bit (ring->id, &to->ppgtt->pd_dirty_rings)) +/* NB: If we inhibit the restore, the context is not allowed to +* die because future work may end up depending on valid address +* space. This means we must enforce that a page table load +* occur when this occurs. */ + }else if (to->ppgtt &&test_and_clear_bit (ring->id, &to->ppgtt->pd_dirty_rings)) hw_flags |=MI_FORCE_RESTORE ; ret =mi_set_context (ring, to, hw_flags);if (ret) goto unpin_out; -if (needs_pd_load_post (ring, to)) { +if (IS_GEN8 (ring->dev) && to->ppgtt && (hw_flags &MI_RESTORE_INHIBIT )) { +/* We have a valid page directory (scratch) to switch to. This +* allows the old VM to be freed. Note that if anything occurs +* between the set context, and here, we are f*cked */ ret = to->ppgtt->switch_mm (to->ppgtt, ring);/* The hardware context switch is emitted, but we haven't * actually changed the state - so it's probably safe to bail
@@ -746,7 +743,7 @@ static int do_switch(struct intel_engine_cs *ring,i915_gem_context_unreference (from); } - uninitialized = !to->legacy_hw_ctx.initialized && from ==NULL ; + uninitialized = !to->legacy_hw_ctx.initialized; to->legacy_hw_ctx.initialized = true; done:
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH v4 13/24 drm/i915: Finish gen6/7 dynamic page table allocation
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
This patch continues on the idea from the previous patch. From here on, in the steady state, PDEs are all pointing to the scratch page table (as recommended in the spec). When an object is allocated in the VA range, the code will determine if we need to allocate a page for the page table. Similarly when the object is destroyed, we will remove, and free the page table pointing the PDE back to the scratch page.
Following patches will work to unify the code a bit as we bring in GEN8 support. GEN6 and GEN8 are different enough that I had a hard time to get to this point with as much common code as I do.
The aliasing PPGTT must pre-allocate all of the page tables. There are a few reasons for this. Two trivial ones: aliasing ppgtt goes through the ggtt paths, so it's hard to maintain, we currently do not restore the default context (assuming the previous force reload is indeed necessary). Most importantly though, the only way (it seems from empirical evidence) to invalidate the CS TLBs on non-render ring is to either use ring sync (which requires actually stopping the rings in order to synchronize when the sync completes vs. where you are in execution), or to reload DCLV. Since without full PPGTT we do not ever reload the DCLV register, there is no good way to achieve this. The simplest solution is just to not support dynamic page table creation/destruction in the aliasing PPGTT.
We could always reload DCLV, but this seems like quite a bit of excess overhead only to save at most 2MB-4k of memory for the aliasing PPGTT page tables.
v2: Make the page table bitmap declared inside the function (Chris) Simplify the way scratching address space works.
Move the alloc/teardown tracepoints up a level in the call stack so that both all implementations get the trace.
v3: Updated trace event to spit out a name
v4: Aliasing ppgtt is now initialized differently (in setup global gtt)
v5: Rebase to latest code. Also removed unnecessary aliasing ppgtt check for trace, as it is no longer possible after the PPGTT cleanup patch series of a couple of months ago (Daniel).
v6: Implement changes from code review (Daniel):
- allocate/teardown_va_range calls added.
- Add a scratch page allocation helper (only need the address).
- Move trace events to a new patch.
- Use updated mark_tlbs_dirty.
- Moved pt preallocation for aliasing ppgtt into gen6_ppgtt_init.
v7: teardown_va_range removed (Daniel).
In init, gen6_ppgtt_clear_range call is only needed for aliasing ppgtt.
--- drivers/gpu/drm/i915/i915_debugfs.c | 3 +- drivers/gpu/drm/i915/i915_gem.c | 9 +++ drivers/gpu/drm/i915/i915_gem_gtt.c | 125 +++++++++++++++++++++++++++++++----- drivers/gpu/drm/i915/i915_gem_gtt.h | 3 + 4 files changed, 123 insertions(+), 17 deletions(-)
drivers/gpu/drm/i915/i915_debugfs.c +2 -1
@@ -2149,6 +2149,8 @@ static void gen6_ppgtt_info(struct seq_file *m, struct drm_device *dev)seq_printf (m,"PP_DIR_BASE_READ: 0x%08x\n" ,I915_READ (RING_PP_DIR_BASE_READ (ring)));seq_printf (m,"PP_DIR_DCLV: 0x%08x\n" ,I915_READ (RING_PP_DIR_DCLV (ring))); } +seq_printf (m,"ECOCHK: 0x%08x\n\n" ,I915_READ (GAM_ECOCHK )); +if (dev_priv->mm.aliasing_ppgtt) {struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
@@ -2165,7 +2167,6 @@ static void gen6_ppgtt_info(struct seq_file *m, struct drm_device *dev) get_pid_task(file->pid, PIDTYPE_PID)->comm);idr_for_each (&file_priv->context_idr, per_file_ctx, m); } -seq_printf (m,"ECOCHK: 0x%08x\n" ,I915_READ (GAM_ECOCHK )); }static int i915_ppgtt_info (struct seq_file *m,void *data)
drivers/gpu/drm/i915/i915_gem.c +9 -0
@@ -3566,6 +3566,15 @@ search_free:if (ret) goto err_remove_node; +/* allocate before insert / bind */ +if (vma->vm->allocate_va_range) { + ret = vma->vm->allocate_va_range (vma->vm, + vma->node.start, + vma->node.size); +if (ret) + goto err_remove_node; + } +trace_i915_vma_bind (vma, flags); ret =i915_vma_bind (vma, obj->cache_level, flags &PIN_GLOBAL ?GLOBAL_BIND :0 );
drivers/gpu/drm/i915/i915_gem_gtt.c +109 -16
217 lines not on this page · raw message
drivers/gpu/drm/i915/i915_gem_gtt.h +3 -0
@@ -195,6 +195,7 @@ struct i915_page_table_entry {dma_addr_t daddr;unsigned long *used_ptes; + unsigned int scratch:1; };struct i915_page_directory_entry {
@@ -305,6 +306,8 @@ struct i915_hw_ppgtt {struct i915_page_directory_entry pd; }; +struct i915_page_table_entry *scratch_pt; +struct drm_i915_file_private *file_priv;gen6_gtt_pte_t __iomem *pd_addr;
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH v4 14/24 drm/i915: Add dynamic page trace events
In reply to Michel ThierryTraces for page directories and tables allocation and map.
v2: Removed references to teardown.
--- drivers/gpu/drm/i915/i915_gem.c | 2 + drivers/gpu/drm/i915/i915_gem_gtt.c | 5 ++ drivers/gpu/drm/i915/i915_trace.h | 99 +++++++++++++++++++++++++++++++++++++ 3 files changed, 106 insertions(+)
drivers/gpu/drm/i915/i915_gem.c +2 -0
@@ -3568,6 +3568,8 @@ search_free:/* allocate before insert / bind */ if (vma->vm->allocate_va_range) { +trace_i915_va_alloc (vma->vm, vma->node.start, vma->node.size, +VM_TO_TRACE_NAME (vma->vm)); ret = vma->vm->allocate_va_range (vma->vm, vma->node.start, vma->node.size);
drivers/gpu/drm/i915/i915_gem_gtt.c +5 -0
@@ -1199,6 +1199,7 @@ static int gen6_alloc_va_range(struct i915_address_space *vm, ppgtt->pd.page_tables[pde] = pt;set_bit (pde, new_page_tables); +trace_i915_page_table_entry_alloc (vm, pde, start,GEN6_PDE_SHIFT ); } start = start_save;
@@ -1214,6 +1215,10 @@ static int gen6_alloc_va_range(struct i915_address_space *vm,if (test_and_clear_bit (pde, new_page_tables))gen6_write_pdes (&ppgtt->pd, pde, pt); +trace_i915_page_table_entry_map (vm, pde, pt, +gen6_pte_index (start), +gen6_pte_count (start, length), +I915_PPGTT_PT_ENTRIES );bitmap_or (pt->used_ptes, tmp_bitmap, pt->used_ptes,I915_PPGTT_PT_ENTRIES ); }
drivers/gpu/drm/i915/i915_trace.h +99 -0
105 lines not on this page · raw message
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH v4 15/24 drm/i915/bdw: Use dynamic allocation idioms on free
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
The page directory freer is left here for now as it's still useful given that GEN8 still preallocates. Once the allocation functions are broken up into more discrete chunks, we'll follow suit and destroy this leftover piece.
v2: Match trace_i915_va_teardown params
v3: Multiple rebases.
v4: Updated to use unmap_and_free_pt.
v5: teardown_va_range logic no longer needed.
--- drivers/gpu/drm/i915/i915_gem_gtt.c | 26 ++++++++++---------- drivers/gpu/drm/i915/i915_gem_gtt.h | 47 +++++++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 13 deletions(-)
drivers/gpu/drm/i915/i915_gem_gtt.c +13 -13
@@ -604,19 +604,6 @@ static void gen8_free_page_tables(struct i915_page_directory_entry *pd, struct d } } -static void gen8_ppgtt_free (struct i915_hw_ppgtt *ppgtt) -{ -int i; - -for (i =0 ; i < ppgtt->num_pd_pages; i++) { -if (WARN_ON (!ppgtt->pdp.page_directory[i])) -continue ; - -gen8_free_page_tables (ppgtt->pdp.page_directory[i], ppgtt->base.dev); -unmap_and_free_pd (ppgtt->pdp.page_directory[i]); - } -} -static void gen8_ppgtt_unmap_pages (struct i915_hw_ppgtt *ppgtt) {struct pci_dev *hwdev = ppgtt->base.dev->pdev;
@@ -649,6 +636,19 @@ static void gen8_ppgtt_unmap_pages(struct i915_hw_ppgtt *ppgtt) } } +static void gen8_ppgtt_free (struct i915_hw_ppgtt *ppgtt) +{ +int i; + +for (i =0 ; i < ppgtt->num_pd_pages; i++) { +if (WARN_ON (!ppgtt->pdp.page_directory[i])) +continue ; + +gen8_free_page_tables (ppgtt->pdp.page_directory[i], ppgtt->base.dev); +unmap_and_free_pd (ppgtt->pdp.page_directory[i]); + } +} +static void gen8_ppgtt_cleanup (struct i915_address_space *vm) { struct i915_hw_ppgtt *ppgtt =
drivers/gpu/drm/i915/i915_gem_gtt.h +47 -0
53 lines not on this page · raw message
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH v4 16/24 drm/i915/bdw: page directories rework allocation
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
Start using gen8_for_each_pdpe macro to allocate the page directories.
v2: Rebased after s/free_pt_*/unmap_and_free_pt/ change.
v3: Rebased after teardown va range logic was removed.
--- drivers/gpu/drm/i915/i915_gem_gtt.c | 43 ++++++++++++++++++++++++++----------- 1 file changed, 30 insertions(+), 13 deletions(-)
drivers/gpu/drm/i915/i915_gem_gtt.c +30 -13
@@ -678,25 +678,39 @@ unwind_out:return -ENOMEM ; } -static int gen8_ppgtt_allocate_page_directories (struct i915_hw_ppgtt *ppgtt, -const int max_pdp) +static int gen8_ppgtt_alloc_page_directories (struct i915_page_directory_pointer_entry *pdp, +uint64_t start, +uint64_t length) { -int i; - -for (i =0 ; i < max_pdp; i++) { - ppgtt->pdp.page_directory[i] =alloc_pd_single (); -if (IS_ERR (ppgtt->pdp.page_directory[i])) +struct i915_hw_ppgtt *ppgtt = + container_of(pdp, struct i915_hw_ppgtt, pdp); +struct i915_page_directory_entry *unused; +uint64_t temp; +uint32_t pdpe; + +/* FIXME: PPGTT container_of won't work for 64b */ +BUG_ON ((start + length) >0x800000000ULL ); + +gen8_for_each_pdpe (unused, pdp, start, length, temp, pdpe) { +BUG_ON (unused); + pdp->page_directory[pdpe] =alloc_pd_single (); +if (IS_ERR (ppgtt->pdp.page_directory[pdpe])) goto unwind_out; + + ppgtt->num_pd_pages++; } - ppgtt->num_pd_pages = max_pdp;BUG_ON (ppgtt->num_pd_pages >GEN8_LEGACY_PDPES );return 0 ; unwind_out: -while (i--) -unmap_and_free_pd (ppgtt->pdp.page_directory[i]); +while (pdpe--) { +unmap_and_free_pd (ppgtt->pdp.page_directory[pdpe]); + ppgtt->num_pd_pages--; + } + +WARN_ON (ppgtt->num_pd_pages);return -ENOMEM ; }
@@ -706,7 +720,8 @@ static int gen8_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt, {int ret; - ret =gen8_ppgtt_allocate_page_directories (ppgtt, max_pdp); + ret =gen8_ppgtt_alloc_page_directories (&ppgtt->pdp, ppgtt->base.start, + ppgtt->base.total);if (ret)return ret;
@@ -783,6 +798,10 @@ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size)if (size % (1 <<30 ))DRM_INFO ("Pages will be wasted unless GTT size (%llu) is divisible by 1GB\n" , size); + ppgtt->base.start =0 ; + ppgtt->base.total = size; +BUG_ON (ppgtt->base.total ==0 ); +/* 1. Do all our allocations for page directories and page tables. */ ret =gen8_ppgtt_alloc (ppgtt, max_pdp);if (ret)
@@ -830,8 +849,6 @@ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size) ppgtt->base.clear_range = gen8_ppgtt_clear_range; ppgtt->base.insert_entries = gen8_ppgtt_insert_entries; ppgtt->base.cleanup = gen8_ppgtt_cleanup; - ppgtt->base.start =0 ; - ppgtt->base.total = ppgtt->num_pd_entries *GEN8_PTES_PER_PAGE *PAGE_SIZE ; ppgtt->base.clear_range (&ppgtt->base,0 , ppgtt->base.total, true);
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH v4 17/24 drm/i915/bdw: pagetable allocation rework
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
Start using gen8_for_each_pde macro to allocate page tables.
v2: teardown_va_range references removed.
--- drivers/gpu/drm/i915/i915_gem_gtt.c | 46 +++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 17 deletions(-)
drivers/gpu/drm/i915/i915_gem_gtt.c +29 -17
@@ -658,22 +658,27 @@ static void gen8_ppgtt_cleanup(struct i915_address_space *vm)gen8_ppgtt_free (ppgtt); } -static int gen8_ppgtt_allocate_page_tables (struct i915_hw_ppgtt *ppgtt) +static int gen8_ppgtt_alloc_pagetabs (struct i915_page_directory_entry *pd, +uint64_t start, +uint64_t length, +struct drm_device *dev) { -int i, ret; +struct i915_page_table_entry *unused; +uint64_t temp; +uint32_t pde; -for (i =0 ; i < ppgtt->num_pd_pages; i++) { - ret =alloc_pt_range (ppgtt->pdp.page_directory[i], -0 ,GEN8_PDES_PER_PAGE , ppgtt->base.dev); -if (ret) +gen8_for_each_pde (unused, pd, start, length, temp, pde) { +BUG_ON (unused); + pd->page_tables[pde] =alloc_pt_single (dev); +if (IS_ERR (pd->page_tables[pde])) goto unwind_out; }return 0 ; unwind_out: -while (i--) -gen8_free_page_tables (ppgtt->pdp.page_directory[i], ppgtt->base.dev); +while (pde--) +unmap_and_free_pt (pd->page_tables[pde], dev);return -ENOMEM ; }
@@ -716,20 +721,28 @@ unwind_out: }static int gen8_ppgtt_alloc (struct i915_hw_ppgtt *ppgtt, -const int max_pdp) +uint64_t start, +uint64_t length) { +struct i915_page_directory_entry *pd; +uint64_t temp; +uint32_t pdpe;int ret; - ret =gen8_ppgtt_alloc_page_directories (&ppgtt->pdp, ppgtt->base.start, - ppgtt->base.total); + ret =gen8_ppgtt_alloc_page_directories (&ppgtt->pdp, start, length);if (ret)return ret; - ret =gen8_ppgtt_allocate_page_tables (ppgtt); -if (ret) - goto err_out; +gen8_for_each_pdpe (pd, &ppgtt->pdp, start, length, temp, pdpe) { + ret =gen8_ppgtt_alloc_pagetabs (pd, start, length, + ppgtt->base.dev); +if (ret) + goto err_out; + + ppgtt->num_pd_entries +=GEN8_PDES_PER_PAGE ; + } - ppgtt->num_pd_entries = max_pdp *GEN8_PDES_PER_PAGE ; +BUG_ON (pdpe > ppgtt->num_pd_pages);return 0 ;
@@ -800,10 +813,9 @@ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size) ppgtt->base.start =0 ; ppgtt->base.total = size; -BUG_ON (ppgtt->base.total ==0 );/* 1. Do all our allocations for page directories and page tables. */ - ret =gen8_ppgtt_alloc (ppgtt, max_pdp); + ret =gen8_ppgtt_alloc (ppgtt, ppgtt->base.start, ppgtt->base.total);if (ret)return ret;
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH v4 18/24 drm/i915/bdw: Update pdp switch and point unused PDPs to scratch page
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
One important part of this patch is we now write a scratch page directory into any unused PDP descriptors. This matters for 2 reasons, first, we're not allowed to just use 0, or an invalid pointer, and second, we must wipe out any previous contents from the last context.
The latter point only matters with full PPGTT. The former point only effect platforms with less than 4GB memory.
v2: Updated commit message to point that we must set unused PDPs to the scratch page.
--- drivers/gpu/drm/i915/i915_gem_gtt.c | 29 ++++++++++++++++++----------- drivers/gpu/drm/i915/i915_gem_gtt.h | 5 ++++- 2 files changed, 22 insertions(+), 12 deletions(-)
drivers/gpu/drm/i915/i915_gem_gtt.c +18 -11
@@ -439,8 +439,9 @@ static struct i915_page_directory_entry *alloc_pd_single(void) }/* Broadwell Page Directory Pointer Descriptors */ -static int gen8_write_pdp (struct intel_engine_cs *ring,unsigned entry, -uint64_t val) +static int gen8_write_pdp (struct intel_engine_cs *ring, +unsigned entry, +dma_addr_t addr) {int ret;
@@ -452,10 +453,10 @@ static int gen8_write_pdp(struct intel_engine_cs *ring, unsigned entry,intel_ring_emit (ring,MI_LOAD_REGISTER_IMM (1 ));intel_ring_emit (ring,GEN8_RING_PDP_UDW (ring, entry)); -intel_ring_emit (ring, (u32)(val >>32 )); +intel_ring_emit (ring,upper_32_bits (addr));intel_ring_emit (ring,MI_LOAD_REGISTER_IMM (1 ));intel_ring_emit (ring,GEN8_RING_PDP_LDW (ring, entry)); -intel_ring_emit (ring, (u32)(val)); +intel_ring_emit (ring,lower_32_bits (addr));intel_ring_advance (ring);return 0 ;
@@ -466,12 +467,12 @@ static int gen8_mm_switch(struct i915_hw_ppgtt *ppgtt, {int i, ret; -/* bit of a hack to find the actual last used pd */ -int used_pd = ppgtt->num_pd_entries /GEN8_PDES_PER_PAGE ; - -for (i = used_pd -1 ; i >=0 ; i--) { -dma_addr_t addr = ppgtt->pdp.page_directory[i]->daddr; - ret =gen8_write_pdp (ring, i, addr); +for (i =GEN8_LEGACY_PDPES -1 ; i >=0 ; i--) { +struct i915_page_directory_entry *pd = ppgtt->pdp.page_directory[i]; +dma_addr_t pd_daddr = pd ? pd->daddr : ppgtt->scratch_pd->daddr; +/* The page directory might be NULL, but we need to clear out +* whatever the previous context might have used. */ + ret =gen8_write_pdp (ring, i, pd_daddr);if (ret)return ret; }
@@ -814,10 +815,16 @@ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size) ppgtt->base.start =0 ; ppgtt->base.total = size; + ppgtt->scratch_pd =alloc_pt_scratch (ppgtt->base.dev); +if (IS_ERR (ppgtt->scratch_pd)) +return PTR_ERR (ppgtt->scratch_pd); +/* 1. Do all our allocations for page directories and page tables. */ ret =gen8_ppgtt_alloc (ppgtt, ppgtt->base.start, ppgtt->base.total); -if (ret) +if (ret) { +unmap_and_free_pt (ppgtt->scratch_pd, ppgtt->base.dev);return ret; + }/* * 2. Create DMA mappings for the page directories and page tables.
drivers/gpu/drm/i915/i915_gem_gtt.h +4 -1
@@ -306,7 +306,10 @@ struct i915_hw_ppgtt {struct i915_page_directory_entry pd; }; -struct i915_page_table_entry *scratch_pt; +union { +struct i915_page_table_entry *scratch_pt; +struct i915_page_table_entry *scratch_pd;/* Just need the daddr */ + };struct drm_i915_file_private *file_priv;
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH v4 19/24 drm/i915: num_pd_pages/num_pd_entries isn't useful
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
These values are never quite useful for dynamic allocations of the page tables. Getting rid of them will help prevent later confusion.
v2: Updated to use unmap_and_free_pd functions.
v3: Updated gen8_ppgtt_free after teardown logic was removed.
--- drivers/gpu/drm/i915/i915_debugfs.c | 2 -- drivers/gpu/drm/i915/i915_gem_gtt.c | 72 ++++++++++++------------------------- drivers/gpu/drm/i915/i915_gem_gtt.h | 7 ++-- 3 files changed, 28 insertions(+), 53 deletions(-)
drivers/gpu/drm/i915/i915_debugfs.c +0 -2
@@ -2117,8 +2117,6 @@ static void gen8_ppgtt_info(struct seq_file *m, struct drm_device *dev)if (!ppgtt)return ; -seq_printf (m,"Page directories: %d\n" , ppgtt->num_pd_pages); -seq_printf (m,"Page tables: %d\n" , ppgtt->num_pd_entries);for_each_ring (ring, dev_priv, unused) {seq_printf (m,"%s\n" , ring->name);for (i =0 ; i <4 ; i++) {
drivers/gpu/drm/i915/i915_gem_gtt.c +23 -49
170 lines not on this page · raw message
drivers/gpu/drm/i915/i915_gem_gtt.h +5 -2
@@ -299,8 +299,6 @@ struct i915_hw_ppgtt {struct kref ref;struct drm_mm_node node;unsigned long pd_dirty_rings; -unsigned num_pd_entries; -unsigned num_pd_pages;/* gen8+ */ union {struct i915_page_directory_pointer_entry pdp;struct i915_page_directory_entry pd;
@@ -338,6 +336,11 @@ struct i915_hw_ppgtt { temp = min_t(unsigned, temp, length), \ start += temp, length -= temp) +#define gen6_for_all_pdes (pt, ppgtt, iter) \ + for (iter = 0, pt = ppgtt->pd.page_tables[iter]; \ + iter < gen6_pde_index(ppgtt->base.total); \ + pt = ppgtt->pd.page_tables[++iter]) +static inline uint32_t i915_pte_index (uint64_t address,uint32_t pde_shift) {const uint32_t mask =NUM_PTE (pde_shift) -1 ;
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH v4 20/24 drm/i915: Extract PPGTT param from page_directory alloc
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
Now that we don't need to trace num_pd_pages, we may as well kill all need for the PPGTT structure in the alloc_page_directorys. This is very useful for when we move to 48b addressing, and the PDP isn't the root of the page table structure.
The param is replaced with drm_device, which is an unavoidable wart throughout the series. (in other words, not extra flagrant).
--- drivers/gpu/drm/i915/i915_gem_gtt.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-)
drivers/gpu/drm/i915/i915_gem_gtt.c +2 -4
@@ -686,8 +686,6 @@ static int gen8_ppgtt_alloc_page_directories(struct i915_page_directory_pointer_ uint64_t start, uint64_t length) { -struct i915_hw_ppgtt *ppgtt = - container_of(pdp, struct i915_hw_ppgtt, pdp);struct i915_page_directory_entry *unused;uint64_t temp;uint32_t pdpe;
@@ -698,7 +696,7 @@ static int gen8_ppgtt_alloc_page_directories(struct i915_page_directory_pointer_gen8_for_each_pdpe (unused, pdp, start, length, temp, pdpe) {BUG_ON (unused); pdp->page_directory[pdpe] =alloc_pd_single (); -if (IS_ERR (ppgtt->pdp.page_directory[pdpe])) +if (IS_ERR (pdp->page_directory[pdpe])) goto unwind_out; }
@@ -706,7 +704,7 @@ static int gen8_ppgtt_alloc_page_directories(struct i915_page_directory_pointer_ unwind_out:while (pdpe--) -unmap_and_free_pd (ppgtt->pdp.page_directory[pdpe]); +unmap_and_free_pd (pdp->page_directory[pdpe]);return -ENOMEM ; }
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH v4 21/24 drm/i915/bdw: Split out mappings
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
When we do dynamic page table allocations for gen8, we'll need to have more control over how and when we map page tables, similar to gen6. In particular, DMA mappings for page directories/tables occur at allocation time.
This patch adds the functionality and calls it at init, which should have no functional change.
The PDPEs are still a special case for now. We'll need a function for that in the future as well.
v2: Handle renamed unmap_and_free_page functions.
v3: Updated after teardown_va logic was removed.
--- drivers/gpu/drm/i915/i915_gem_gtt.c | 177 ++++++++++++++---------------------- 1 file changed, 69 insertions(+), 108 deletions(-)
drivers/gpu/drm/i915/i915_gem_gtt.c +69 -108
@@ -413,17 +413,20 @@ err_out:return ret; } -static void unmap_and_free_pd (struct i915_page_directory_entry *pd) +static void unmap_and_free_pd (struct i915_page_directory_entry *pd, +struct drm_device *dev) {if (pd->page) { +i915_dma_unmap_single (pd, dev);__free_page (pd->page);kfree (pd); } } -static struct i915_page_directory_entry *alloc_pd_single (void ) +static struct i915_page_directory_entry *alloc_pd_single (struct drm_device *dev) {struct i915_page_directory_entry *pd; +int ret; pd =kzalloc (sizeof (*pd),GFP_KERNEL );if (!pd)
@@ -435,6 +438,13 @@ static struct i915_page_directory_entry *alloc_pd_single(void)return ERR_PTR (-ENOMEM ); } + ret =i915_dma_map_px_single (pd, dev); +if (ret) { +__free_page (pd->page); +kfree (pd); +return ERR_PTR (ret); + } +return pd; }
@@ -589,6 +599,36 @@ static void gen8_ppgtt_insert_entries(struct i915_address_space *vm, } } +static void __gen8_do_map_pt (gen8_ppgtt_pde_t *pde, +struct i915_page_table_entry *pt, +struct drm_device *dev) +{ +gen8_ppgtt_pde_t entry = +gen8_pde_encode (dev, pt->daddr,I915_CACHE_LLC ); + *pde = entry; +} + +/* It's likely we'll map more than one pagetable at a time. This function will +* save us unnecessary kmap calls, but do no more functionally than multiple +* calls to map_pt. */ +static void gen8_map_pagetable_range (struct i915_page_directory_entry *pd, +uint64_t start, +uint64_t length, +struct drm_device *dev) +{ +gen8_ppgtt_pde_t *page_directory =kmap_atomic (pd->page); +struct i915_page_table_entry *pt; +uint64_t temp, pde; + +gen8_for_each_pde (pt, pd, start, length, temp, pde) +__gen8_do_map_pt (page_directory + pde, pt, dev); + +if (!HAS_LLC (dev)) +drm_clflush_virt_range (page_directory,PAGE_SIZE ); + +kunmap_atomic (page_directory); +} +static void gen8_free_page_tables (struct i915_page_directory_entry *pd,struct drm_device *dev) {int i;
@@ -644,7 +684,7 @@ static void gen8_ppgtt_free(struct i915_hw_ppgtt *ppgtt)continue ;gen8_free_page_tables (ppgtt->pdp.page_directory[i], ppgtt->base.dev); -unmap_and_free_pd (ppgtt->pdp.page_directory[i]); +unmap_and_free_pd (ppgtt->pdp.page_directory[i], ppgtt->base.dev); } }
@@ -684,7 +724,8 @@ unwind_out:static int gen8_ppgtt_alloc_page_directories (struct i915_page_directory_pointer_entry *pdp,uint64_t start, -uint64_t length) +uint64_t length, +struct drm_device *dev) {struct i915_page_directory_entry *unused;uint64_t temp;
@@ -695,7 +736,7 @@ static int gen8_ppgtt_alloc_page_directories(struct i915_page_directory_pointer_gen8_for_each_pdpe (unused, pdp, start, length, temp, pdpe) {BUG_ON (unused); - pdp->page_directory[pdpe] =alloc_pd_single (); + pdp->page_directory[pdpe] =alloc_pd_single (dev);if (IS_ERR (pdp->page_directory[pdpe])) goto unwind_out; }
@@ -704,21 +745,24 @@ static int gen8_ppgtt_alloc_page_directories(struct i915_page_directory_pointer_ unwind_out:while (pdpe--) -unmap_and_free_pd (pdp->page_directory[pdpe]); +unmap_and_free_pd (pdp->page_directory[pdpe], dev);return -ENOMEM ; } -static int gen8_ppgtt_alloc (struct i915_hw_ppgtt *ppgtt, -uint64_t start, -uint64_t length) +static int gen8_alloc_va_range (struct i915_address_space *vm, +uint64_t start, +uint64_t length) { +struct i915_hw_ppgtt *ppgtt = + container_of(vm, struct i915_hw_ppgtt, base);struct i915_page_directory_entry *pd;uint64_t temp;uint32_t pdpe;int ret; - ret =gen8_ppgtt_alloc_page_directories (&ppgtt->pdp, start, length); + ret =gen8_ppgtt_alloc_page_directories (&ppgtt->pdp, start, length, + ppgtt->base.dev);if (ret)return ret;
@@ -731,134 +775,51 @@ static int gen8_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt,return 0 ; -/* TODO: Check this for all cases */ err_out:gen8_ppgtt_free (ppgtt);return ret; } -static int gen8_ppgtt_setup_page_directories (struct i915_hw_ppgtt *ppgtt, -const int pd) -{ -dma_addr_t pd_addr; -int ret; - - pd_addr =pci_map_page (ppgtt->base.dev->pdev, - ppgtt->pdp.page_directory[pd]->page,0 , -PAGE_SIZE ,PCI_DMA_BIDIRECTIONAL ); - - ret =pci_dma_mapping_error (ppgtt->base.dev->pdev, pd_addr); -if (ret) -return ret; - - ppgtt->pdp.page_directory[pd]->daddr = pd_addr; - -return 0 ; -} - -static int gen8_ppgtt_setup_page_tables (struct i915_hw_ppgtt *ppgtt, -const int pd, -const int pt) -{ -dma_addr_t pt_addr; -struct i915_page_directory_entry *pdir = ppgtt->pdp.page_directory[pd]; -struct i915_page_table_entry *ptab = pdir->page_tables[pt]; -struct page *p = ptab->page; -int ret; - - pt_addr =pci_map_page (ppgtt->base.dev->pdev, - p,0 ,PAGE_SIZE ,PCI_DMA_BIDIRECTIONAL ); - ret =pci_dma_mapping_error (ppgtt->base.dev->pdev, pt_addr); -if (ret) -return ret; - - ptab->daddr = pt_addr; - -return 0 ; -} -/** * GEN8 legacy ppgtt programming is accomplished through a max 4 PDP registers * with a net effect resembling a 2-level page table in normal x86 terms. Each * PDP represents 1GB of memory 4 * 512 * 512 * 4096 = 4GB legacy 32b address * space. * -* FIXME: split allocation into smaller pieces. For now we only ever do this -* once, but with full PPGTT, the multiple contiguous allocations will be bad. -* TODO: Do something with the size parameter */ static int gen8_ppgtt_init (struct i915_hw_ppgtt *ppgtt,uint64_t size) { -const int max_pdp =DIV_ROUND_UP (size,1 <<30 ); -int i, j, ret; - -if (size % (1 <<30 )) -DRM_INFO ("Pages will be wasted unless GTT size (%llu) is divisible by 1GB\n" , size); +struct i915_page_directory_entry *pd; +uint64_t temp, start =0 ; +const uint64_t orig_length = size; +uint32_t pdpe; +int ret; ppgtt->base.start =0 ; ppgtt->base.total = size; + ppgtt->base.clear_range = gen8_ppgtt_clear_range; + ppgtt->base.insert_entries = gen8_ppgtt_insert_entries; + ppgtt->base.cleanup = gen8_ppgtt_cleanup; + ppgtt->switch_mm = gen8_mm_switch; ppgtt->scratch_pd =alloc_pt_scratch (ppgtt->base.dev);if (IS_ERR (ppgtt->scratch_pd))return PTR_ERR (ppgtt->scratch_pd); -/* 1. Do all our allocations for page directories and page tables. */ - ret =gen8_ppgtt_alloc (ppgtt, ppgtt->base.start, ppgtt->base.total); + ret =gen8_alloc_va_range (&ppgtt->base, start, size);if (ret) {unmap_and_free_pt (ppgtt->scratch_pd, ppgtt->base.dev);return ret; } -/* -* 2. Create DMA mappings for the page directories and page tables. -*/ -for (i =0 ; i < max_pdp; i++) { - ret =gen8_ppgtt_setup_page_directories (ppgtt, i); -if (ret) - goto bail; + start =0 ; + size = orig_length; -for (j =0 ; j <GEN8_PDES_PER_PAGE ; j++) { - ret =gen8_ppgtt_setup_page_tables (ppgtt, i, j); -if (ret) - goto bail; - } - } - -/* -* 3. Map all the page directory entires to point to the page tables -* we've allocated. -* -* For now, the PPGTT helper functions all require that the PDEs are -* plugged in correctly. So we do that now/here. For aliasing PPGTT, we -* will never need to touch the PDEs again. -*/ -for (i =0 ; i < max_pdp; i++) { -struct i915_page_directory_entry *pd = ppgtt->pdp.page_directory[i]; -gen8_ppgtt_pde_t *pd_vaddr; - pd_vaddr =kmap_atomic (ppgtt->pdp.page_directory[i]->page); -for (j =0 ; j <GEN8_PDES_PER_PAGE ; j++) { -struct i915_page_table_entry *pt = pd->page_tables[j]; -dma_addr_t addr = pt->daddr; - pd_vaddr[j] =gen8_pde_encode (ppgtt->base.dev, addr, -I915_CACHE_LLC ); - } -if (!HAS_LLC (ppgtt->base.dev)) -drm_clflush_virt_range (pd_vaddr,PAGE_SIZE ); -kunmap_atomic (pd_vaddr); - } - - ppgtt->switch_mm = gen8_mm_switch; - ppgtt->base.clear_range = gen8_ppgtt_clear_range; - ppgtt->base.insert_entries = gen8_ppgtt_insert_entries; - ppgtt->base.cleanup = gen8_ppgtt_cleanup; +gen8_for_each_pdpe (pd, &ppgtt->pdp, start, size, temp, pdpe) +gen8_map_pagetable_range (pd, start, size, ppgtt->base.dev); ppgtt->base.clear_range (&ppgtt->base,0 , ppgtt->base.total, true);return 0 ; - -bail: -gen8_ppgtt_unmap_pages (ppgtt); -gen8_ppgtt_free (ppgtt); -return ret; }static void gen6_dump_ppgtt (struct i915_hw_ppgtt *ppgtt,struct seq_file *m)
@@ -1265,7 +1226,7 @@ static void gen6_ppgtt_free(struct i915_hw_ppgtt *ppgtt) }unmap_and_free_pt (ppgtt->scratch_pt, ppgtt->base.dev); -unmap_and_free_pd (&ppgtt->pd); +unmap_and_free_pd (&ppgtt->pd, ppgtt->base.dev); }static void gen6_ppgtt_cleanup (struct i915_address_space *vm)
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH v4 22/24 drm/i915/bdw: begin bitmap tracking
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
Like with gen6/7, we can enable bitmap tracking with all the preallocations to make sure things actually don't blow up.
v2: Rebased to match changes from previous patches.
v3: Without teardown logic, rely on used_pdpes and used_pdes when freeing page tables.
--- drivers/gpu/drm/i915/i915_gem_gtt.c | 75 ++++++++++++++++++++++++++++--------- drivers/gpu/drm/i915/i915_gem_gtt.h | 24 ++++++++++++ 2 files changed, 81 insertions(+), 18 deletions(-)
drivers/gpu/drm/i915/i915_gem_gtt.c +57 -18
@@ -419,6 +419,7 @@ static void unmap_and_free_pd(struct i915_page_directory_entry *pd,if (pd->page) {i915_dma_unmap_single (pd, dev);__free_page (pd->page); +kfree (pd->used_pdes);kfree (pd); } }
@@ -426,26 +427,35 @@ static void unmap_and_free_pd(struct i915_page_directory_entry *pd,static struct i915_page_directory_entry *alloc_pd_single (struct drm_device *dev) {struct i915_page_directory_entry *pd; -int ret; +int ret = -ENOMEM ; pd =kzalloc (sizeof (*pd),GFP_KERNEL );if (!pd)return ERR_PTR (-ENOMEM ); + pd->used_pdes =kcalloc (BITS_TO_LONGS (GEN8_PDES_PER_PAGE ), +sizeof (*pd->used_pdes),GFP_KERNEL ); +if (!pd->used_pdes) + goto free_pd; + pd->page =alloc_page (GFP_KERNEL | __GFP_ZERO); -if (!pd->page) { -kfree (pd); -return ERR_PTR (-ENOMEM ); - } +if (!pd->page) + goto free_bitmap; ret =i915_dma_map_px_single (pd, dev); -if (ret) { -__free_page (pd->page); -kfree (pd); -return ERR_PTR (ret); - } +if (ret) + goto free_page;return pd; + +free_page: +__free_page (pd->page); +free_bitmap: +kfree (pd->used_pdes); +free_pd: +kfree (pd); + +return ERR_PTR (ret); }/* Broadwell Page Directory Pointer Descriptors */
@@ -636,7 +646,7 @@ static void gen8_free_page_tables(struct i915_page_directory_entry *pd, struct dif (!pd->page)return ; -for (i =0 ; i <GEN8_PDES_PER_PAGE ; i++) { +for_each_set_bit (i, pd->used_pdes,GEN8_PDES_PER_PAGE ) {if (WARN_ON (!pd->page_tables[i]))continue ;
@@ -650,15 +660,18 @@ static void gen8_ppgtt_unmap_pages(struct i915_hw_ppgtt *ppgtt)struct pci_dev *hwdev = ppgtt->base.dev->pdev;int i, j; -for (i =0 ; i <GEN8_LEGACY_PDPES ; i++) { -if (!ppgtt->pdp.page_directory[i]->daddr) +for_each_set_bit (i, ppgtt->pdp.used_pdpes,GEN8_LEGACY_PDPES ) { +struct i915_page_directory_entry *pd; + +if (WARN_ON (!ppgtt->pdp.page_directory[i]))continue ; -pci_unmap_page (hwdev, ppgtt->pdp.page_directory[i]->daddr,PAGE_SIZE , -PCI_DMA_BIDIRECTIONAL ); + pd = ppgtt->pdp.page_directory[i]; +if (!pd->daddr) +pci_unmap_page (hwdev, pd->daddr,PAGE_SIZE , +PCI_DMA_BIDIRECTIONAL ); -for (j =0 ; j <GEN8_PDES_PER_PAGE ; j++) { -struct i915_page_directory_entry *pd = ppgtt->pdp.page_directory[i]; +for_each_set_bit (j, pd->used_pdes,GEN8_PDES_PER_PAGE ) {struct i915_page_table_entry *pt;dma_addr_t addr;
@@ -679,7 +692,7 @@ static void gen8_ppgtt_free(struct i915_hw_ppgtt *ppgtt) {int i; -for (i =0 ; i <GEN8_LEGACY_PDPES ; i++) { +for_each_set_bit (i, ppgtt->pdp.used_pdpes,GEN8_LEGACY_PDPES ) {if (WARN_ON (!ppgtt->pdp.page_directory[i]))continue ;
@@ -722,6 +735,7 @@ unwind_out:return -ENOMEM ; } +/* bitmap of new page_directories */ static int gen8_ppgtt_alloc_page_directories(struct i915_page_directory_pointer_entry *pdp, uint64_t start, uint64_t length,
@@ -737,6 +751,7 @@ static int gen8_ppgtt_alloc_page_directories(struct i915_page_directory_pointer_gen8_for_each_pdpe (unused, pdp, start, length, temp, pdpe) {BUG_ON (unused); pdp->page_directory[pdpe] =alloc_pd_single (dev); +if (IS_ERR (pdp->page_directory[pdpe])) goto unwind_out; }
@@ -757,10 +772,13 @@ static int gen8_alloc_va_range(struct i915_address_space *vm,struct i915_hw_ppgtt *ppgtt = container_of(vm, struct i915_hw_ppgtt, base);struct i915_page_directory_entry *pd; +const uint64_t orig_start = start; +const uint64_t orig_length = length;uint64_t temp;uint32_t pdpe;int ret; +/* Do the allocations first so we can easily bail out */ ret =gen8_ppgtt_alloc_page_directories (&ppgtt->pdp, start, length, ppgtt->base.dev);if (ret)
@@ -773,6 +791,27 @@ static int gen8_alloc_va_range(struct i915_address_space *vm, goto err_out; } +/* Now mark everything we've touched as used. This doesn't allow for +* robust error checking, but it makes the code a hell of a lot simpler. +*/ + start = orig_start; + length = orig_length; + +gen8_for_each_pdpe (pd, &ppgtt->pdp, start, length, temp, pdpe) { +struct i915_page_table_entry *pt; +uint64_t pd_len =gen8_clamp_pd (start, length); +uint64_t pd_start = start; +uint32_t pde; + +gen8_for_each_pde (pt, &ppgtt->pd, pd_start, pd_len, temp, pde) { +bitmap_set (pd->page_tables[pde]->used_ptes, +gen8_pte_index (start), +gen8_pte_count (start, length)); +set_bit (pde, pd->used_pdes); + } +set_bit (pdpe, ppgtt->pdp.used_pdpes); + } +return 0 ; err_out:
drivers/gpu/drm/i915/i915_gem_gtt.h +24 -0
@@ -205,11 +205,13 @@ struct i915_page_directory_entry {dma_addr_t daddr; }; +unsigned long *used_pdes;struct i915_page_table_entry *page_tables[GEN6_PPGTT_PD_ENTRIES ];/* PDEs */ };struct i915_page_directory_pointer_entry {/* struct page *page; */ + DECLARE_BITMAP(used_pdpes, GEN8_LEGACY_PDPES);struct i915_page_directory_entry *page_directory[GEN8_LEGACY_PDPES ]; };
@@ -436,6 +438,28 @@ static inline uint32_t gen8_pml4e_index(uint64_t address)BUG ();/* For 64B */ } +static inline size_t gen8_pte_count (uint64_t addr,uint64_t length) +{ +return i915_pte_count (addr, length,GEN8_PDE_SHIFT ); +} + +static inline size_t gen8_pde_count (uint64_t addr,uint64_t length) +{ +const uint32_t pdp_shift =GEN8_PDE_SHIFT +9 ; +const uint64_t mask = ~((1 << pdp_shift) -1 ); +uint64_t end; + +BUG_ON (length ==0 ); +BUG_ON (offset_in_page (addr|length)); + + end = addr + length; + +if ((addr & mask) != (end & mask)) +return GEN8_PDES_PER_PAGE -i915_pde_index (addr,GEN8_PDE_SHIFT ); + +return i915_pde_index (end,GEN8_PDE_SHIFT ) -i915_pde_index (addr,GEN8_PDE_SHIFT ); +} +int i915_gem_gtt_init (struct drm_device *dev);void i915_gem_init_global_gtt (struct drm_device *dev);void i915_global_gtt_cleanup (struct drm_device *dev);
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH v4 23/24 drm/i915/bdw: Dynamic page table allocations
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
This finishes off the dynamic page tables allocations, in the legacy 3 level style that already exists. Most everything has already been setup to this point, the patch finishes off the enabling by setting the appropriate function pointers.
v2: Update aliasing/true ppgtt allocate/teardown/clear functions for gen 6 & 7.
v3: Rebase.
v4: Remove BUG() from ppgtt_unbind_vma, but keep checking that either teardown_va_range or clear_range functions exist (Daniel).
v5: Similar to gen6, in init, gen8_ppgtt_clear_range call is only needed for aliasing ppgtt. Zombie tracking was originally added for teardown function and is no longer required.
--- drivers/gpu/drm/i915/i915_gem_gtt.c | 295 +++++++++++++++++++++++++++++------- 1 file changed, 242 insertions(+), 53 deletions(-)
drivers/gpu/drm/i915/i915_gem_gtt.c +242 -53
@@ -609,7 +609,7 @@ static void gen8_ppgtt_insert_entries(struct i915_address_space *vm, } } -static void __gen8_do_map_pt (gen8_ppgtt_pde_t *pde, +static void __gen8_do_map_pt (gen8_ppgtt_pde_t *const pde,struct i915_page_table_entry *pt,struct drm_device *dev) {
@@ -626,7 +626,7 @@ static void gen8_map_pagetable_range(struct i915_page_directory_entry *pd,uint64_t length, struct drm_device *dev) { - gen8_ppgtt_pde_t *page_directory = kmap_atomic(pd->page); + gen8_ppgtt_pde_t * const page_directory = kmap_atomic(pd->page);struct i915_page_table_entry *pt;uint64_t temp, pde;
@@ -710,67 +710,173 @@ static void gen8_ppgtt_cleanup(struct i915_address_space *vm)gen8_ppgtt_free (ppgtt); } -static int gen8_ppgtt_alloc_pagetabs (struct i915_page_directory_entry *pd, +/** +* gen8_ppgtt_alloc_pagetabs() - Allocate page tables for VA range. +* @ppgtt: Master ppgtt structure. +* @pd: Page directory for this address range. +* @start: Starting virtual address to begin allocations. +* @length Size of the allocations. +* @new_pts: Bitmap set by function with new allocations. Likely used by the +* caller to free on error. +* +* Allocate the required number of page tables. Extremely similar to +* gen8_ppgtt_alloc_page_directories(). The main difference is here we are limited by +* the page directory boundary (instead of the page directory pointer). That +* boundary is 1GB virtual. Therefore, unlike gen8_ppgtt_alloc_page_directories(), it is +* possible, and likely that the caller will need to use multiple calls of this +* function to achieve the appropriate allocation. +* +* Return: 0 if success; negative error code otherwise. +*/ +static int gen8_ppgtt_alloc_pagetabs (struct i915_hw_ppgtt *ppgtt, +struct i915_page_directory_entry *pd,uint64_t start,uint64_t length, -struct drm_device *dev) +unsigned long *new_pts) { -struct i915_page_table_entry *unused; +struct i915_page_table_entry *pt;uint64_t temp;uint32_t pde; -gen8_for_each_pde (unused, pd, start, length, temp, pde) { -BUG_ON (unused); - pd->page_tables[pde] =alloc_pt_single (dev); -if (IS_ERR (pd->page_tables[pde])) +gen8_for_each_pde (pt, pd, start, length, temp, pde) { +/* Don't reallocate page tables */ +if (pt) { +/* Scratch is never allocated this way */ +WARN_ON (pt->scratch); +continue ; + } + + pt =alloc_pt_single (ppgtt->base.dev); +if (IS_ERR (pt)) goto unwind_out; + + pd->page_tables[pde] = pt; +set_bit (pde, new_pts); }return 0 ; unwind_out: -while (pde--) -unmap_and_free_pt (pd->page_tables[pde], dev); +for_each_set_bit (pde, new_pts,GEN8_PDES_PER_PAGE ) +unmap_and_free_pt (pd->page_tables[pde], ppgtt->base.dev);return -ENOMEM ; } -/* bitmap of new page_directories */ -static int gen8_ppgtt_alloc_page_directories (struct i915_page_directory_pointer_entry *pdp, +/** +* gen8_ppgtt_alloc_page_directories() - Allocate page directories for VA range. +* @ppgtt: Master ppgtt structure. +* @pdp: Page directory pointer for this address range. +* @start: Starting virtual address to begin allocations. +* @length Size of the allocations. +* @new_pds Bitmap set by function with new allocations. Likely used by the +* caller to free on error. +* +* Allocate the required number of page directories starting at the pde index of +* @start, and ending at the pde index @start + @length. This function will skip +* over already allocated page directories within the range, and only allocate +* new ones, setting the appropriate pointer within the pdp as well as the +* correct position in the bitmap @new_pds. +* +* The function will only allocate the pages within the range for a give page +* directory pointer. In other words, if @start + @length straddles a virtually +* addressed PDP boundary (512GB for 4k pages), there will be more allocations +* required by the caller, This is not currently possible, and the BUG in the +* code will prevent it. +* +* Return: 0 if success; negative error code otherwise. +*/ +static int gen8_ppgtt_alloc_page_directories (struct i915_hw_ppgtt *ppgtt, +struct i915_page_directory_pointer_entry *pdp,uint64_t start,uint64_t length, -struct drm_device *dev) +unsigned long *new_pds) { -struct i915_page_directory_entry *unused; +struct i915_page_directory_entry *pd;uint64_t temp;uint32_t pdpe; +BUG_ON (!bitmap_empty (new_pds,GEN8_LEGACY_PDPES )); +/* FIXME: PPGTT container_of won't work for 64b */ BUG_ON ((start + length) >0x800000000ULL ); -gen8_for_each_pdpe (unused, pdp, start, length, temp, pdpe) { -BUG_ON (unused); - pdp->page_directory[pdpe] =alloc_pd_single (dev); +gen8_for_each_pdpe (pd, pdp, start, length, temp, pdpe) { +if (pd) +continue ; -if (IS_ERR (pdp->page_directory[pdpe])) + pd =alloc_pd_single (ppgtt->base.dev); +if (IS_ERR (pd)) goto unwind_out; + + pdp->page_directory[pdpe] = pd; +set_bit (pdpe, new_pds); }return 0 ; unwind_out: -while (pdpe--) -unmap_and_free_pd (pdp->page_directory[pdpe], dev); +for_each_set_bit (pdpe, new_pds,GEN8_LEGACY_PDPES ) +unmap_and_free_pd (pdp->page_directory[pdpe], ppgtt->base.dev);return -ENOMEM ; } +static inline void +free_gen8_temp_bitmaps (unsigned long *new_pds,unsigned long **new_pts) +{ +int i; + +for (i =0 ; i <GEN8_LEGACY_PDPES ; i++) +kfree (new_pts[i]); +kfree (new_pts); +kfree (new_pds); +} + +/* Fills in the page directory bitmap, ant the array of page tables bitmap. Both +* of these are based on the number of PDPEs in the system. +*/ +int __must_check alloc_gen8_temp_bitmaps (unsigned long **new_pds, +unsigned long ***new_pts) +{ +int i; +unsigned long *pds; +unsigned long **pts; + + pds =kcalloc (BITS_TO_LONGS (GEN8_LEGACY_PDPES ),sizeof (unsigned long ),GFP_KERNEL ); +if (!pds) +return -ENOMEM ; + + pts =kcalloc (GEN8_PDES_PER_PAGE ,sizeof (unsigned long *),GFP_KERNEL ); +if (!pts) { +kfree (pds); +return -ENOMEM ; + } + +for (i =0 ; i <GEN8_LEGACY_PDPES ; i++) { + pts[i] =kcalloc (BITS_TO_LONGS (GEN8_PDES_PER_PAGE ), +sizeof (unsigned long ),GFP_KERNEL ); +if (!pts[i]) + goto err_out; + } + + *new_pds = pds; + *new_pts = (unsigned long **)pts; + +return 0 ; + +err_out: +free_gen8_temp_bitmaps (pds, pts); +return -ENOMEM ; +} +static int gen8_alloc_va_range (struct i915_address_space *vm,uint64_t start,uint64_t length) {struct i915_hw_ppgtt *ppgtt = container_of(vm, struct i915_hw_ppgtt, base); +unsigned long *new_page_dirs, **new_page_tables;struct i915_page_directory_entry *pd;const uint64_t orig_start = start;const uint64_t orig_length = length;
@@ -778,44 +884,96 @@ static int gen8_alloc_va_range(struct i915_address_space *vm,uint32_t pdpe;int ret; -/* Do the allocations first so we can easily bail out */ - ret =gen8_ppgtt_alloc_page_directories (&ppgtt->pdp, start, length, - ppgtt->base.dev); +#ifndef CONFIG_64BIT +/* Disallow 64b address on 32b platforms. Nothing is wrong with doing +* this in hardware, but a lot of the drm code is not prepared to handle +* 64b offset on 32b platforms. +* This will be addressed when 48b PPGTT is added */ +if (start + length >0x100000000ULL ) +return -E2BIG ; +#endif + +/* Wrap is never okay since we can only represent 48b, and we don't +* actually use the other side of the canonical address space. +*/ +if (WARN_ON (start + length < start)) +return -ERANGE ; + + ret =alloc_gen8_temp_bitmaps (&new_page_dirs, &new_page_tables);if (ret)return ret; +/* Do the allocations first so we can easily bail out */ + ret =gen8_ppgtt_alloc_page_directories (ppgtt, &ppgtt->pdp, start, length, + new_page_dirs); +if (ret) { +free_gen8_temp_bitmaps (new_page_dirs, new_page_tables); +return ret; + } + +/* For every page directory referenced, allocate page tables */ gen8_for_each_pdpe (pd, &ppgtt->pdp, start, length, temp, pdpe) { - ret =gen8_ppgtt_alloc_pagetabs (pd, start, length, - ppgtt->base.dev); +bitmap_zero (new_page_tables[pdpe],GEN8_PDES_PER_PAGE ); + ret =gen8_ppgtt_alloc_pagetabs (ppgtt, pd, start, length, + new_page_tables[pdpe]);if (ret) goto err_out; } -/* Now mark everything we've touched as used. This doesn't allow for -* robust error checking, but it makes the code a hell of a lot simpler. -*/ start = orig_start; length = orig_length; +/* Allocations have completed successfully, so set the bitmaps, and do +* the mappings. */ gen8_for_each_pdpe (pd, &ppgtt->pdp, start, length, temp, pdpe) { +gen8_ppgtt_pde_t *const page_directory =kmap_atomic (pd->page);struct i915_page_table_entry *pt;uint64_t pd_len =gen8_clamp_pd (start, length);uint64_t pd_start = start;uint32_t pde; -gen8_for_each_pde (pt, &ppgtt->pd, pd_start, pd_len, temp, pde) { -bitmap_set (pd->page_tables[pde]->used_ptes, -gen8_pte_index (start), -gen8_pte_count (start, length)); +/* Every pd should be allocated, we just did that above. */ +BUG_ON (!pd); + +gen8_for_each_pde (pt, pd, pd_start, pd_len, temp, pde) { +/* Same reasoning as pd */ +BUG_ON (!pt); +BUG_ON (!pd_len); +BUG_ON (!gen8_pte_count (pd_start, pd_len)); + +/* Set our used ptes within the page table */ +bitmap_set (pt->used_ptes, +gen8_pte_index (pd_start), +gen8_pte_count (pd_start, pd_len)); + +/* Our pde is now pointing to the pagetable, pt */ set_bit (pde, pd->used_pdes); + +/* Map the PDE to the page table */ +__gen8_do_map_pt (page_directory + pde, pt, vm->dev); + +/* NB: We haven't yet mapped ptes to pages. At this +* point we're still relying on insert_entries() */ } + +if (!HAS_LLC (vm->dev)) +drm_clflush_virt_range (page_directory,PAGE_SIZE ); + +kunmap_atomic (page_directory); +set_bit (pdpe, ppgtt->pdp.used_pdpes); } +free_gen8_temp_bitmaps (new_page_dirs, new_page_tables);return 0 ; err_out:gen8_ppgtt_free (ppgtt); + +for_each_set_bit (pdpe, new_page_dirs,GEN8_LEGACY_PDPES ) +unmap_and_free_pd (ppgtt->pdp.page_directory[pdpe], vm->dev); + +free_gen8_temp_bitmaps (new_page_dirs, new_page_tables);return ret; }
@@ -826,38 +984,67 @@ err_out:* space. * */ -static int gen8_ppgtt_init (struct i915_hw_ppgtt *ppgtt,uint64_t size) +static int gen8_ppgtt_init_common (struct i915_hw_ppgtt *ppgtt,uint64_t size) { -struct i915_page_directory_entry *pd; -uint64_t temp, start =0 ; -const uint64_t orig_length = size; -uint32_t pdpe; -int ret; + ppgtt->scratch_pd =alloc_pt_scratch (ppgtt->base.dev); +if (IS_ERR (ppgtt->scratch_pd)) +return PTR_ERR (ppgtt->scratch_pd); ppgtt->base.start =0 ; ppgtt->base.total = size; - ppgtt->base.clear_range = gen8_ppgtt_clear_range; - ppgtt->base.insert_entries = gen8_ppgtt_insert_entries; ppgtt->base.cleanup = gen8_ppgtt_cleanup; + ppgtt->base.insert_entries = gen8_ppgtt_insert_entries; + ppgtt->switch_mm = gen8_mm_switch; - ppgtt->scratch_pd =alloc_pt_scratch (ppgtt->base.dev); -if (IS_ERR (ppgtt->scratch_pd)) -return PTR_ERR (ppgtt->scratch_pd); +return 0 ; +} +static int gen8_aliasing_ppgtt_init (struct i915_hw_ppgtt *ppgtt) +{ +struct drm_device *dev = ppgtt->base.dev; +struct drm_i915_private *dev_priv = dev->dev_private; +struct i915_page_directory_entry *pd; +uint64_t temp, start =0 , size = dev_priv->gtt.base.total; +uint32_t pdpe; +int ret; + + ret =gen8_ppgtt_init_common (ppgtt, dev_priv->gtt.base.total); +if (ret) +return ret; + +/* Aliasing PPGTT has to always work and be mapped because of the way we +* use RESTORE_INHIBIT in the context switch. This will be fixed +* eventually. */ ret =gen8_alloc_va_range (&ppgtt->base, start, size);if (ret) {unmap_and_free_pt (ppgtt->scratch_pd, ppgtt->base.dev);return ret; } - start =0 ; - size = orig_length; -gen8_for_each_pdpe (pd, &ppgtt->pdp, start, size, temp, pdpe)gen8_map_pagetable_range (pd, start, size, ppgtt->base.dev); + ppgtt->base.allocate_va_range =NULL ; + ppgtt->base.clear_range = gen8_ppgtt_clear_range; ppgtt->base.clear_range (&ppgtt->base,0 , ppgtt->base.total, true); + +return 0 ; +} + +static int gen8_ppgtt_init (struct i915_hw_ppgtt *ppgtt) +{ +struct drm_device *dev = ppgtt->base.dev; +struct drm_i915_private *dev_priv = dev->dev_private; +int ret; + + ret =gen8_ppgtt_init_common (ppgtt, dev_priv->gtt.base.total); +if (ret) +return ret; + + ppgtt->base.allocate_va_range = gen8_alloc_va_range; + ppgtt->base.clear_range = gen8_ppgtt_clear_range; +return 0 ; }
@@ -1380,7 +1567,7 @@ static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt, bool aliasing) } } - ppgtt->base.allocate_va_range = gen6_alloc_va_range; + ppgtt->base.allocate_va_range = aliasing ?NULL : gen6_alloc_va_range; ppgtt->base.clear_range = gen6_ppgtt_clear_range; ppgtt->base.insert_entries = gen6_ppgtt_insert_entries; ppgtt->base.cleanup = gen6_ppgtt_cleanup;
@@ -1421,8 +1608,10 @@ static int __hw_ppgtt_init(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt,if (INTEL_INFO (dev)->gen <8 )return gen6_ppgtt_init (ppgtt, aliasing); +else if (aliasing) +return gen8_aliasing_ppgtt_init (ppgtt);else -return gen8_ppgtt_init (ppgtt, dev_priv->gtt.base.total); +return gen8_ppgtt_init (ppgtt); }int i915_ppgtt_init (struct drm_device *dev,struct i915_hw_ppgtt *ppgtt) {
@@ -1531,10 +1720,10 @@ ppgtt_bind_vma(struct i915_vma *vma,static void ppgtt_unbind_vma (struct i915_vma *vma) { - vma->vm->clear_range (vma->vm, - vma->node.start, - vma->obj->base.size, - true); + vma->vm->clear_range (vma->vm, + vma->node.start, + vma->obj->base.size, + true); }extern int intel_iommu_gfx_mapped;
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH v4 24/24 drm/i915/bdw: Support dynamic pdp updates in lrc mode
In reply to Michel ThierryLogic ring contexts need to know the PDPs when they are populated. With dynamic page table allocations, these PDPs may not exist yet.
Check if PDPs have been allocated and use the scratch page if they do not exist yet.
Before submission, update the PDPs in the logic ring context as PDPs have been allocated.
v2: Renamed commit title (Daniel)
--- drivers/gpu/drm/i915/intel_lrc.c | 80 +++++++++++++++++++++++++++++++++++----- 1 file changed, 70 insertions(+), 10 deletions(-)
drivers/gpu/drm/i915/intel_lrc.c +70 -10
@@ -358,6 +358,7 @@ static void execlists_elsp_write(struct intel_engine_cs *ring,static int execlists_update_context (struct drm_i915_gem_object *ctx_obj,struct drm_i915_gem_object *ring_obj, +struct i915_hw_ppgtt *ppgtt,u32 tail) {struct page *page;
@@ -369,6 +370,40 @@ static int execlists_update_context(struct drm_i915_gem_object *ctx_obj, reg_state[CTX_RING_TAIL +1 ] = tail; reg_state[CTX_RING_BUFFER_START +1 ] =i915_gem_obj_ggtt_offset (ring_obj); +/* True PPGTT with dynamic page allocation: update PDP registers and +* point the unallocated PDPs to the scratch page +*/ +if (ppgtt) { +if (test_bit (3 , ppgtt->pdp.used_pdpes)) { + reg_state[CTX_PDP3_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[3 ]->daddr); + reg_state[CTX_PDP3_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[3 ]->daddr); + }else { + reg_state[CTX_PDP3_UDW +1 ] =upper_32_bits (ppgtt->scratch_pd->daddr); + reg_state[CTX_PDP3_LDW +1 ] =lower_32_bits (ppgtt->scratch_pd->daddr); + } +if (test_bit (2 , ppgtt->pdp.used_pdpes)) { + reg_state[CTX_PDP2_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[2 ]->daddr); + reg_state[CTX_PDP2_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[2 ]->daddr); + }else { + reg_state[CTX_PDP2_UDW +1 ] =upper_32_bits (ppgtt->scratch_pd->daddr); + reg_state[CTX_PDP2_LDW +1 ] =lower_32_bits (ppgtt->scratch_pd->daddr); + } +if (test_bit (1 , ppgtt->pdp.used_pdpes)) { + reg_state[CTX_PDP1_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[1 ]->daddr); + reg_state[CTX_PDP1_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[1 ]->daddr); + }else { + reg_state[CTX_PDP1_UDW +1 ] =upper_32_bits (ppgtt->scratch_pd->daddr); + reg_state[CTX_PDP1_LDW +1 ] =lower_32_bits (ppgtt->scratch_pd->daddr); + } +if (test_bit (0 , ppgtt->pdp.used_pdpes)) { + reg_state[CTX_PDP0_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[0 ]->daddr); + reg_state[CTX_PDP0_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[0 ]->daddr); + }else { + reg_state[CTX_PDP0_UDW +1 ] =upper_32_bits (ppgtt->scratch_pd->daddr); + reg_state[CTX_PDP0_LDW +1 ] =lower_32_bits (ppgtt->scratch_pd->daddr); + } + } +kunmap_atomic (reg_state);return 0 ;
@@ -387,7 +422,7 @@ static void execlists_submit_contexts(struct intel_engine_cs *ring,WARN_ON (!i915_gem_obj_is_pinned (ctx_obj0));WARN_ON (!i915_gem_obj_is_pinned (ringbuf0->obj)); -execlists_update_context (ctx_obj0, ringbuf0->obj, tail0); +execlists_update_context (ctx_obj0, ringbuf0->obj, to0->ppgtt, tail0);if (to1) { ringbuf1 = to1->engine[ring->id].ringbuf;
@@ -396,7 +431,7 @@ static void execlists_submit_contexts(struct intel_engine_cs *ring,WARN_ON (!i915_gem_obj_is_pinned (ctx_obj1));WARN_ON (!i915_gem_obj_is_pinned (ringbuf1->obj)); -execlists_update_context (ctx_obj1, ringbuf1->obj, tail1); +execlists_update_context (ctx_obj1, ringbuf1->obj, to1->ppgtt, tail1); }execlists_elsp_write (ring, ctx_obj0, ctx_obj1);
@@ -1731,14 +1766,39 @@ populate_lr_context(struct intel_context *ctx, struct drm_i915_gem_object *ctx_o reg_state[CTX_PDP1_LDW ] =GEN8_RING_PDP_LDW (ring,1 ); reg_state[CTX_PDP0_UDW ] =GEN8_RING_PDP_UDW (ring,0 ); reg_state[CTX_PDP0_LDW ] =GEN8_RING_PDP_LDW (ring,0 ); - reg_state[CTX_PDP3_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[3 ]->daddr); - reg_state[CTX_PDP3_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[3 ]->daddr); - reg_state[CTX_PDP2_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[2 ]->daddr); - reg_state[CTX_PDP2_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[2 ]->daddr); - reg_state[CTX_PDP1_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[1 ]->daddr); - reg_state[CTX_PDP1_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[1 ]->daddr); - reg_state[CTX_PDP0_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[0 ]->daddr); - reg_state[CTX_PDP0_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[0 ]->daddr); + +/* With dynamic page allocation, PDPs may not be allocated at this point, +* Point the unallocated PDPs to the scratch page +*/ +if (test_bit (3 , ppgtt->pdp.used_pdpes)) { + reg_state[CTX_PDP3_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[3 ]->daddr); + reg_state[CTX_PDP3_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[3 ]->daddr); + }else { + reg_state[CTX_PDP3_UDW +1 ] =upper_32_bits (ppgtt->scratch_pd->daddr); + reg_state[CTX_PDP3_LDW +1 ] =lower_32_bits (ppgtt->scratch_pd->daddr); + } +if (test_bit (2 , ppgtt->pdp.used_pdpes)) { + reg_state[CTX_PDP2_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[2 ]->daddr); + reg_state[CTX_PDP2_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[2 ]->daddr); + }else { + reg_state[CTX_PDP2_UDW +1 ] =upper_32_bits (ppgtt->scratch_pd->daddr); + reg_state[CTX_PDP2_LDW +1 ] =lower_32_bits (ppgtt->scratch_pd->daddr); + } +if (test_bit (1 , ppgtt->pdp.used_pdpes)) { + reg_state[CTX_PDP1_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[1 ]->daddr); + reg_state[CTX_PDP1_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[1 ]->daddr); + }else { + reg_state[CTX_PDP1_UDW +1 ] =upper_32_bits (ppgtt->scratch_pd->daddr); + reg_state[CTX_PDP1_LDW +1 ] =lower_32_bits (ppgtt->scratch_pd->daddr); + } +if (test_bit (0 , ppgtt->pdp.used_pdpes)) { + reg_state[CTX_PDP0_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[0 ]->daddr); + reg_state[CTX_PDP0_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[0 ]->daddr); + }else { + reg_state[CTX_PDP0_UDW +1 ] =upper_32_bits (ppgtt->scratch_pd->daddr); + reg_state[CTX_PDP0_LDW +1 ] =lower_32_bits (ppgtt->scratch_pd->daddr); + } +if (ring->id ==RCS ) { reg_state[CTX_LRI_HEADER_2 ] =MI_LOAD_REGISTER_IMM (1 ); reg_state[CTX_R_PWR_CLK_STATE ] =0x20c8 ;
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
v5 cover PPGTT dynamic page allocations and 48b addressing
In reply to Michel ThierryThis patchset starts addressing comments from v4 by Mika, and also has been rebased on top of nightly.
For GEN8, it has also been extended to work in logical ring submission (lrc) mode, as it will be the preferred mode of operation.
I also tried to update the lrc code at the same time the ppgtt refactoring occurred, leaving only one patch that is exclusively for lrc.
I'm also now including the required patches for PPGTT with 48b addressing. In order expand the GPU address space, a 4th level translation is added, the Page Map Level 4 (PML4). This PML4 has 256 PML4 Entries (PML4E), PML4[0-255], each pointing to a PDP.
For now, this feature will only be available in BDW, in LRC submission mode (execlists) and when i915.enable_ppgtt=3 is set.
Also note that this expanded address space is only available for full PPGTT, aliasing PPGTT remains 32b.
This list can be seen in 3 parts:
[01-10] Add page table allocation for GEN6/GEN7 [11-20] Enable dynamic allocation in GEN8,for both legacy and execlist submission modes.
[21-32] PML4 support in BDW.
Ben Widawsky (26):
drm/i915: page table abstractions
drm/i915: Complete page table structures
drm/i915: Create page table allocators
drm/i915: Track GEN6 page table usage
drm/i915: Extract context switch skip and pd load logic drm/i915: Track page table reload need
drm/i915: Initialize all contexts
drm/i915: Finish gen6/7 dynamic page table allocation drm/i915/bdw: Use dynamic allocation idioms on free drm/i915/bdw: page directories rework allocation drm/i915/bdw: pagetable allocation rework
drm/i915/bdw: Update pdp switch and point unused PDPs to scratch page drm/i915: num_pd_pages/num_pd_entries isn't useful drm/i915: Extract PPGTT param from page_directory alloc drm/i915/bdw: Split out mappings
drm/i915/bdw: begin bitmap tracking
drm/i915/bdw: Dynamic page table allocations
drm/i915/bdw: Make pdp allocation more dynamic drm/i915/bdw: Abstract PDP usage
drm/i915/bdw: Add dynamic page trace events
drm/i915/bdw: Add ppgtt info for dynamic pages drm/i915/bdw: implement alloc/free for 4lvl
drm/i915/bdw: Add 4 level switching infrastructure drm/i915/bdw: Generalize PTE writing for GEN8 PPGTT drm/i915: Plumb sg_iter through va allocation ->maps drm/i915: Expand error state's address width to 64b
Michel Thierry (6):
drm/i915: Plumb drm_device through page tables operations drm/i915: Add dynamic page trace events
drm/i915/bdw: Support dynamic pdp updates in lrc mode drm/i915/bdw: Support 64 bit PPGTT in lrc mode drm/i915/bdw: Add 4 level support in insert_entries and clear_range drm/i915/bdw: Flip the 48b switch
drivers/gpu/drm/i915/i915_debugfs.c | 26 +- drivers/gpu/drm/i915/i915_drv.h | 11 +- drivers/gpu/drm/i915/i915_gem.c | 11 + drivers/gpu/drm/i915/i915_gem_context.c | 64 +- drivers/gpu/drm/i915/i915_gem_execbuffer.c | 11 + drivers/gpu/drm/i915/i915_gem_gtt.c | 1534 ++++++++++++++++++++++------ drivers/gpu/drm/i915/i915_gem_gtt.h | 248 ++++- drivers/gpu/drm/i915/i915_gpu_error.c | 17 +- drivers/gpu/drm/i915/i915_params.c | 2 +- drivers/gpu/drm/i915/i915_reg.h | 1 + drivers/gpu/drm/i915/i915_trace.h | 111 ++ drivers/gpu/drm/i915/intel_lrc.c | 149 ++- 12 files changed, 1786 insertions(+), 399 deletions(-)
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH v5 1/32 drm/i915: page table abstractions
From: Ben Widawsky <[email protected]>
When we move to dynamic page allocation, keeping page_directory and pagetabs as separate structures will help to break actions into simpler tasks.
To help transition the code nicely there is some wasted space in gen6/7. This will be ameliorated shortly.
Following the x86 pagetable terminology:
PDPE = struct i915_page_directory_pointer_entry. PDE = struct i915_page_directory_entry [page_directory]. PTE = struct i915_page_table_entry [page_tables].
v2: fixed mismatches after clean-up/rebase.
v3: Clarify the names of the multiple levels of page tables (Daniel)
v4: Addressing Mika's review comments. s/gen8_free_page_directories/gen8_free_page_directory and free the page tables for the directory there.
In gen8_ppgtt_allocate_page_directories, do not leak previously allocated pt in case the page_directory alloc fails.
Update error return handling in gen8_ppgtt_alloc.
--- drivers/gpu/drm/i915/i915_gem_gtt.c | 178 ++++++++++++++++++------------------ drivers/gpu/drm/i915/i915_gem_gtt.h | 23 ++++- 2 files changed, 109 insertions(+), 92 deletions(-)
drivers/gpu/drm/i915/i915_gem_gtt.c +91 -87
@@ -338,7 +338,8 @@ static void gen8_ppgtt_clear_range(struct i915_address_space *vm, I915_CACHE_LLC, use_scratch);while (num_entries) { -struct page *page_table = ppgtt->gen8_pt_pages[pdpe][pde]; +struct i915_page_directory_entry *pd = &ppgtt->pdp.page_directory[pdpe]; +struct page *page_table = pd->page_tables[pde].page; last_pte = pte + num_entries;if (last_pte >GEN8_PTES_PER_PAGE )
@@ -382,8 +383,12 @@ static void gen8_ppgtt_insert_entries(struct i915_address_space *vm,if (WARN_ON (pdpe >=GEN8_LEGACY_PDPES ))break ; -if (pt_vaddr ==NULL ) - pt_vaddr =kmap_atomic (ppgtt->gen8_pt_pages[pdpe][pde]); +if (pt_vaddr ==NULL ) { +struct i915_page_directory_entry *pd = &ppgtt->pdp.page_directory[pdpe]; +struct page *page_table = pd->page_tables[pde].page; + + pt_vaddr =kmap_atomic (page_table); + } pt_vaddr[pte] = gen8_pte_encode(sg_page_iter_dma_address(&sg_iter),
@@ -407,29 +412,33 @@ static void gen8_ppgtt_insert_entries(struct i915_address_space *vm, } } -static void gen8_free_page_tables (struct page **pt_pages) +static void gen8_free_page_tables (struct i915_page_directory_entry *pd) {int i; -if (pt_pages ==NULL ) +if (pd->page_tables ==NULL )return ;for (i =0 ; i <GEN8_PDES_PER_PAGE ; i++) -if (pt_pages[i]) -__free_pages (pt_pages[i],0 ); +if (pd->page_tables[i].page) +__free_page (pd->page_tables[i].page); } -static void gen8_ppgtt_free (const struct i915_hw_ppgtt *ppgtt) +static void gen8_free_page_directory (struct i915_page_directory_entry *pd) +{ +gen8_free_page_tables (pd); +kfree (pd->page_tables); +__free_page (pd->page); +} + +static void gen8_ppgtt_free (struct i915_hw_ppgtt *ppgtt) {int i;for (i =0 ; i < ppgtt->num_pd_pages; i++) { -gen8_free_page_tables (ppgtt->gen8_pt_pages[i]); -kfree (ppgtt->gen8_pt_pages[i]); +gen8_free_page_directory (&ppgtt->pdp.page_directory[i]);kfree (ppgtt->gen8_pt_dma_addr[i]); } - -__free_pages (ppgtt->pd_pages,get_order (ppgtt->num_pd_pages <<PAGE_SHIFT )); }static void gen8_ppgtt_unmap_pages (struct i915_hw_ppgtt *ppgtt)
@@ -464,86 +473,77 @@ static void gen8_ppgtt_cleanup(struct i915_address_space *vm)gen8_ppgtt_free (ppgtt); } -static struct page **__gen8_alloc_page_tables (void ) +static int gen8_ppgtt_allocate_dma (struct i915_hw_ppgtt *ppgtt) { -struct page **pt_pages;int i; - pt_pages =kcalloc (GEN8_PDES_PER_PAGE ,sizeof (struct page *),GFP_KERNEL ); -if (!pt_pages) -return ERR_PTR (-ENOMEM ); - -for (i =0 ; i <GEN8_PDES_PER_PAGE ; i++) { - pt_pages[i] =alloc_page (GFP_KERNEL ); -if (!pt_pages[i]) - goto bail; +for (i =0 ; i < ppgtt->num_pd_pages; i++) { + ppgtt->gen8_pt_dma_addr[i] =kcalloc (GEN8_PDES_PER_PAGE , +sizeof (dma_addr_t), +GFP_KERNEL ); +if (!ppgtt->gen8_pt_dma_addr[i]) +return -ENOMEM ; } -return pt_pages; - -bail: -gen8_free_page_tables (pt_pages); -kfree (pt_pages); -return ERR_PTR (-ENOMEM ); +return 0 ; } -static int gen8_ppgtt_allocate_page_tables (struct i915_hw_ppgtt *ppgtt, -const int max_pdp) +static int gen8_ppgtt_allocate_page_tables (struct i915_hw_ppgtt *ppgtt) { -struct page **pt_pages[GEN8_LEGACY_PDPES ]; -int i, ret; +int i, j; -for (i =0 ; i < max_pdp; i++) { - pt_pages[i] =__gen8_alloc_page_tables (); -if (IS_ERR (pt_pages[i])) { - ret =PTR_ERR (pt_pages[i]); - goto unwind_out; +for (i =0 ; i < ppgtt->num_pd_pages; i++) { +for (j =0 ; j <GEN8_PDES_PER_PAGE ; j++) { +struct i915_page_table_entry *pt = &ppgtt->pdp.page_directory[i].page_tables[j]; + + pt->page =alloc_page (GFP_KERNEL | __GFP_ZERO); +if (!pt->page) + goto unwind_out; } } -/* NB: Avoid touching gen8_pt_pages until last to keep the allocation, -* "atomic" - for cleanup purposes. -*/ -for (i =0 ; i < max_pdp; i++) - ppgtt->gen8_pt_pages[i] = pt_pages[i]; -return 0 ; unwind_out: -while (i--) { -gen8_free_page_tables (pt_pages[i]); -kfree (pt_pages[i]); - } +while (i--) +gen8_free_page_tables (&ppgtt->pdp.page_directory[i]); -return ret; +return -ENOMEM ; } -static int gen8_ppgtt_allocate_dma (struct i915_hw_ppgtt *ppgtt) +static int gen8_ppgtt_allocate_page_directories (struct i915_hw_ppgtt *ppgtt, +const int max_pdp) {int i; -for (i =0 ; i < ppgtt->num_pd_pages; i++) { - ppgtt->gen8_pt_dma_addr[i] =kcalloc (GEN8_PDES_PER_PAGE , -sizeof (dma_addr_t), -GFP_KERNEL ); -if (!ppgtt->gen8_pt_dma_addr[i]) -return -ENOMEM ; - } +for (i =0 ; i < max_pdp; i++) { +struct i915_page_table_entry *pt; -return 0 ; -} + pt =kcalloc (GEN8_PDES_PER_PAGE ,sizeof (*pt),GFP_KERNEL ); +if (!pt) + goto unwind_out; -static int gen8_ppgtt_allocate_page_directories (struct i915_hw_ppgtt *ppgtt, -const int max_pdp) -{ - ppgtt->pd_pages =alloc_pages (GFP_KERNEL ,get_order (max_pdp <<PAGE_SHIFT )); -if (!ppgtt->pd_pages) -return -ENOMEM ; + ppgtt->pdp.page_directory[i].page =alloc_page (GFP_KERNEL ); +if (!ppgtt->pdp.page_directory[i].page) { +kfree (pt); + goto unwind_out; + } + + ppgtt->pdp.page_directory[i].page_tables = pt; + } - ppgtt->num_pd_pages =1 <<get_order (max_pdp <<PAGE_SHIFT ); + ppgtt->num_pd_pages = max_pdp;BUG_ON (ppgtt->num_pd_pages >GEN8_LEGACY_PDPES );return 0 ; + +unwind_out: +while (i--) { +kfree (ppgtt->pdp.page_directory[i].page_tables); +__free_page (ppgtt->pdp.page_directory[i].page); + } + +return -ENOMEM ; } static int gen8_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt,
@@ -555,18 +555,20 @@ static int gen8_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt,if (ret)return ret; - ret =gen8_ppgtt_allocate_page_tables (ppgtt, max_pdp); -if (ret) { -__free_pages (ppgtt->pd_pages,get_order (max_pdp <<PAGE_SHIFT )); -return ret; - } + ret =gen8_ppgtt_allocate_page_tables (ppgtt); +if (ret) + goto err_out; ppgtt->num_pd_entries = max_pdp *GEN8_PDES_PER_PAGE ; ret =gen8_ppgtt_allocate_dma (ppgtt);if (ret) -gen8_ppgtt_free (ppgtt); + goto err_out; +return 0 ; + +err_out: +gen8_ppgtt_free (ppgtt);return ret; }
@@ -577,7 +579,7 @@ static int gen8_ppgtt_setup_page_directories(struct i915_hw_ppgtt *ppgtt,int ret; pd_addr =pci_map_page (ppgtt->base.dev->pdev, - &ppgtt->pd_pages[pd],0 , + ppgtt->pdp.page_directory[pd].page,0 ,PAGE_SIZE ,PCI_DMA_BIDIRECTIONAL ); ret =pci_dma_mapping_error (ppgtt->base.dev->pdev, pd_addr);
@@ -597,7 +599,7 @@ static int gen8_ppgtt_setup_page_tables(struct i915_hw_ppgtt *ppgtt,struct page *p;int ret; - p = ppgtt->gen8_pt_pages[pd][pt]; + p = ppgtt->pdp.page_directory[pd].page_tables[pt].page; pt_addr =pci_map_page (ppgtt->base.dev->pdev, p,0 ,PAGE_SIZE ,PCI_DMA_BIDIRECTIONAL ); ret =pci_dma_mapping_error (ppgtt->base.dev->pdev, pt_addr);
@@ -658,7 +660,7 @@ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size)*/ for (i =0 ; i < max_pdp; i++) {gen8_ppgtt_pde_t *pd_vaddr; - pd_vaddr =kmap_atomic (&ppgtt->pd_pages[i]); + pd_vaddr =kmap_atomic (ppgtt->pdp.page_directory[i].page);for (j =0 ; j <GEN8_PDES_PER_PAGE ; j++) {dma_addr_t addr = ppgtt->gen8_pt_dma_addr[i][j]; pd_vaddr[j] = gen8_pde_encode(ppgtt->base.dev, addr,
@@ -721,7 +723,7 @@ static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m) expected);seq_printf (m,"\tPDE: %x\n" , pd_entry); - pt_vaddr =kmap_atomic (ppgtt->pt_pages[pde]); + pt_vaddr =kmap_atomic (ppgtt->pd.page_tables[pde].page);for (pte =0 ; pte <I915_PPGTT_PT_ENTRIES ; pte+=4 ) {unsigned long va = (pde * PAGE_SIZE * I915_PPGTT_PT_ENTRIES) +
@@ -936,7 +938,7 @@ static void gen6_ppgtt_clear_range(struct i915_address_space *vm,if (last_pte >I915_PPGTT_PT_ENTRIES ) last_pte =I915_PPGTT_PT_ENTRIES ; - pt_vaddr =kmap_atomic (ppgtt->pt_pages[act_pt]); + pt_vaddr =kmap_atomic (ppgtt->pd.page_tables[act_pt].page);for (i = first_pte; i < last_pte; i++) pt_vaddr[i] = scratch_pte;
@@ -965,7 +967,7 @@ static void gen6_ppgtt_insert_entries(struct i915_address_space *vm, pt_vaddr =NULL ;for_each_sg_page (pages->sgl, &sg_iter, pages->nents,0 ) {if (pt_vaddr ==NULL ) - pt_vaddr =kmap_atomic (ppgtt->pt_pages[act_pt]); + pt_vaddr =kmap_atomic (ppgtt->pd.page_tables[act_pt].page); pt_vaddr[act_pte] = vm->pte_encode(sg_page_iter_dma_address(&sg_iter),
@@ -1000,8 +1002,8 @@ static void gen6_ppgtt_free(struct i915_hw_ppgtt *ppgtt)kfree (ppgtt->pt_dma_addr);for (i =0 ; i < ppgtt->num_pd_entries; i++) -__free_page (ppgtt->pt_pages[i]); -kfree (ppgtt->pt_pages); +__free_page (ppgtt->pd.page_tables[i].page); +kfree (ppgtt->pd.page_tables); }static void gen6_ppgtt_cleanup (struct i915_address_space *vm)
@@ -1058,22 +1060,22 @@ alloc:static int gen6_ppgtt_allocate_page_tables (struct i915_hw_ppgtt *ppgtt) { +struct i915_page_table_entry *pt;int i; - ppgtt->pt_pages =kcalloc (ppgtt->num_pd_entries,sizeof (struct page *), -GFP_KERNEL ); - -if (!ppgtt->pt_pages) + pt =kcalloc (ppgtt->num_pd_entries,sizeof (*pt),GFP_KERNEL ); +if (!pt)return -ENOMEM ;for (i =0 ; i < ppgtt->num_pd_entries; i++) { - ppgtt->pt_pages[i] =alloc_page (GFP_KERNEL ); -if (!ppgtt->pt_pages[i]) { + pt[i].page =alloc_page (GFP_KERNEL ); +if (!pt->page) {gen6_ppgtt_free (ppgtt);return -ENOMEM ; } } + ppgtt->pd.page_tables = pt;return 0 ; }
@@ -1108,9 +1110,11 @@ static int gen6_ppgtt_setup_page_tables(struct i915_hw_ppgtt *ppgtt)int i;for (i =0 ; i < ppgtt->num_pd_entries; i++) { +struct page *page;dma_addr_t pt_addr; - pt_addr =pci_map_page (dev->pdev, ppgtt->pt_pages[i],0 ,4096 , + page = ppgtt->pd.page_tables[i].page; + pt_addr =pci_map_page (dev->pdev, page,0 ,4096 ,PCI_DMA_BIDIRECTIONAL );if (pci_dma_mapping_error (dev->pdev, pt_addr)) {
@@ -1157,7 +1161,7 @@ static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt) ppgtt->base.insert_entries = gen6_ppgtt_insert_entries; ppgtt->base.cleanup = gen6_ppgtt_cleanup; ppgtt->base.start =0 ; - ppgtt->base.total = ppgtt->num_pd_entries *I915_PPGTT_PT_ENTRIES *PAGE_SIZE ; + ppgtt->base.total = ppgtt->num_pd_entries *I915_PPGTT_PT_ENTRIES *PAGE_SIZE ; ppgtt->debug_dump = gen6_dump_ppgtt; ppgtt->pd_offset =
drivers/gpu/drm/i915/i915_gem_gtt.h +18 -5
@@ -187,6 +187,20 @@ struct i915_vma { u32 flags); }; +struct i915_page_table_entry { +struct page *page; +}; + +struct i915_page_directory_entry { +struct page *page;/* NULL for GEN6-GEN7 */ +struct i915_page_table_entry *page_tables; +}; + +struct i915_page_directory_pointer_entry { +/* struct page *page; */ +struct i915_page_directory_entry page_directory[GEN8_LEGACY_PDPES ]; +}; +struct i915_address_space {struct drm_mm mm;struct drm_device *dev;
@@ -272,11 +286,6 @@ struct i915_hw_ppgtt {unsigned num_pd_entries;unsigned num_pd_pages;/* gen8+ */ union { -struct page **pt_pages; -struct page **gen8_pt_pages[GEN8_LEGACY_PDPES ]; - }; -struct page *pd_pages; -union {uint32_t pd_offset;dma_addr_t pd_dma_addr[GEN8_LEGACY_PDPES ]; };
@@ -284,6 +293,10 @@ struct i915_hw_ppgtt {dma_addr_t *pt_dma_addr;dma_addr_t *gen8_pt_dma_addr[GEN8_LEGACY_PDPES ]; }; +union { +struct i915_page_directory_pointer_entry pdp; +struct i915_page_directory_entry pd; + };struct drm_i915_file_private *file_priv;
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH 1/1 drm/i915: page table abstractions
From: Ben Widawsky <[email protected]>
When we move to dynamic page allocation, keeping page_directory and pagetabs as separate structures will help to break actions into simpler tasks.
To help transition the code nicely there is some wasted space in gen6/7. This will be ameliorated shortly.
Following the x86 pagetable terminology:
PDPE = struct i915_page_directory_pointer_entry. PDE = struct i915_page_directory_entry [page_directory]. PTE = struct i915_page_table_entry [page_tables].
v2: fixed mismatches after clean-up/rebase.
v3: Clarify the names of the multiple levels of page tables (Daniel)
v4: Addressing Mika's review comments. s/gen8_free_page_directories/gen8_free_page_directory and free the page tables for the directory there.
In gen8_ppgtt_allocate_page_directories, do not leak previously allocated pt in case the page_directory alloc fails.
Update error return handling in gen8_ppgtt_alloc.
v5: Do not leak pt on error in gen6_ppgtt_allocate_page_tables. (Mika)
--- drivers/gpu/drm/i915/i915_gem_gtt.c | 180 +++++++++++++++++++----------------- drivers/gpu/drm/i915/i915_gem_gtt.h | 23 ++++- 2 files changed, 111 insertions(+), 92 deletions(-)
drivers/gpu/drm/i915/i915_gem_gtt.c +93 -87
@@ -338,7 +338,8 @@ static void gen8_ppgtt_clear_range(struct i915_address_space *vm, I915_CACHE_LLC, use_scratch);while (num_entries) { -struct page *page_table = ppgtt->gen8_pt_pages[pdpe][pde]; +struct i915_page_directory_entry *pd = &ppgtt->pdp.page_directory[pdpe]; +struct page *page_table = pd->page_tables[pde].page; last_pte = pte + num_entries;if (last_pte >GEN8_PTES_PER_PAGE )
@@ -382,8 +383,12 @@ static void gen8_ppgtt_insert_entries(struct i915_address_space *vm,if (WARN_ON (pdpe >=GEN8_LEGACY_PDPES ))break ; -if (pt_vaddr ==NULL ) - pt_vaddr =kmap_atomic (ppgtt->gen8_pt_pages[pdpe][pde]); +if (pt_vaddr ==NULL ) { +struct i915_page_directory_entry *pd = &ppgtt->pdp.page_directory[pdpe]; +struct page *page_table = pd->page_tables[pde].page; + + pt_vaddr =kmap_atomic (page_table); + } pt_vaddr[pte] = gen8_pte_encode(sg_page_iter_dma_address(&sg_iter),
@@ -407,29 +412,33 @@ static void gen8_ppgtt_insert_entries(struct i915_address_space *vm, } } -static void gen8_free_page_tables (struct page **pt_pages) +static void gen8_free_page_tables (struct i915_page_directory_entry *pd) {int i; -if (pt_pages ==NULL ) +if (pd->page_tables ==NULL )return ;for (i =0 ; i <GEN8_PDES_PER_PAGE ; i++) -if (pt_pages[i]) -__free_pages (pt_pages[i],0 ); +if (pd->page_tables[i].page) +__free_page (pd->page_tables[i].page); } -static void gen8_ppgtt_free (const struct i915_hw_ppgtt *ppgtt) +static void gen8_free_page_directory (struct i915_page_directory_entry *pd) +{ +gen8_free_page_tables (pd); +kfree (pd->page_tables); +__free_page (pd->page); +} + +static void gen8_ppgtt_free (struct i915_hw_ppgtt *ppgtt) {int i;for (i =0 ; i < ppgtt->num_pd_pages; i++) { -gen8_free_page_tables (ppgtt->gen8_pt_pages[i]); -kfree (ppgtt->gen8_pt_pages[i]); +gen8_free_page_directory (&ppgtt->pdp.page_directory[i]);kfree (ppgtt->gen8_pt_dma_addr[i]); } - -__free_pages (ppgtt->pd_pages,get_order (ppgtt->num_pd_pages <<PAGE_SHIFT )); }static void gen8_ppgtt_unmap_pages (struct i915_hw_ppgtt *ppgtt)
@@ -464,86 +473,77 @@ static void gen8_ppgtt_cleanup(struct i915_address_space *vm)gen8_ppgtt_free (ppgtt); } -static struct page **__gen8_alloc_page_tables (void ) +static int gen8_ppgtt_allocate_dma (struct i915_hw_ppgtt *ppgtt) { -struct page **pt_pages;int i; - pt_pages =kcalloc (GEN8_PDES_PER_PAGE ,sizeof (struct page *),GFP_KERNEL ); -if (!pt_pages) -return ERR_PTR (-ENOMEM ); - -for (i =0 ; i <GEN8_PDES_PER_PAGE ; i++) { - pt_pages[i] =alloc_page (GFP_KERNEL ); -if (!pt_pages[i]) - goto bail; +for (i =0 ; i < ppgtt->num_pd_pages; i++) { + ppgtt->gen8_pt_dma_addr[i] =kcalloc (GEN8_PDES_PER_PAGE , +sizeof (dma_addr_t), +GFP_KERNEL ); +if (!ppgtt->gen8_pt_dma_addr[i]) +return -ENOMEM ; } -return pt_pages; - -bail: -gen8_free_page_tables (pt_pages); -kfree (pt_pages); -return ERR_PTR (-ENOMEM ); +return 0 ; } -static int gen8_ppgtt_allocate_page_tables (struct i915_hw_ppgtt *ppgtt, -const int max_pdp) +static int gen8_ppgtt_allocate_page_tables (struct i915_hw_ppgtt *ppgtt) { -struct page **pt_pages[GEN8_LEGACY_PDPES ]; -int i, ret; +int i, j; -for (i =0 ; i < max_pdp; i++) { - pt_pages[i] =__gen8_alloc_page_tables (); -if (IS_ERR (pt_pages[i])) { - ret =PTR_ERR (pt_pages[i]); - goto unwind_out; +for (i =0 ; i < ppgtt->num_pd_pages; i++) { +for (j =0 ; j <GEN8_PDES_PER_PAGE ; j++) { +struct i915_page_table_entry *pt = &ppgtt->pdp.page_directory[i].page_tables[j]; + + pt->page =alloc_page (GFP_KERNEL | __GFP_ZERO); +if (!pt->page) + goto unwind_out; } } -/* NB: Avoid touching gen8_pt_pages until last to keep the allocation, -* "atomic" - for cleanup purposes. -*/ -for (i =0 ; i < max_pdp; i++) - ppgtt->gen8_pt_pages[i] = pt_pages[i]; -return 0 ; unwind_out: -while (i--) { -gen8_free_page_tables (pt_pages[i]); -kfree (pt_pages[i]); - } +while (i--) +gen8_free_page_tables (&ppgtt->pdp.page_directory[i]); -return ret; +return -ENOMEM ; } -static int gen8_ppgtt_allocate_dma (struct i915_hw_ppgtt *ppgtt) +static int gen8_ppgtt_allocate_page_directories (struct i915_hw_ppgtt *ppgtt, +const int max_pdp) {int i; -for (i =0 ; i < ppgtt->num_pd_pages; i++) { - ppgtt->gen8_pt_dma_addr[i] =kcalloc (GEN8_PDES_PER_PAGE , -sizeof (dma_addr_t), -GFP_KERNEL ); -if (!ppgtt->gen8_pt_dma_addr[i]) -return -ENOMEM ; - } +for (i =0 ; i < max_pdp; i++) { +struct i915_page_table_entry *pt; -return 0 ; -} + pt =kcalloc (GEN8_PDES_PER_PAGE ,sizeof (*pt),GFP_KERNEL ); +if (!pt) + goto unwind_out; -static int gen8_ppgtt_allocate_page_directories (struct i915_hw_ppgtt *ppgtt, -const int max_pdp) -{ - ppgtt->pd_pages =alloc_pages (GFP_KERNEL ,get_order (max_pdp <<PAGE_SHIFT )); -if (!ppgtt->pd_pages) -return -ENOMEM ; + ppgtt->pdp.page_directory[i].page =alloc_page (GFP_KERNEL ); +if (!ppgtt->pdp.page_directory[i].page) { +kfree (pt); + goto unwind_out; + } + + ppgtt->pdp.page_directory[i].page_tables = pt; + } - ppgtt->num_pd_pages =1 <<get_order (max_pdp <<PAGE_SHIFT ); + ppgtt->num_pd_pages = max_pdp;BUG_ON (ppgtt->num_pd_pages >GEN8_LEGACY_PDPES );return 0 ; + +unwind_out: +while (i--) { +kfree (ppgtt->pdp.page_directory[i].page_tables); +__free_page (ppgtt->pdp.page_directory[i].page); + } + +return -ENOMEM ; } static int gen8_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt,
@@ -555,18 +555,20 @@ static int gen8_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt,if (ret)return ret; - ret =gen8_ppgtt_allocate_page_tables (ppgtt, max_pdp); -if (ret) { -__free_pages (ppgtt->pd_pages,get_order (max_pdp <<PAGE_SHIFT )); -return ret; - } + ret =gen8_ppgtt_allocate_page_tables (ppgtt); +if (ret) + goto err_out; ppgtt->num_pd_entries = max_pdp *GEN8_PDES_PER_PAGE ; ret =gen8_ppgtt_allocate_dma (ppgtt);if (ret) -gen8_ppgtt_free (ppgtt); + goto err_out; + +return 0 ; +err_out: +gen8_ppgtt_free (ppgtt);return ret; }
@@ -577,7 +579,7 @@ static int gen8_ppgtt_setup_page_directories(struct i915_hw_ppgtt *ppgtt,int ret; pd_addr =pci_map_page (ppgtt->base.dev->pdev, - &ppgtt->pd_pages[pd],0 , + ppgtt->pdp.page_directory[pd].page,0 ,PAGE_SIZE ,PCI_DMA_BIDIRECTIONAL ); ret =pci_dma_mapping_error (ppgtt->base.dev->pdev, pd_addr);
@@ -597,7 +599,7 @@ static int gen8_ppgtt_setup_page_tables(struct i915_hw_ppgtt *ppgtt,struct page *p;int ret; - p = ppgtt->gen8_pt_pages[pd][pt]; + p = ppgtt->pdp.page_directory[pd].page_tables[pt].page; pt_addr =pci_map_page (ppgtt->base.dev->pdev, p,0 ,PAGE_SIZE ,PCI_DMA_BIDIRECTIONAL ); ret =pci_dma_mapping_error (ppgtt->base.dev->pdev, pt_addr);
@@ -658,7 +660,7 @@ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size)*/ for (i =0 ; i < max_pdp; i++) {gen8_ppgtt_pde_t *pd_vaddr; - pd_vaddr =kmap_atomic (&ppgtt->pd_pages[i]); + pd_vaddr =kmap_atomic (ppgtt->pdp.page_directory[i].page);for (j =0 ; j <GEN8_PDES_PER_PAGE ; j++) {dma_addr_t addr = ppgtt->gen8_pt_dma_addr[i][j]; pd_vaddr[j] = gen8_pde_encode(ppgtt->base.dev, addr,
@@ -721,7 +723,7 @@ static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m) expected);seq_printf (m,"\tPDE: %x\n" , pd_entry); - pt_vaddr =kmap_atomic (ppgtt->pt_pages[pde]); + pt_vaddr =kmap_atomic (ppgtt->pd.page_tables[pde].page);for (pte =0 ; pte <I915_PPGTT_PT_ENTRIES ; pte+=4 ) {unsigned long va = (pde * PAGE_SIZE * I915_PPGTT_PT_ENTRIES) +
@@ -936,7 +938,7 @@ static void gen6_ppgtt_clear_range(struct i915_address_space *vm,if (last_pte >I915_PPGTT_PT_ENTRIES ) last_pte =I915_PPGTT_PT_ENTRIES ; - pt_vaddr =kmap_atomic (ppgtt->pt_pages[act_pt]); + pt_vaddr =kmap_atomic (ppgtt->pd.page_tables[act_pt].page);for (i = first_pte; i < last_pte; i++) pt_vaddr[i] = scratch_pte;
@@ -965,7 +967,7 @@ static void gen6_ppgtt_insert_entries(struct i915_address_space *vm, pt_vaddr =NULL ;for_each_sg_page (pages->sgl, &sg_iter, pages->nents,0 ) {if (pt_vaddr ==NULL ) - pt_vaddr =kmap_atomic (ppgtt->pt_pages[act_pt]); + pt_vaddr =kmap_atomic (ppgtt->pd.page_tables[act_pt].page); pt_vaddr[act_pte] = vm->pte_encode(sg_page_iter_dma_address(&sg_iter),
@@ -1000,8 +1002,9 @@ static void gen6_ppgtt_free(struct i915_hw_ppgtt *ppgtt)kfree (ppgtt->pt_dma_addr);for (i =0 ; i < ppgtt->num_pd_entries; i++) -__free_page (ppgtt->pt_pages[i]); -kfree (ppgtt->pt_pages); +if (ppgtt->pd.page_tables[i].page) +__free_page (ppgtt->pd.page_tables[i].page); +kfree (ppgtt->pd.page_tables); }static void gen6_ppgtt_cleanup (struct i915_address_space *vm)
@@ -1058,17 +1061,18 @@ alloc:static int gen6_ppgtt_allocate_page_tables (struct i915_hw_ppgtt *ppgtt) { +struct i915_page_table_entry *pt;int i; - ppgtt->pt_pages =kcalloc (ppgtt->num_pd_entries,sizeof (struct page *), -GFP_KERNEL ); - -if (!ppgtt->pt_pages) + pt =kcalloc (ppgtt->num_pd_entries,sizeof (*pt),GFP_KERNEL ); +if (!pt)return -ENOMEM ; + ppgtt->pd.page_tables = pt; +for (i =0 ; i < ppgtt->num_pd_entries; i++) { - ppgtt->pt_pages[i] =alloc_page (GFP_KERNEL ); -if (!ppgtt->pt_pages[i]) { + pt[i].page =alloc_page (GFP_KERNEL ); +if (!pt->page) {gen6_ppgtt_free (ppgtt);return -ENOMEM ; }
@@ -1108,9 +1112,11 @@ static int gen6_ppgtt_setup_page_tables(struct i915_hw_ppgtt *ppgtt)int i;for (i =0 ; i < ppgtt->num_pd_entries; i++) { +struct page *page;dma_addr_t pt_addr; - pt_addr =pci_map_page (dev->pdev, ppgtt->pt_pages[i],0 ,4096 , + page = ppgtt->pd.page_tables[i].page; + pt_addr =pci_map_page (dev->pdev, page,0 ,4096 ,PCI_DMA_BIDIRECTIONAL );if (pci_dma_mapping_error (dev->pdev, pt_addr)) {
@@ -1157,7 +1163,7 @@ static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt) ppgtt->base.insert_entries = gen6_ppgtt_insert_entries; ppgtt->base.cleanup = gen6_ppgtt_cleanup; ppgtt->base.start =0 ; - ppgtt->base.total = ppgtt->num_pd_entries *I915_PPGTT_PT_ENTRIES *PAGE_SIZE ; + ppgtt->base.total = ppgtt->num_pd_entries *I915_PPGTT_PT_ENTRIES *PAGE_SIZE ; ppgtt->debug_dump = gen6_dump_ppgtt; ppgtt->pd_offset =
drivers/gpu/drm/i915/i915_gem_gtt.h +18 -5
@@ -187,6 +187,20 @@ struct i915_vma { u32 flags); }; +struct i915_page_table_entry { +struct page *page; +}; + +struct i915_page_directory_entry { +struct page *page;/* NULL for GEN6-GEN7 */ +struct i915_page_table_entry *page_tables; +}; + +struct i915_page_directory_pointer_entry { +/* struct page *page; */ +struct i915_page_directory_entry page_directory[GEN8_LEGACY_PDPES ]; +}; +struct i915_address_space {struct drm_mm mm;struct drm_device *dev;
@@ -272,11 +286,6 @@ struct i915_hw_ppgtt {unsigned num_pd_entries;unsigned num_pd_pages;/* gen8+ */ union { -struct page **pt_pages; -struct page **gen8_pt_pages[GEN8_LEGACY_PDPES ]; - }; -struct page *pd_pages; -union {uint32_t pd_offset;dma_addr_t pd_dma_addr[GEN8_LEGACY_PDPES ]; };
@@ -284,6 +293,10 @@ struct i915_hw_ppgtt {dma_addr_t *pt_dma_addr;dma_addr_t *gen8_pt_dma_addr[GEN8_LEGACY_PDPES ]; }; +union { +struct i915_page_directory_pointer_entry pdp; +struct i915_page_directory_entry pd; + };struct drm_i915_file_private *file_priv;
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
Re: [PATCH] drm/i915: page table abstractions
Quoted 24 lines · Michel Thierry
From: Ben Widawsky <[email protected]>
When we move to dynamic page allocation, keeping
page_directoryand pagetabs as separate structures will help to break actions into simpler tasks.To help transition the code nicely there is some wasted space in gen6/7. This will be ameliorated shortly.
Following the x86 pagetable terminology:
PDPE = structi915_page_directory_pointer_entry. PDE = structi915_page_directory_entry[page_directory]. PTE = structi915_page_table_entry[page_tables].v2: fixed mismatches after clean-up/rebase.
v3: Clarify the names of the multiple levels of page tables (Daniel)
v4: Addressing Mika's review comments. s/gen8_free_page_directories/gen8_free_page_directory and free the page tables for the directory there.
Ingen8_ppgtt_allocate_page_directories, do not leak previously allocated pt in case thepage_directoryalloc fails.
Update error return handling ingen8_ppgtt_alloc.
Quoted 5 lines
v5: Do not leak pt on error in
gen6_ppgtt_allocate_page_tables. (Mika)Cc Mika Kuoppala <[email protected]>Signed-off-by Ben Widawsky <[email protected]>Signed-off-by Michel Thierry <[email protected]> (v2+)
Quoted 398 lines
--- drivers/gpu/drm/i915/i915_gem_gtt.c | 180 +++++++++++++++++++----------------- drivers/gpu/drm/i915/i915_gem_gtt.h | 23 ++++- 2 files changed, 111 insertions(+), 92 deletions(-)drivers/gpu/drm/i915/i915_gem_gtt.c +93
-87@@ -338,7 +338,8 @@ static void gen8_ppgtt_clear_range(struct i915_address_space *vm, I915_CACHE_LLC, use_scratch);while (num_entries) { -struct page *page_table = ppgtt->gen8_pt_pages[pdpe][pde]; +struct i915_page_directory_entry *pd = &ppgtt->pdp.page_directory[pdpe]; +struct page *page_table = pd->page_tables[pde].page; last_pte = pte + num_entries;if (last_pte >GEN8_PTES_PER_PAGE )@@ -382,8 +383,12 @@ static void gen8_ppgtt_insert_entries(struct i915_address_space *vm,if (WARN_ON (pdpe >=GEN8_LEGACY_PDPES ))break ; -if (pt_vaddr ==NULL ) - pt_vaddr =kmap_atomic (ppgtt->gen8_pt_pages[pdpe][pde]); +if (pt_vaddr ==NULL ) { +struct i915_page_directory_entry *pd = &ppgtt->pdp.page_directory[pdpe]; +struct page *page_table = pd->page_tables[pde].page; + + pt_vaddr =kmap_atomic (page_table); + } pt_vaddr[pte] = gen8_pte_encode(sg_page_iter_dma_address(&sg_iter),@@ -407,29 +412,33 @@ static void gen8_ppgtt_insert_entries(struct i915_address_space *vm, } } -static void gen8_free_page_tables (struct page **pt_pages) +static void gen8_free_page_tables (struct i915_page_directory_entry *pd) {int i; -if (pt_pages ==NULL ) +if (pd->page_tables ==NULL )return ;for (i =0 ; i <GEN8_PDES_PER_PAGE ; i++) -if (pt_pages[i]) -__free_pages (pt_pages[i],0 ); +if (pd->page_tables[i].page) +__free_page (pd->page_tables[i].page); } -static void gen8_ppgtt_free (const struct i915_hw_ppgtt *ppgtt) +static void gen8_free_page_directory (struct i915_page_directory_entry *pd) +{ +gen8_free_page_tables (pd); +kfree (pd->page_tables); +__free_page (pd->page); +} + +static void gen8_ppgtt_free (struct i915_hw_ppgtt *ppgtt) {int i;for (i =0 ; i < ppgtt->num_pd_pages; i++) { -gen8_free_page_tables (ppgtt->gen8_pt_pages[i]); -kfree (ppgtt->gen8_pt_pages[i]); +gen8_free_page_directory (&ppgtt->pdp.page_directory[i]);kfree (ppgtt->gen8_pt_dma_addr[i]); } - -__free_pages (ppgtt->pd_pages,get_order (ppgtt->num_pd_pages <<PAGE_SHIFT )); }static void gen8_ppgtt_unmap_pages (struct i915_hw_ppgtt *ppgtt)@@ -464,86 +473,77 @@ static void gen8_ppgtt_cleanup(struct i915_address_space *vm)gen8_ppgtt_free (ppgtt); } -static struct page **__gen8_alloc_page_tables (void ) +static int gen8_ppgtt_allocate_dma (struct i915_hw_ppgtt *ppgtt) { -struct page **pt_pages;int i; - pt_pages =kcalloc (GEN8_PDES_PER_PAGE ,sizeof (struct page *),GFP_KERNEL ); -if (!pt_pages) -return ERR_PTR (-ENOMEM ); - -for (i =0 ; i <GEN8_PDES_PER_PAGE ; i++) { - pt_pages[i] =alloc_page (GFP_KERNEL ); -if (!pt_pages[i]) - goto bail; +for (i =0 ; i < ppgtt->num_pd_pages; i++) { + ppgtt->gen8_pt_dma_addr[i] =kcalloc (GEN8_PDES_PER_PAGE , +sizeof (dma_addr_t), +GFP_KERNEL ); +if (!ppgtt->gen8_pt_dma_addr[i]) +return -ENOMEM ; } -return pt_pages; - -bail: -gen8_free_page_tables (pt_pages); -kfree (pt_pages); -return ERR_PTR (-ENOMEM ); +return 0 ; } -static int gen8_ppgtt_allocate_page_tables (struct i915_hw_ppgtt *ppgtt, -const int max_pdp) +static int gen8_ppgtt_allocate_page_tables (struct i915_hw_ppgtt *ppgtt) { -struct page **pt_pages[GEN8_LEGACY_PDPES ]; -int i, ret; +int i, j; -for (i =0 ; i < max_pdp; i++) { - pt_pages[i] =__gen8_alloc_page_tables (); -if (IS_ERR (pt_pages[i])) { - ret =PTR_ERR (pt_pages[i]); - goto unwind_out; +for (i =0 ; i < ppgtt->num_pd_pages; i++) { +for (j =0 ; j <GEN8_PDES_PER_PAGE ; j++) { +struct i915_page_table_entry *pt = &ppgtt->pdp.page_directory[i].page_tables[j]; + + pt->page =alloc_page (GFP_KERNEL | __GFP_ZERO); +if (!pt->page) + goto unwind_out; } } -/* NB: Avoid touching gen8_pt_pages until last to keep the allocation, -* "atomic" - for cleanup purposes. -*/ -for (i =0 ; i < max_pdp; i++) - ppgtt->gen8_pt_pages[i] = pt_pages[i]; -return 0 ; unwind_out: -while (i--) { -gen8_free_page_tables (pt_pages[i]); -kfree (pt_pages[i]); - } +while (i--) +gen8_free_page_tables (&ppgtt->pdp.page_directory[i]); -return ret; +return -ENOMEM ; } -static int gen8_ppgtt_allocate_dma (struct i915_hw_ppgtt *ppgtt) +static int gen8_ppgtt_allocate_page_directories (struct i915_hw_ppgtt *ppgtt, +const int max_pdp) {int i; -for (i =0 ; i < ppgtt->num_pd_pages; i++) { - ppgtt->gen8_pt_dma_addr[i] =kcalloc (GEN8_PDES_PER_PAGE , -sizeof (dma_addr_t), -GFP_KERNEL ); -if (!ppgtt->gen8_pt_dma_addr[i]) -return -ENOMEM ; - } +for (i =0 ; i < max_pdp; i++) { +struct i915_page_table_entry *pt; -return 0 ; -} + pt =kcalloc (GEN8_PDES_PER_PAGE ,sizeof (*pt),GFP_KERNEL ); +if (!pt) + goto unwind_out; -static int gen8_ppgtt_allocate_page_directories (struct i915_hw_ppgtt *ppgtt, -const int max_pdp) -{ - ppgtt->pd_pages =alloc_pages (GFP_KERNEL ,get_order (max_pdp <<PAGE_SHIFT )); -if (!ppgtt->pd_pages) -return -ENOMEM ; + ppgtt->pdp.page_directory[i].page =alloc_page (GFP_KERNEL ); +if (!ppgtt->pdp.page_directory[i].page) { +kfree (pt); + goto unwind_out; + } + + ppgtt->pdp.page_directory[i].page_tables = pt; + } - ppgtt->num_pd_pages =1 <<get_order (max_pdp <<PAGE_SHIFT ); + ppgtt->num_pd_pages = max_pdp;BUG_ON (ppgtt->num_pd_pages >GEN8_LEGACY_PDPES );return 0 ; + +unwind_out: +while (i--) { +kfree (ppgtt->pdp.page_directory[i].page_tables); +__free_page (ppgtt->pdp.page_directory[i].page); + } + +return -ENOMEM ; } static int gen8_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt,@@ -555,18 +555,20 @@ static int gen8_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt,if (ret)return ret; - ret =gen8_ppgtt_allocate_page_tables (ppgtt, max_pdp); -if (ret) { -__free_pages (ppgtt->pd_pages,get_order (max_pdp <<PAGE_SHIFT )); -return ret; - } + ret =gen8_ppgtt_allocate_page_tables (ppgtt); +if (ret) + goto err_out; ppgtt->num_pd_entries = max_pdp *GEN8_PDES_PER_PAGE ; ret =gen8_ppgtt_allocate_dma (ppgtt);if (ret) -gen8_ppgtt_free (ppgtt); + goto err_out; + +return 0 ; +err_out: +gen8_ppgtt_free (ppgtt);return ret; }@@ -577,7 +579,7 @@ static int gen8_ppgtt_setup_page_directories(struct i915_hw_ppgtt *ppgtt,int ret; pd_addr =pci_map_page (ppgtt->base.dev->pdev, - &ppgtt->pd_pages[pd],0 , + ppgtt->pdp.page_directory[pd].page,0 ,PAGE_SIZE ,PCI_DMA_BIDIRECTIONAL ); ret =pci_dma_mapping_error (ppgtt->base.dev->pdev, pd_addr);@@ -597,7 +599,7 @@ static int gen8_ppgtt_setup_page_tables(struct i915_hw_ppgtt *ppgtt,struct page *p;int ret; - p = ppgtt->gen8_pt_pages[pd][pt]; + p = ppgtt->pdp.page_directory[pd].page_tables[pt].page; pt_addr =pci_map_page (ppgtt->base.dev->pdev, p,0 ,PAGE_SIZE ,PCI_DMA_BIDIRECTIONAL ); ret =pci_dma_mapping_error (ppgtt->base.dev->pdev, pt_addr);@@ -658,7 +660,7 @@ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size)*/ for (i =0 ; i < max_pdp; i++) {gen8_ppgtt_pde_t *pd_vaddr; - pd_vaddr =kmap_atomic (&ppgtt->pd_pages[i]); + pd_vaddr =kmap_atomic (ppgtt->pdp.page_directory[i].page);for (j =0 ; j <GEN8_PDES_PER_PAGE ; j++) {dma_addr_t addr = ppgtt->gen8_pt_dma_addr[i][j]; pd_vaddr[j] = gen8_pde_encode(ppgtt->base.dev, addr,@@ -721,7 +723,7 @@ static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m) expected);seq_printf (m,"\tPDE: %x\n" , pd_entry); - pt_vaddr =kmap_atomic (ppgtt->pt_pages[pde]); + pt_vaddr =kmap_atomic (ppgtt->pd.page_tables[pde].page);for (pte =0 ; pte <I915_PPGTT_PT_ENTRIES ; pte+=4 ) {unsigned long va = (pde * PAGE_SIZE * I915_PPGTT_PT_ENTRIES) +@@ -936,7 +938,7 @@ static void gen6_ppgtt_clear_range(struct i915_address_space *vm,if (last_pte >I915_PPGTT_PT_ENTRIES ) last_pte =I915_PPGTT_PT_ENTRIES ; - pt_vaddr =kmap_atomic (ppgtt->pt_pages[act_pt]); + pt_vaddr =kmap_atomic (ppgtt->pd.page_tables[act_pt].page);for (i = first_pte; i < last_pte; i++) pt_vaddr[i] = scratch_pte;@@ -965,7 +967,7 @@ static void gen6_ppgtt_insert_entries(struct i915_address_space *vm, pt_vaddr =NULL ;for_each_sg_page (pages->sgl, &sg_iter, pages->nents,0 ) {if (pt_vaddr ==NULL ) - pt_vaddr =kmap_atomic (ppgtt->pt_pages[act_pt]); + pt_vaddr =kmap_atomic (ppgtt->pd.page_tables[act_pt].page); pt_vaddr[act_pte] = vm->pte_encode(sg_page_iter_dma_address(&sg_iter),@@ -1000,8 +1002,9 @@ static void gen6_ppgtt_free(struct i915_hw_ppgtt *ppgtt)kfree (ppgtt->pt_dma_addr);for (i =0 ; i < ppgtt->num_pd_entries; i++) -__free_page (ppgtt->pt_pages[i]); -kfree (ppgtt->pt_pages); +if (ppgtt->pd.page_tables[i].page) +__free_page (ppgtt->pd.page_tables[i].page); +kfree (ppgtt->pd.page_tables); }static void gen6_ppgtt_cleanup (struct i915_address_space *vm)@@ -1058,17 +1061,18 @@ alloc:static int gen6_ppgtt_allocate_page_tables (struct i915_hw_ppgtt *ppgtt) { +struct i915_page_table_entry *pt;int i; - ppgtt->pt_pages =kcalloc (ppgtt->num_pd_entries,sizeof (struct page *), -GFP_KERNEL ); - -if (!ppgtt->pt_pages) + pt =kcalloc (ppgtt->num_pd_entries,sizeof (*pt),GFP_KERNEL ); +if (!pt)return -ENOMEM ; + ppgtt->pd.page_tables = pt; +for (i =0 ; i < ppgtt->num_pd_entries; i++) { - ppgtt->pt_pages[i] =alloc_page (GFP_KERNEL ); -if (!ppgtt->pt_pages[i]) { + pt[i].page =alloc_page (GFP_KERNEL ); +if (!pt->page) {gen6_ppgtt_free (ppgtt);return -ENOMEM ; }@@ -1108,9 +1112,11 @@ static int gen6_ppgtt_setup_page_tables(struct i915_hw_ppgtt *ppgtt)int i;for (i =0 ; i < ppgtt->num_pd_entries; i++) { +struct page *page;dma_addr_t pt_addr; - pt_addr =pci_map_page (dev->pdev, ppgtt->pt_pages[i],0 ,4096 , + page = ppgtt->pd.page_tables[i].page; + pt_addr =pci_map_page (dev->pdev, page,0 ,4096 ,PCI_DMA_BIDIRECTIONAL );if (pci_dma_mapping_error (dev->pdev, pt_addr)) {@@ -1157,7 +1163,7 @@ static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt) ppgtt->base.insert_entries = gen6_ppgtt_insert_entries; ppgtt->base.cleanup = gen6_ppgtt_cleanup; ppgtt->base.start =0 ; - ppgtt->base.total = ppgtt->num_pd_entries *I915_PPGTT_PT_ENTRIES *PAGE_SIZE ; + ppgtt->base.total = ppgtt->num_pd_entries *I915_PPGTT_PT_ENTRIES *PAGE_SIZE ; ppgtt->debug_dump = gen6_dump_ppgtt; ppgtt->pd_offset =drivers/gpu/drm/i915/i915_gem_gtt.h +18
-5@@ -187,6 +187,20 @@ struct i915_vma { u32 flags); }; +struct i915_page_table_entry { +struct page *page; +}; + +struct i915_page_directory_entry { +struct page *page;/* NULL for GEN6-GEN7 */ +struct i915_page_table_entry *page_tables; +}; + +struct i915_page_directory_pointer_entry { +/* struct page *page; */ +struct i915_page_directory_entry page_directory[GEN8_LEGACY_PDPES ]; +}; +struct i915_address_space {struct drm_mm mm;struct drm_device *dev;@@ -272,11 +286,6 @@ struct i915_hw_ppgtt {unsigned num_pd_entries;unsigned num_pd_pages;/* gen8+ */ union { -struct page **pt_pages; -struct page **gen8_pt_pages[GEN8_LEGACY_PDPES ]; - }; -struct page *pd_pages; -union {uint32_t pd_offset;dma_addr_t pd_dma_addr[GEN8_LEGACY_PDPES ]; };@@ -284,6 +293,10 @@ struct i915_hw_ppgtt {dma_addr_t *pt_dma_addr;dma_addr_t *gen8_pt_dma_addr[GEN8_LEGACY_PDPES ]; }; +union { +struct i915_page_directory_pointer_entry pdp; +struct i915_page_directory_entry pd; + };struct drm_i915_file_private *file_priv;Signature · 1 line
2.1.1
_______________________________________________
Intel-gfx mailing list
[email protected]
http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH v5 2/32 drm/i915: Complete page table structures
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
Move the remaining members over to the new page table structures.
This can be squashed with the previous commit if desire. The reasoning is the same as that patch. I simply felt it is easier to review if split.
v2: In lrc: s/ppgtt->pd_dma_addr[i]/ppgtt->pdp.page_directory[i].daddr/
v3: Rebase.
--- drivers/gpu/drm/i915/i915_debugfs.c | 2 +- drivers/gpu/drm/i915/i915_gem_gtt.c | 85 +++++++++++++------------------------ drivers/gpu/drm/i915/i915_gem_gtt.h | 14 +++--- drivers/gpu/drm/i915/intel_lrc.c | 16 +++---- 4 files changed, 44 insertions(+), 73 deletions(-)
drivers/gpu/drm/i915/i915_debugfs.c +1 -1
@@ -2185,7 +2185,7 @@ static void gen6_ppgtt_info(struct seq_file *m, struct drm_device *dev)struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;seq_puts (m,"aliasing PPGTT:\n" ); -seq_printf (m,"pd gtt offset: 0x%08x\n" , ppgtt->pd_offset); +seq_printf (m,"pd gtt offset: 0x%08x\n" , ppgtt->pd.pd_offset); ppgtt->debug_dump (ppgtt, m); }
drivers/gpu/drm/i915/i915_gem_gtt.c +29 -56
220 lines not on this page · raw message
drivers/gpu/drm/i915/i915_gem_gtt.h +6 -8
@@ -189,10 +189,16 @@ struct i915_vma {struct i915_page_table_entry {struct page *page; +dma_addr_t daddr; };struct i915_page_directory_entry {struct page *page;/* NULL for GEN6-GEN7 */ +union { +uint32_t pd_offset; +dma_addr_t daddr; + }; +struct i915_page_table_entry *page_tables; };
@@ -286,14 +292,6 @@ struct i915_hw_ppgtt {unsigned num_pd_entries;unsigned num_pd_pages;/* gen8+ */ union { -uint32_t pd_offset; -dma_addr_t pd_dma_addr[GEN8_LEGACY_PDPES ]; - }; -union { -dma_addr_t *pt_dma_addr; -dma_addr_t *gen8_pt_dma_addr[GEN8_LEGACY_PDPES ]; - }; -union {struct i915_page_directory_pointer_entry pdp;struct i915_page_directory_entry pd; };
drivers/gpu/drm/i915/intel_lrc.c +8 -8
@@ -1735,14 +1735,14 @@ populate_lr_context(struct intel_context *ctx, struct drm_i915_gem_object *ctx_o reg_state[CTX_PDP1_LDW ] =GEN8_RING_PDP_LDW (ring,1 ); reg_state[CTX_PDP0_UDW ] =GEN8_RING_PDP_UDW (ring,0 ); reg_state[CTX_PDP0_LDW ] =GEN8_RING_PDP_LDW (ring,0 ); - reg_state[CTX_PDP3_UDW +1 ] =upper_32_bits (ppgtt->pd_dma_addr[3 ]); - reg_state[CTX_PDP3_LDW +1 ] =lower_32_bits (ppgtt->pd_dma_addr[3 ]); - reg_state[CTX_PDP2_UDW +1 ] =upper_32_bits (ppgtt->pd_dma_addr[2 ]); - reg_state[CTX_PDP2_LDW +1 ] =lower_32_bits (ppgtt->pd_dma_addr[2 ]); - reg_state[CTX_PDP1_UDW +1 ] =upper_32_bits (ppgtt->pd_dma_addr[1 ]); - reg_state[CTX_PDP1_LDW +1 ] =lower_32_bits (ppgtt->pd_dma_addr[1 ]); - reg_state[CTX_PDP0_UDW +1 ] =upper_32_bits (ppgtt->pd_dma_addr[0 ]); - reg_state[CTX_PDP0_LDW +1 ] =lower_32_bits (ppgtt->pd_dma_addr[0 ]); + reg_state[CTX_PDP3_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[3 ].daddr); + reg_state[CTX_PDP3_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[3 ].daddr); + reg_state[CTX_PDP2_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[2 ].daddr); + reg_state[CTX_PDP2_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[2 ].daddr); + reg_state[CTX_PDP1_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[1 ].daddr); + reg_state[CTX_PDP1_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[1 ].daddr); + reg_state[CTX_PDP0_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[0 ].daddr); + reg_state[CTX_PDP0_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[0 ].daddr);if (ring->id ==RCS ) { reg_state[CTX_LRI_HEADER_2 ] =MI_LOAD_REGISTER_IMM (1 ); reg_state[CTX_R_PWR_CLK_STATE ] =0x20c8 ;
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
Re: [PATCH v5 02/32] drm/i915: Complete page table structures
Quoted 94 lines · Michel Thierry
From: Ben Widawsky <[email protected]>
Move the remaining members over to the new page table structures.
This can be squashed with the previous commit if desire. The reasoning is the same as that patch. I simply felt it is easier to review if split.
v2: In lrc: s/ppgtt->pd_dma_addr[i]/ppgtt->pdp.page_directory[i].daddr/
v3: Rebase.Signed-off-by Ben Widawsky <[email protected]>Signed-off-by Michel Thierry <[email protected]> (v2, v3)--- drivers/gpu/drm/i915/i915_debugfs.c | 2 +- drivers/gpu/drm/i915/i915_gem_gtt.c | 85 +++++++++++++------------------------ drivers/gpu/drm/i915/i915_gem_gtt.h | 14 +++--- drivers/gpu/drm/i915/intel_lrc.c | 16 +++---- 4 files changed, 44 insertions(+), 73 deletions(-)drivers/gpu/drm/i915/i915_debugfs.c +1
-1@@ -2185,7 +2185,7 @@ static void gen6_ppgtt_info(struct seq_file *m, struct drm_device *dev)struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;seq_puts (m,"aliasing PPGTT:\n" ); -seq_printf (m,"pd gtt offset: 0x%08x\n" , ppgtt->pd_offset); +seq_printf (m,"pd gtt offset: 0x%08x\n" , ppgtt->pd.pd_offset); ppgtt->debug_dump (ppgtt, m); }drivers/gpu/drm/i915/i915_gem_gtt.c +4
-20@@ -311,7 +311,7 @@ static int gen8_mm_switch(struct i915_hw_ppgtt *ppgtt,int used_pd = ppgtt->num_pd_entries /GEN8_PDES_PER_PAGE ;for (i = used_pd -1 ; i >=0 ; i--) { -dma_addr_t addr = ppgtt->pd_dma_addr[i]; +dma_addr_t addr = ppgtt->pdp.page_directory[i].daddr; ret =gen8_write_pdp (ring, i, addr);if (ret)return ret;@@ -437,7 +437,6 @@ static void gen8_ppgtt_free(struct i915_hw_ppgtt *ppgtt)for (i =0 ; i < ppgtt->num_pd_pages; i++) {gen8_free_page_directory (&ppgtt->pdp.page_directory[i]); -kfree (ppgtt->gen8_pt_dma_addr[i]); } }@@ -449,14 +448,14 @@ static void gen8_ppgtt_unmap_pages(struct i915_hw_ppgtt *ppgtt)for (i =0 ; i < ppgtt->num_pd_pages; i++) {/* TODO: In the future we'll support sparse mappings, so this * will have to change. */ -if (!ppgtt->pd_dma_addr[i]) +if (!ppgtt->pdp.page_directory[i].daddr)continue ; -pci_unmap_page (hwdev, ppgtt->pd_dma_addr[i],PAGE_SIZE , +pci_unmap_page (hwdev, ppgtt->pdp.page_directory[i].daddr,PAGE_SIZE ,PCI_DMA_BIDIRECTIONAL );for (j =0 ; j <GEN8_PDES_PER_PAGE ; j++) { -dma_addr_t addr = ppgtt->gen8_pt_dma_addr[i][j]; +dma_addr_t addr = ppgtt->pdp.page_directory[i].page_tables[j].daddr;if (addr)pci_unmap_page (hwdev, addr,PAGE_SIZE ,PCI_DMA_BIDIRECTIONAL );@@ -473,32 +472,19 @@ static void gen8_ppgtt_cleanup(struct i915_address_space *vm)gen8_ppgtt_free (ppgtt); } -static int gen8_ppgtt_allocate_dma (struct i915_hw_ppgtt *ppgtt) -{ -int i; - -for (i =0 ; i < ppgtt->num_pd_pages; i++) { - ppgtt->gen8_pt_dma_addr[i] =kcalloc (GEN8_PDES_PER_PAGE , -sizeof (dma_addr_t), -GFP_KERNEL ); -if (!ppgtt->gen8_pt_dma_addr[i]) -return -ENOMEM ; - } - -return 0 ; -} -static int gen8_ppgtt_allocate_page_tables (struct i915_hw_ppgtt *ppgtt) {int i, j;
drivers/gpu/drm/i915/i915_gem_gtt.c static void gen8_ppgtt_cleanup(struct i915_address_space *vm) · line 487
Quoted 10 lines
drivers/gpu/drm/i915/i915_gem_gtt.c +3
-1for (i =0 ; i < ppgtt->num_pd_pages; i++) { +struct i915_page_directory_entry *pd = &ppgtt->pdp.page_directory[i];for (j =0 ; j <GEN8_PDES_PER_PAGE ; j++) { -struct i915_page_table_entry *pt = &ppgtt->pdp.page_directory[i].page_tables[j]; +struct i915_page_table_entry *pt = &pd->page_tables[j]; pt->page =alloc_page (GFP_KERNEL | __GFP_ZERO);if (!pt->page) goto unwind_out; +
This hunk should have been in the previous patch, oh well..
drivers/gpu/drm/i915/i915_gem_gtt.c static int gen8_ppgtt_setup_page_directories(struct i915_hw_ppgtt *ppgtt,
Quoted 15 lines
drivers/gpu/drm/i915/i915_gem_gtt.c +0
-4} }@@ -561,10 +547,6 @@ static int gen8_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt, ppgtt->num_pd_entries = max_pdp *GEN8_PDES_PER_PAGE ; - ret =gen8_ppgtt_allocate_dma (ppgtt); -if (ret) - goto err_out; -return 0 ; err_out:@@ -586,7 +568,7 @@ static int gen8_ppgtt_setup_page_directories(struct i915_hw_ppgtt *ppgtt,
Here we are again setting only one page directory. But as it is not problem with this patch:
Quoted 223 lines
drivers/gpu/drm/i915/i915_gem_gtt.c +22
-31if (ret)return ret; - ppgtt->pd_dma_addr[pd] = pd_addr; + ppgtt->pdp.page_directory[pd].daddr = pd_addr;return 0 ; }@@ -596,17 +578,18 @@ static int gen8_ppgtt_setup_page_tables(struct i915_hw_ppgtt *ppgtt, const int pt) {dma_addr_t pt_addr; -struct page *p; +struct i915_page_directory_entry *pdir = &ppgtt->pdp.page_directory[pd]; +struct i915_page_table_entry *ptab = &pdir->page_tables[pt]; +struct page *p = ptab->page;int ret; - p = ppgtt->pdp.page_directory[pd].page_tables[pt].page; pt_addr =pci_map_page (ppgtt->base.dev->pdev, p,0 ,PAGE_SIZE ,PCI_DMA_BIDIRECTIONAL ); ret =pci_dma_mapping_error (ppgtt->base.dev->pdev, pt_addr);if (ret)return ret; - ppgtt->gen8_pt_dma_addr[pd][pt] = pt_addr; + ptab->daddr = pt_addr;return 0 ; }@@ -662,7 +645,7 @@ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size)gen8_ppgtt_pde_t *pd_vaddr; pd_vaddr =kmap_atomic (ppgtt->pdp.page_directory[i].page);for (j =0 ; j <GEN8_PDES_PER_PAGE ; j++) { -dma_addr_t addr = ppgtt->gen8_pt_dma_addr[i][j]; +dma_addr_t addr = ppgtt->pdp.page_directory[i].page_tables[j].daddr; pd_vaddr[j] =gen8_pde_encode (ppgtt->base.dev, addr,I915_CACHE_LLC ); }@@ -705,14 +688,15 @@ static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m) scratch_pte = vm->pte_encode (vm->scratch.addr,I915_CACHE_LLC , true,0 ); pd_addr = (gen6_gtt_pte_t __iomem *)dev_priv->gtt.gsm + - ppgtt->pd_offset /sizeof (gen6_gtt_pte_t); + ppgtt->pd.pd_offset /sizeof (gen6_gtt_pte_t);seq_printf (m," VM %p (pd_offset %x-%x):\n" , vm, - ppgtt->pd_offset, ppgtt->pd_offset + ppgtt->num_pd_entries); + ppgtt->pd.pd_offset, + ppgtt->pd.pd_offset + ppgtt->num_pd_entries);for (pde =0 ; pde < ppgtt->num_pd_entries; pde++) {u32 expected;gen6_gtt_pte_t *pt_vaddr; -dma_addr_t pt_addr = ppgtt->pt_dma_addr[pde]; +dma_addr_t pt_addr = ppgtt->pd.page_tables[pde].daddr; pd_entry =readl (pd_addr + pde); expected = (GEN6_PDE_ADDR_ENCODE (pt_addr) |GEN6_PDE_VALID );@@ -756,13 +740,13 @@ static void gen6_write_pdes(struct i915_hw_ppgtt *ppgtt)uint32_t pd_entry;int i; -WARN_ON (ppgtt->pd_offset &0x3f ); +WARN_ON (ppgtt->pd.pd_offset &0x3f ); pd_addr = (gen6_gtt_pte_t __iomem *)dev_priv->gtt.gsm + - ppgtt->pd_offset /sizeof (gen6_gtt_pte_t); + ppgtt->pd.pd_offset /sizeof (gen6_gtt_pte_t);for (i =0 ; i < ppgtt->num_pd_entries; i++) {dma_addr_t pt_addr; - pt_addr = ppgtt->pt_dma_addr[i]; + pt_addr = ppgtt->pd.page_tables[i].daddr; pd_entry =GEN6_PDE_ADDR_ENCODE (pt_addr); pd_entry |=GEN6_PDE_VALID ;@@ -773,9 +757,9 @@ static void gen6_write_pdes(struct i915_hw_ppgtt *ppgtt)static uint32_t get_pd_offset (struct i915_hw_ppgtt *ppgtt) { -BUG_ON (ppgtt->pd_offset &0x3f ); +BUG_ON (ppgtt->pd.pd_offset &0x3f ); -return (ppgtt->pd_offset /64 ) <<16 ; +return (ppgtt->pd.pd_offset /64 ) <<16 ; } static int hsw_mm_switch(struct i915_hw_ppgtt *ppgtt,@@ -988,19 +972,16 @@ static void gen6_ppgtt_unmap_pages(struct i915_hw_ppgtt *ppgtt) {int i; -if (ppgtt->pt_dma_addr) { -for (i =0 ; i < ppgtt->num_pd_entries; i++) -pci_unmap_page (ppgtt->base.dev->pdev, - ppgtt->pt_dma_addr[i], -4096 ,PCI_DMA_BIDIRECTIONAL ); - } +for (i =0 ; i < ppgtt->num_pd_entries; i++) +pci_unmap_page (ppgtt->base.dev->pdev, + ppgtt->pd.page_tables[i].daddr, +4096 ,PCI_DMA_BIDIRECTIONAL ); }static void gen6_ppgtt_free (struct i915_hw_ppgtt *ppgtt) {int i; -kfree (ppgtt->pt_dma_addr);for (i =0 ; i < ppgtt->num_pd_entries; i++)__free_page (ppgtt->pd.page_tables[i].page);kfree (ppgtt->pd.page_tables);@@ -1093,14 +1074,6 @@ static int gen6_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt)return ret; } - ppgtt->pt_dma_addr =kcalloc (ppgtt->num_pd_entries,sizeof (dma_addr_t), -GFP_KERNEL ); -if (!ppgtt->pt_dma_addr) { -drm_mm_remove_node (&ppgtt->node); -gen6_ppgtt_free (ppgtt); -return -ENOMEM ; - } -return 0 ; }@@ -1122,7 +1095,7 @@ static int gen6_ppgtt_setup_page_tables(struct i915_hw_ppgtt *ppgtt)return -EIO ; } - ppgtt->pt_dma_addr[i] = pt_addr; + ppgtt->pd.page_tables[i].daddr = pt_addr; }return 0 ;@@ -1164,7 +1137,7 @@ static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt) ppgtt->base.total = ppgtt->num_pd_entries *I915_PPGTT_PT_ENTRIES *PAGE_SIZE ; ppgtt->debug_dump = gen6_dump_ppgtt; - ppgtt->pd_offset = + ppgtt->pd.pd_offset = ppgtt->node.start /PAGE_SIZE *sizeof (gen6_gtt_pte_t); ppgtt->base.clear_range (&ppgtt->base,0 , ppgtt->base.total, true);@@ -1175,7 +1148,7 @@ static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt)gen6_write_pdes (ppgtt);DRM_DEBUG ("Adding PPGTT at offset %x\n" , - ppgtt->pd_offset <<10 ); + ppgtt->pd.pd_offset <<10 );return 0 ; }drivers/gpu/drm/i915/i915_gem_gtt.h +6
-8@@ -189,10 +189,16 @@ struct i915_vma {struct i915_page_table_entry {struct page *page; +dma_addr_t daddr; };struct i915_page_directory_entry {struct page *page;/* NULL for GEN6-GEN7 */ +union { +uint32_t pd_offset; +dma_addr_t daddr; + }; +struct i915_page_table_entry *page_tables; };@@ -286,14 +292,6 @@ struct i915_hw_ppgtt {unsigned num_pd_entries;unsigned num_pd_pages;/* gen8+ */ union { -uint32_t pd_offset; -dma_addr_t pd_dma_addr[GEN8_LEGACY_PDPES ]; - }; -union { -dma_addr_t *pt_dma_addr; -dma_addr_t *gen8_pt_dma_addr[GEN8_LEGACY_PDPES ]; - }; -union {struct i915_page_directory_pointer_entry pdp;struct i915_page_directory_entry pd; };drivers/gpu/drm/i915/intel_lrc.c +8
-8@@ -1735,14 +1735,14 @@ populate_lr_context(struct intel_context *ctx, struct drm_i915_gem_object *ctx_o reg_state[CTX_PDP1_LDW ] =GEN8_RING_PDP_LDW (ring,1 ); reg_state[CTX_PDP0_UDW ] =GEN8_RING_PDP_UDW (ring,0 ); reg_state[CTX_PDP0_LDW ] =GEN8_RING_PDP_LDW (ring,0 ); - reg_state[CTX_PDP3_UDW +1 ] =upper_32_bits (ppgtt->pd_dma_addr[3 ]); - reg_state[CTX_PDP3_LDW +1 ] =lower_32_bits (ppgtt->pd_dma_addr[3 ]); - reg_state[CTX_PDP2_UDW +1 ] =upper_32_bits (ppgtt->pd_dma_addr[2 ]); - reg_state[CTX_PDP2_LDW +1 ] =lower_32_bits (ppgtt->pd_dma_addr[2 ]); - reg_state[CTX_PDP1_UDW +1 ] =upper_32_bits (ppgtt->pd_dma_addr[1 ]); - reg_state[CTX_PDP1_LDW +1 ] =lower_32_bits (ppgtt->pd_dma_addr[1 ]); - reg_state[CTX_PDP0_UDW +1 ] =upper_32_bits (ppgtt->pd_dma_addr[0 ]); - reg_state[CTX_PDP0_LDW +1 ] =lower_32_bits (ppgtt->pd_dma_addr[0 ]); + reg_state[CTX_PDP3_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[3 ].daddr); + reg_state[CTX_PDP3_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[3 ].daddr); + reg_state[CTX_PDP2_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[2 ].daddr); + reg_state[CTX_PDP2_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[2 ].daddr); + reg_state[CTX_PDP1_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[1 ].daddr); + reg_state[CTX_PDP1_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[1 ].daddr); + reg_state[CTX_PDP0_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[0 ].daddr); + reg_state[CTX_PDP0_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[0 ].daddr);if (ring->id ==RCS ) { reg_state[CTX_LRI_HEADER_2 ] =MI_LOAD_REGISTER_IMM (1 ); reg_state[CTX_R_PWR_CLK_STATE ] =0x20c8 ;Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
_______________________________________________
Intel-gfx mailing list
[email protected]
http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH v5 3/32 drm/i915: Create page table allocators
In reply to Michel ThierryFrom: Ben Widawsky <[email protected]>
As we move toward dynamic page table allocation, it becomes much easier to manage our data structures if break do things less coarsely by breaking up all of our actions into individual tasks. This makes the code easier to write, read, and verify.
Aside from the dissection of the allocation functions, the patch statically allocates the page table structures without a page directory. This remains the same for all platforms,
The patch itself should not have much functional difference. The primary noticeable difference is the fact that page tables are no longer allocated, but rather statically declared as part of the page directory. This has non-zero overhead, but things gain non-trivial complexity as a result.
This patch exists for a few reasons:
1. Splitting out the functions allows easily combining GEN6 and GEN8 code. Page tables have no difference based on GEN8. As we'll see in a future patch when we add the DMA mappings to the allocations, it requires only one small change to make work, and error handling should just fall into place.
2. Unless we always want to allocate all page tables under a given PDE, we'll have to eventually break this up into an array of pointers (or pointer to pointer).
3. Having the discrete functions is easier to review, and understand. All allocations and frees now take place in just a couple of locations. Reviewing, and catching leaks should be easy.
4. Less important: the GFP flags are confined to one location, which makes playing around with such things trivial.
v2: Updated commit message to explain why this patch exists
v3: For lrc, s/pdp.page_directory[i].daddr/pdp.page_directory[i]->daddr/
v4: Renamed free_pt/pd_single functions to unmap_and_free_pt/pd (Daniel)
v5: Added additional safety checks in gen8 clear/free/unmap.
v6: Use WARN_ON and return -EINVAL in alloc_pt_range (Mika).
--- drivers/gpu/drm/i915/i915_gem_gtt.c | 252 ++++++++++++++++++++++++------------ drivers/gpu/drm/i915/i915_gem_gtt.h | 4 +- drivers/gpu/drm/i915/intel_lrc.c | 16 +-- 3 files changed, 178 insertions(+), 94 deletions(-)
drivers/gpu/drm/i915/i915_gem_gtt.c +168 -84
@@ -279,6 +279,98 @@ static gen6_gtt_pte_t iris_pte_encode(dma_addr_t addr,return pte; } +static void unmap_and_free_pt (struct i915_page_table_entry *pt) +{ +if (WARN_ON (!pt->page)) +return ; +__free_page (pt->page); +kfree (pt); +} + +static struct i915_page_table_entry *alloc_pt_single (void ) +{ +struct i915_page_table_entry *pt; + + pt =kzalloc (sizeof (*pt),GFP_KERNEL ); +if (!pt) +return ERR_PTR (-ENOMEM ); + + pt->page =alloc_page (GFP_KERNEL | __GFP_ZERO); +if (!pt->page) { +kfree (pt); +return ERR_PTR (-ENOMEM ); + } + +return pt; +} + +/** +* alloc_pt_range() - Allocate a multiple page tables +* @pd: The page directory which will have at least @count entries +* available to point to the allocated page tables. +* @pde: First page directory entry for which we are allocating. +* @count: Number of pages to allocate. +* +* Allocates multiple page table pages and sets the appropriate entries in the +* page table structure within the page directory. Function cleans up after +* itself on any failures. +* +* Return: 0 if allocation succeeded. +*/ +static int alloc_pt_range (struct i915_page_directory_entry *pd,uint16_t pde,size_t count) +{ +int i, ret; + +/* 512 is the max page tables per page_directory on any platform. */ +if (WARN_ON (pde + count >GEN6_PPGTT_PD_ENTRIES )) +return -EINVAL ; + +for (i = pde; i < pde + count; i++) { +struct i915_page_table_entry *pt =alloc_pt_single (); + +if (IS_ERR (pt)) { + ret =PTR_ERR (pt); + goto err_out; + } +WARN (pd->page_tables[i], +"Leaking page directory entry %d (%pa)\n" , + i, pd->page_tables[i]); + pd->page_tables[i] = pt; + } + +return 0 ; + +err_out: +while (i--) +unmap_and_free_pt (pd->page_tables[i]); +return ret; +} + +static void unmap_and_free_pd (struct i915_page_directory_entry *pd) +{ +if (pd->page) { +__free_page (pd->page); +kfree (pd); + } +} + +static struct i915_page_directory_entry *alloc_pd_single (void ) +{ +struct i915_page_directory_entry *pd; + + pd =kzalloc (sizeof (*pd),GFP_KERNEL ); +if (!pd) +return ERR_PTR (-ENOMEM ); + + pd->page =alloc_page (GFP_KERNEL | __GFP_ZERO); +if (!pd->page) { +kfree (pd); +return ERR_PTR (-ENOMEM ); + } + +return pd; +} +/* Broadwell Page Directory Pointer Descriptors */ static int gen8_write_pdp (struct intel_engine_cs *ring,unsigned entry,uint64_t val)
@@ -311,7 +403,7 @@ static int gen8_mm_switch(struct i915_hw_ppgtt *ppgtt,int used_pd = ppgtt->num_pd_entries /GEN8_PDES_PER_PAGE ;for (i = used_pd -1 ; i >=0 ; i--) { -dma_addr_t addr = ppgtt->pdp.page_directory[i].daddr; +dma_addr_t addr = ppgtt->pdp.page_directory[i]->daddr; ret =gen8_write_pdp (ring, i, addr);if (ret)return ret;
@@ -338,8 +430,24 @@ static void gen8_ppgtt_clear_range(struct i915_address_space *vm, I915_CACHE_LLC, use_scratch);while (num_entries) { -struct i915_page_directory_entry *pd = &ppgtt->pdp.page_directory[pdpe]; -struct page *page_table = pd->page_tables[pde].page; +struct i915_page_directory_entry *pd; +struct i915_page_table_entry *pt; +struct page *page_table; + +if (WARN_ON (!ppgtt->pdp.page_directory[pdpe])) +continue ; + + pd = ppgtt->pdp.page_directory[pdpe]; + +if (WARN_ON (!pd->page_tables[pde])) +continue ; + + pt = pd->page_tables[pde]; + +if (WARN_ON (!pt->page)) +continue ; + + page_table = pt->page; last_pte = pte + num_entries;if (last_pte >GEN8_PTES_PER_PAGE )
@@ -384,8 +492,9 @@ static void gen8_ppgtt_insert_entries(struct i915_address_space *vm,break ;if (pt_vaddr ==NULL ) { -struct i915_page_directory_entry *pd = &ppgtt->pdp.page_directory[pdpe]; -struct page *page_table = pd->page_tables[pde].page; +struct i915_page_directory_entry *pd = ppgtt->pdp.page_directory[pdpe]; +struct i915_page_table_entry *pt = pd->page_tables[pde]; +struct page *page_table = pt->page; pt_vaddr =kmap_atomic (page_table); }
@@ -416,19 +525,16 @@ static void gen8_free_page_tables(struct i915_page_directory_entry *pd) {int i; -if (pd->page_tables ==NULL ) +if (!pd->page)return ; -for (i =0 ; i <GEN8_PDES_PER_PAGE ; i++) -if (pd->page_tables[i].page) -__free_page (pd->page_tables[i].page); -} +for (i =0 ; i <GEN8_PDES_PER_PAGE ; i++) { +if (WARN_ON (!pd->page_tables[i])) +continue ; -static void gen8_free_page_directory (struct i915_page_directory_entry *pd) -{ -gen8_free_page_tables (pd); -kfree (pd->page_tables); -__free_page (pd->page); +unmap_and_free_pt (pd->page_tables[i]); + pd->page_tables[i] =NULL ; + } }static void gen8_ppgtt_free (struct i915_hw_ppgtt *ppgtt)
@@ -436,7 +542,11 @@ static void gen8_ppgtt_free(struct i915_hw_ppgtt *ppgtt)int i;for (i =0 ; i < ppgtt->num_pd_pages; i++) { -gen8_free_page_directory (&ppgtt->pdp.page_directory[i]); +if (WARN_ON (!ppgtt->pdp.page_directory[i])) +continue ; + +gen8_free_page_tables (ppgtt->pdp.page_directory[i]); +unmap_and_free_pd (ppgtt->pdp.page_directory[i]); } }
@@ -448,14 +558,23 @@ static void gen8_ppgtt_unmap_pages(struct i915_hw_ppgtt *ppgtt)for (i =0 ; i < ppgtt->num_pd_pages; i++) {/* TODO: In the future we'll support sparse mappings, so this * will have to change. */ -if (!ppgtt->pdp.page_directory[i].daddr) +if (!ppgtt->pdp.page_directory[i]->daddr)continue ; -pci_unmap_page (hwdev, ppgtt->pdp.page_directory[i].daddr,PAGE_SIZE , +pci_unmap_page (hwdev, ppgtt->pdp.page_directory[i]->daddr,PAGE_SIZE ,PCI_DMA_BIDIRECTIONAL );for (j =0 ; j <GEN8_PDES_PER_PAGE ; j++) { -dma_addr_t addr = ppgtt->pdp.page_directory[i].page_tables[j].daddr; +struct i915_page_directory_entry *pd = ppgtt->pdp.page_directory[i]; +struct i915_page_table_entry *pt; +dma_addr_t addr; + +if (WARN_ON (!pd->page_tables[j])) +continue ; + + pt = pd->page_tables[j]; + addr = pt->daddr; +if (addr)pci_unmap_page (hwdev, addr,PAGE_SIZE ,PCI_DMA_BIDIRECTIONAL );
@@ -474,25 +593,20 @@ static void gen8_ppgtt_cleanup(struct i915_address_space *vm)static int gen8_ppgtt_allocate_page_tables (struct i915_hw_ppgtt *ppgtt) { -int i, j; +int i, ret;for (i =0 ; i < ppgtt->num_pd_pages; i++) { -struct i915_page_directory_entry *pd = &ppgtt->pdp.page_directory[i]; -for (j =0 ; j <GEN8_PDES_PER_PAGE ; j++) { -struct i915_page_table_entry *pt = &pd->page_tables[j]; - - pt->page =alloc_page (GFP_KERNEL | __GFP_ZERO); -if (!pt->page) - goto unwind_out; - - } + ret =alloc_pt_range (ppgtt->pdp.page_directory[i], +0 ,GEN8_PDES_PER_PAGE ); +if (ret) + goto unwind_out; }return 0 ; unwind_out:while (i--) -gen8_free_page_tables (&ppgtt->pdp.page_directory[i]); +gen8_free_page_tables (ppgtt->pdp.page_directory[i]);return -ENOMEM ; }
@@ -503,19 +617,9 @@ static int gen8_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt,int i;for (i =0 ; i < max_pdp; i++) { -struct i915_page_table_entry *pt; - - pt =kcalloc (GEN8_PDES_PER_PAGE ,sizeof (*pt),GFP_KERNEL ); -if (!pt) + ppgtt->pdp.page_directory[i] =alloc_pd_single (); +if (IS_ERR (ppgtt->pdp.page_directory[i])) goto unwind_out; - - ppgtt->pdp.page_directory[i].page =alloc_page (GFP_KERNEL ); -if (!ppgtt->pdp.page_directory[i].page) { -kfree (pt); - goto unwind_out; - } - - ppgtt->pdp.page_directory[i].page_tables = pt; } ppgtt->num_pd_pages = max_pdp;
@@ -524,10 +628,8 @@ static int gen8_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt,return 0 ; unwind_out: -while (i--) { -kfree (ppgtt->pdp.page_directory[i].page_tables); -__free_page (ppgtt->pdp.page_directory[i].page); - } +while (i--) +unmap_and_free_pd (ppgtt->pdp.page_directory[i]);return -ENOMEM ; }
@@ -561,14 +663,14 @@ static int gen8_ppgtt_setup_page_directories(struct i915_hw_ppgtt *ppgtt,int ret; pd_addr =pci_map_page (ppgtt->base.dev->pdev, - ppgtt->pdp.page_directory[pd].page,0 , + ppgtt->pdp.page_directory[pd]->page,0 ,PAGE_SIZE ,PCI_DMA_BIDIRECTIONAL ); ret =pci_dma_mapping_error (ppgtt->base.dev->pdev, pd_addr);if (ret)return ret; - ppgtt->pdp.page_directory[pd].daddr = pd_addr; + ppgtt->pdp.page_directory[pd]->daddr = pd_addr;return 0 ; }
@@ -578,8 +680,8 @@ static int gen8_ppgtt_setup_page_tables(struct i915_hw_ppgtt *ppgtt, const int pt) {dma_addr_t pt_addr; -struct i915_page_directory_entry *pdir = &ppgtt->pdp.page_directory[pd]; -struct i915_page_table_entry *ptab = &pdir->page_tables[pt]; +struct i915_page_directory_entry *pdir = ppgtt->pdp.page_directory[pd]; +struct i915_page_table_entry *ptab = pdir->page_tables[pt];struct page *p = ptab->page;int ret;
@@ -642,10 +744,12 @@ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size)* will never need to touch the PDEs again. */ for (i =0 ; i < max_pdp; i++) { +struct i915_page_directory_entry *pd = ppgtt->pdp.page_directory[i];gen8_ppgtt_pde_t *pd_vaddr; - pd_vaddr =kmap_atomic (ppgtt->pdp.page_directory[i].page); + pd_vaddr =kmap_atomic (ppgtt->pdp.page_directory[i]->page);for (j =0 ; j <GEN8_PDES_PER_PAGE ; j++) { -dma_addr_t addr = ppgtt->pdp.page_directory[i].page_tables[j].daddr; +struct i915_page_table_entry *pt = pd->page_tables[j]; +dma_addr_t addr = pt->daddr; pd_vaddr[j] =gen8_pde_encode (ppgtt->base.dev, addr,I915_CACHE_LLC ); }
@@ -696,7 +800,7 @@ static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m)for (pde =0 ; pde < ppgtt->num_pd_entries; pde++) {u32 expected;gen6_gtt_pte_t *pt_vaddr; -dma_addr_t pt_addr = ppgtt->pd.page_tables[pde].daddr; +dma_addr_t pt_addr = ppgtt->pd.page_tables[pde]->daddr; pd_entry =readl (pd_addr + pde); expected = (GEN6_PDE_ADDR_ENCODE (pt_addr) |GEN6_PDE_VALID );
@@ -707,7 +811,7 @@ static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m) expected);seq_printf (m,"\tPDE: %x\n" , pd_entry); - pt_vaddr =kmap_atomic (ppgtt->pd.page_tables[pde].page); + pt_vaddr =kmap_atomic (ppgtt->pd.page_tables[pde]->page);for (pte =0 ; pte <I915_PPGTT_PT_ENTRIES ; pte+=4 ) {unsigned long va = (pde * PAGE_SIZE * I915_PPGTT_PT_ENTRIES) +
@@ -746,7 +850,7 @@ static void gen6_write_pdes(struct i915_hw_ppgtt *ppgtt)for (i =0 ; i < ppgtt->num_pd_entries; i++) {dma_addr_t pt_addr; - pt_addr = ppgtt->pd.page_tables[i].daddr; + pt_addr = ppgtt->pd.page_tables[i]->daddr; pd_entry =GEN6_PDE_ADDR_ENCODE (pt_addr); pd_entry |=GEN6_PDE_VALID ;
@@ -922,7 +1026,7 @@ static void gen6_ppgtt_clear_range(struct i915_address_space *vm,if (last_pte >I915_PPGTT_PT_ENTRIES ) last_pte =I915_PPGTT_PT_ENTRIES ; - pt_vaddr =kmap_atomic (ppgtt->pd.page_tables[act_pt].page); + pt_vaddr =kmap_atomic (ppgtt->pd.page_tables[act_pt]->page);for (i = first_pte; i < last_pte; i++) pt_vaddr[i] = scratch_pte;
@@ -951,7 +1055,7 @@ static void gen6_ppgtt_insert_entries(struct i915_address_space *vm, pt_vaddr =NULL ;for_each_sg_page (pages->sgl, &sg_iter, pages->nents,0 ) {if (pt_vaddr ==NULL ) - pt_vaddr =kmap_atomic (ppgtt->pd.page_tables[act_pt].page); + pt_vaddr =kmap_atomic (ppgtt->pd.page_tables[act_pt]->page); pt_vaddr[act_pte] = vm->pte_encode(sg_page_iter_dma_address(&sg_iter),
@@ -974,7 +1078,7 @@ static void gen6_ppgtt_unmap_pages(struct i915_hw_ppgtt *ppgtt)for (i =0 ; i < ppgtt->num_pd_entries; i++)pci_unmap_page (ppgtt->base.dev->pdev, - ppgtt->pd.page_tables[i].daddr, + ppgtt->pd.page_tables[i]->daddr,4096 ,PCI_DMA_BIDIRECTIONAL ); }
@@ -983,8 +1087,9 @@ static void gen6_ppgtt_free(struct i915_hw_ppgtt *ppgtt)int i;for (i =0 ; i < ppgtt->num_pd_entries; i++) -__free_page (ppgtt->pd.page_tables[i].page); -kfree (ppgtt->pd.page_tables); +unmap_and_free_pt (ppgtt->pd.page_tables[i]); + +unmap_and_free_pd (&ppgtt->pd); }static void gen6_ppgtt_cleanup (struct i915_address_space *vm)
@@ -1039,27 +1144,6 @@ alloc:return 0 ; } -static int gen6_ppgtt_allocate_page_tables (struct i915_hw_ppgtt *ppgtt) -{ -struct i915_page_table_entry *pt; -int i; - - pt =kcalloc (ppgtt->num_pd_entries,sizeof (*pt),GFP_KERNEL ); -if (!pt) -return -ENOMEM ; - -for (i =0 ; i < ppgtt->num_pd_entries; i++) { - pt[i].page =alloc_page (GFP_KERNEL ); -if (!pt->page) { -gen6_ppgtt_free (ppgtt); -return -ENOMEM ; - } - } - - ppgtt->pd.page_tables = pt; -return 0 ; -} -static int gen6_ppgtt_alloc (struct i915_hw_ppgtt *ppgtt) {int ret;
@@ -1068,7 +1152,7 @@ static int gen6_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt)if (ret)return ret; - ret =gen6_ppgtt_allocate_page_tables (ppgtt); + ret =alloc_pt_range (&ppgtt->pd,0 , ppgtt->num_pd_entries);if (ret) {drm_mm_remove_node (&ppgtt->node);return ret;
@@ -1086,7 +1170,7 @@ static int gen6_ppgtt_setup_page_tables(struct i915_hw_ppgtt *ppgtt)struct page *page;dma_addr_t pt_addr; - page = ppgtt->pd.page_tables[i].page; + page = ppgtt->pd.page_tables[i]->page; pt_addr =pci_map_page (dev->pdev, page,0 ,4096 ,PCI_DMA_BIDIRECTIONAL );
@@ -1095,7 +1179,7 @@ static int gen6_ppgtt_setup_page_tables(struct i915_hw_ppgtt *ppgtt)return -EIO ; } - ppgtt->pd.page_tables[i].daddr = pt_addr; + ppgtt->pd.page_tables[i]->daddr = pt_addr; }return 0 ;
drivers/gpu/drm/i915/i915_gem_gtt.h +2 -2
@@ -199,12 +199,12 @@ struct i915_page_directory_entry {dma_addr_t daddr; }; -struct i915_page_table_entry *page_tables; +struct i915_page_table_entry *page_tables[GEN6_PPGTT_PD_ENTRIES ];/* PDEs */ };struct i915_page_directory_pointer_entry {/* struct page *page; */ -struct i915_page_directory_entry page_directory[GEN8_LEGACY_PDPES ]; +struct i915_page_directory_entry *page_directory[GEN8_LEGACY_PDPES ]; };struct i915_address_space {
drivers/gpu/drm/i915/intel_lrc.c +8 -8
@@ -1735,14 +1735,14 @@ populate_lr_context(struct intel_context *ctx, struct drm_i915_gem_object *ctx_o reg_state[CTX_PDP1_LDW ] =GEN8_RING_PDP_LDW (ring,1 ); reg_state[CTX_PDP0_UDW ] =GEN8_RING_PDP_UDW (ring,0 ); reg_state[CTX_PDP0_LDW ] =GEN8_RING_PDP_LDW (ring,0 ); - reg_state[CTX_PDP3_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[3 ].daddr); - reg_state[CTX_PDP3_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[3 ].daddr); - reg_state[CTX_PDP2_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[2 ].daddr); - reg_state[CTX_PDP2_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[2 ].daddr); - reg_state[CTX_PDP1_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[1 ].daddr); - reg_state[CTX_PDP1_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[1 ].daddr); - reg_state[CTX_PDP0_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[0 ].daddr); - reg_state[CTX_PDP0_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[0 ].daddr); + reg_state[CTX_PDP3_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[3 ]->daddr); + reg_state[CTX_PDP3_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[3 ]->daddr); + reg_state[CTX_PDP2_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[2 ]->daddr); + reg_state[CTX_PDP2_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[2 ]->daddr); + reg_state[CTX_PDP1_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[1 ]->daddr); + reg_state[CTX_PDP1_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[1 ]->daddr); + reg_state[CTX_PDP0_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[0 ]->daddr); + reg_state[CTX_PDP0_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[0 ]->daddr);if (ring->id ==RCS ) { reg_state[CTX_LRI_HEADER_2 ] =MI_LOAD_REGISTER_IMM (1 ); reg_state[CTX_R_PWR_CLK_STATE ] =0x20c8 ;
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
Re: [PATCH v5 03/32] drm/i915: Create page table allocators
Quoted 11 lines · Michel Thierry
From: Ben Widawsky <[email protected]>
As we move toward dynamic page table allocation, it becomes much easier to manage our data structures if break do things less coarsely by breaking up all of our actions into individual tasks. This makes the code easier to write, read, and verify.
Aside from the dissection of the allocation functions, the patch statically allocates the page table structures without a page directory. This remains the same for all platforms,
Quoted 6 lines
The patch itself should not have much functional difference. The primary noticeable difference is the fact that page tables are no longer allocated, but rather statically declared as part of the page directory. This has non-zero overhead, but things gain non-trivial complexity as a result.
I don't quite understand the last sentence here. We gain overhead and complexity.
s/non-trivial/trivial?
Quoted 99 lines
This patch exists for a few reasons:
1. Splitting out the functions allows easily combining GEN6 and GEN8 code. Page tables have no difference based on GEN8. As we'll see in a future patch when we add the DMA mappings to the allocations, it requires only one small change to make work, and error handling should just fall into place.2. Unless we always want to allocate all page tables under a given PDE, we'll have to eventually break this up into an array of pointers (or pointer to pointer).
3. Having the discrete functions is easier to review, and understand. All allocations and frees now take place in just a couple of locations. Reviewing, and catching leaks should be easy.
4. Less important: the GFP flags are confined to one location, which makes playing around with such things trivial.
v2: Updated commit message to explain why this patch exists
v3: For lrc, s/pdp.page_directory[i].daddr/pdp.page_directory[i]->daddr/
v4: Renamed free_pt/pd_single functions to unmap_and_free_pt/pd (Daniel)
v5: Added additional safety checks in gen8 clear/free/unmap.
v6: Use
WARN_ONand return-EINVALinalloc_pt_range(Mika).Cc Mika Kuoppala <[email protected]>Signed-off-by Ben Widawsky <[email protected]>Signed-off-by Michel Thierry <[email protected]> (v3+)--- drivers/gpu/drm/i915/i915_gem_gtt.c | 252 ++++++++++++++++++++++++------------ drivers/gpu/drm/i915/i915_gem_gtt.h | 4 +- drivers/gpu/drm/i915/intel_lrc.c | 16 +-- 3 files changed, 178 insertions(+), 94 deletions(-)drivers/gpu/drm/i915/i915_gem_gtt.c +54
-0@@ -279,6 +279,98 @@ static gen6_gtt_pte_t iris_pte_encode(dma_addr_t addr,return pte; } +static void unmap_and_free_pt (struct i915_page_table_entry *pt) +{ +if (WARN_ON (!pt->page)) +return ; +__free_page (pt->page); +kfree (pt); +} + +static struct i915_page_table_entry *alloc_pt_single (void ) +{ +struct i915_page_table_entry *pt; + + pt =kzalloc (sizeof (*pt),GFP_KERNEL ); +if (!pt) +return ERR_PTR (-ENOMEM ); + + pt->page =alloc_page (GFP_KERNEL | __GFP_ZERO); +if (!pt->page) { +kfree (pt); +return ERR_PTR (-ENOMEM ); + } + +return pt; +} + +/** +* alloc_pt_range() - Allocate a multiple page tables +* @pd: The page directory which will have at least @count entries +* available to point to the allocated page tables. +* @pde: First page directory entry for which we are allocating. +* @count: Number of pages to allocate. +* +* Allocates multiple page table pages and sets the appropriate entries in the +* page table structure within the page directory. Function cleans up after +* itself on any failures. +* +* Return: 0 if allocation succeeded. +*/ +static int alloc_pt_range (struct i915_page_directory_entry *pd,uint16_t pde,size_t count) +{ +int i, ret; + +/* 512 is the max page tables per page_directory on any platform. */ +if (WARN_ON (pde + count >GEN6_PPGTT_PD_ENTRIES )) +return -EINVAL ; + +for (i = pde; i < pde + count; i++) { +struct i915_page_table_entry *pt =alloc_pt_single (); + +if (IS_ERR (pt)) { + ret =PTR_ERR (pt); + goto err_out; + } + WARN(pd->page_tables[i],
drivers/gpu/drm/i915/i915_gem_gtt.c static gen6_gtt_pte_t iris_pte_encode(dma_addr_t addr, · line 345
Quoted 10 lines
drivers/gpu/drm/i915/i915_gem_gtt.c +10
-0+"Leaking page directory entry %d (%pa)\n" , + i, pd->page_tables[i]); + pd->page_tables[i] = pt; + } + +return 0 ; + +err_out: +while (i--) +unmap_and_free_pt (pd->page_tables[i]);
This is suspicious as it is non symmetrical of how we allocate. If the plan is to free everything below pde, that should be mentioned in the comment above.
On this patch we call this with pde == 0, but I suspect later in the series there will be other usecases for this.
Quoted 382 lines
drivers/gpu/drm/i915/i915_gem_gtt.c +104
-84+return ret; +} + +static void unmap_and_free_pd (struct i915_page_directory_entry *pd) +{ +if (pd->page) { +__free_page (pd->page); +kfree (pd); + } +} + +static struct i915_page_directory_entry *alloc_pd_single (void ) +{ +struct i915_page_directory_entry *pd; + + pd =kzalloc (sizeof (*pd),GFP_KERNEL ); +if (!pd) +return ERR_PTR (-ENOMEM ); + + pd->page =alloc_page (GFP_KERNEL | __GFP_ZERO); +if (!pd->page) { +kfree (pd); +return ERR_PTR (-ENOMEM ); + } + +return pd; +} +/* Broadwell Page Directory Pointer Descriptors */ static int gen8_write_pdp (struct intel_engine_cs *ring,unsigned entry,uint64_t val)@@ -311,7 +403,7 @@ static int gen8_mm_switch(struct i915_hw_ppgtt *ppgtt,int used_pd = ppgtt->num_pd_entries /GEN8_PDES_PER_PAGE ;for (i = used_pd -1 ; i >=0 ; i--) { -dma_addr_t addr = ppgtt->pdp.page_directory[i].daddr; +dma_addr_t addr = ppgtt->pdp.page_directory[i]->daddr; ret =gen8_write_pdp (ring, i, addr);if (ret)return ret;@@ -338,8 +430,24 @@ static void gen8_ppgtt_clear_range(struct i915_address_space *vm, I915_CACHE_LLC, use_scratch);while (num_entries) { -struct i915_page_directory_entry *pd = &ppgtt->pdp.page_directory[pdpe]; -struct page *page_table = pd->page_tables[pde].page; +struct i915_page_directory_entry *pd; +struct i915_page_table_entry *pt; +struct page *page_table; + +if (WARN_ON (!ppgtt->pdp.page_directory[pdpe])) +continue ; + + pd = ppgtt->pdp.page_directory[pdpe]; + +if (WARN_ON (!pd->page_tables[pde])) +continue ; + + pt = pd->page_tables[pde]; + +if (WARN_ON (!pt->page)) +continue ; + + page_table = pt->page; last_pte = pte + num_entries;if (last_pte >GEN8_PTES_PER_PAGE )@@ -384,8 +492,9 @@ static void gen8_ppgtt_insert_entries(struct i915_address_space *vm,break ;if (pt_vaddr ==NULL ) { -struct i915_page_directory_entry *pd = &ppgtt->pdp.page_directory[pdpe]; -struct page *page_table = pd->page_tables[pde].page; +struct i915_page_directory_entry *pd = ppgtt->pdp.page_directory[pdpe]; +struct i915_page_table_entry *pt = pd->page_tables[pde]; +struct page *page_table = pt->page; pt_vaddr =kmap_atomic (page_table); }@@ -416,19 +525,16 @@ static void gen8_free_page_tables(struct i915_page_directory_entry *pd) {int i; -if (pd->page_tables ==NULL ) +if (!pd->page)return ; -for (i =0 ; i <GEN8_PDES_PER_PAGE ; i++) -if (pd->page_tables[i].page) -__free_page (pd->page_tables[i].page); -} +for (i =0 ; i <GEN8_PDES_PER_PAGE ; i++) { +if (WARN_ON (!pd->page_tables[i])) +continue ; -static void gen8_free_page_directory (struct i915_page_directory_entry *pd) -{ -gen8_free_page_tables (pd); -kfree (pd->page_tables); -__free_page (pd->page); +unmap_and_free_pt (pd->page_tables[i]); + pd->page_tables[i] =NULL ; + } }static void gen8_ppgtt_free (struct i915_hw_ppgtt *ppgtt)@@ -436,7 +542,11 @@ static void gen8_ppgtt_free(struct i915_hw_ppgtt *ppgtt)int i;for (i =0 ; i < ppgtt->num_pd_pages; i++) { -gen8_free_page_directory (&ppgtt->pdp.page_directory[i]); +if (WARN_ON (!ppgtt->pdp.page_directory[i])) +continue ; + +gen8_free_page_tables (ppgtt->pdp.page_directory[i]); +unmap_and_free_pd (ppgtt->pdp.page_directory[i]); } }@@ -448,14 +558,23 @@ static void gen8_ppgtt_unmap_pages(struct i915_hw_ppgtt *ppgtt)for (i =0 ; i < ppgtt->num_pd_pages; i++) {/* TODO: In the future we'll support sparse mappings, so this * will have to change. */ -if (!ppgtt->pdp.page_directory[i].daddr) +if (!ppgtt->pdp.page_directory[i]->daddr)continue ; -pci_unmap_page (hwdev, ppgtt->pdp.page_directory[i].daddr,PAGE_SIZE , +pci_unmap_page (hwdev, ppgtt->pdp.page_directory[i]->daddr,PAGE_SIZE ,PCI_DMA_BIDIRECTIONAL );for (j =0 ; j <GEN8_PDES_PER_PAGE ; j++) { -dma_addr_t addr = ppgtt->pdp.page_directory[i].page_tables[j].daddr; +struct i915_page_directory_entry *pd = ppgtt->pdp.page_directory[i]; +struct i915_page_table_entry *pt; +dma_addr_t addr; + +if (WARN_ON (!pd->page_tables[j])) +continue ; + + pt = pd->page_tables[j]; + addr = pt->daddr; +if (addr)pci_unmap_page (hwdev, addr,PAGE_SIZE ,PCI_DMA_BIDIRECTIONAL );@@ -474,25 +593,20 @@ static void gen8_ppgtt_cleanup(struct i915_address_space *vm)static int gen8_ppgtt_allocate_page_tables (struct i915_hw_ppgtt *ppgtt) { -int i, j; +int i, ret;for (i =0 ; i < ppgtt->num_pd_pages; i++) { -struct i915_page_directory_entry *pd = &ppgtt->pdp.page_directory[i]; -for (j =0 ; j <GEN8_PDES_PER_PAGE ; j++) { -struct i915_page_table_entry *pt = &pd->page_tables[j]; - - pt->page =alloc_page (GFP_KERNEL | __GFP_ZERO); -if (!pt->page) - goto unwind_out; - - } + ret =alloc_pt_range (ppgtt->pdp.page_directory[i], +0 ,GEN8_PDES_PER_PAGE ); +if (ret) + goto unwind_out; }return 0 ; unwind_out:while (i--) -gen8_free_page_tables (&ppgtt->pdp.page_directory[i]); +gen8_free_page_tables (ppgtt->pdp.page_directory[i]);return -ENOMEM ; }@@ -503,19 +617,9 @@ static int gen8_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt,int i;for (i =0 ; i < max_pdp; i++) { -struct i915_page_table_entry *pt; - - pt =kcalloc (GEN8_PDES_PER_PAGE ,sizeof (*pt),GFP_KERNEL ); -if (!pt) + ppgtt->pdp.page_directory[i] =alloc_pd_single (); +if (IS_ERR (ppgtt->pdp.page_directory[i])) goto unwind_out; - - ppgtt->pdp.page_directory[i].page =alloc_page (GFP_KERNEL ); -if (!ppgtt->pdp.page_directory[i].page) { -kfree (pt); - goto unwind_out; - } - - ppgtt->pdp.page_directory[i].page_tables = pt; } ppgtt->num_pd_pages = max_pdp;@@ -524,10 +628,8 @@ static int gen8_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt,return 0 ; unwind_out: -while (i--) { -kfree (ppgtt->pdp.page_directory[i].page_tables); -__free_page (ppgtt->pdp.page_directory[i].page); - } +while (i--) +unmap_and_free_pd (ppgtt->pdp.page_directory[i]);return -ENOMEM ; }@@ -561,14 +663,14 @@ static int gen8_ppgtt_setup_page_directories(struct i915_hw_ppgtt *ppgtt,int ret; pd_addr =pci_map_page (ppgtt->base.dev->pdev, - ppgtt->pdp.page_directory[pd].page,0 , + ppgtt->pdp.page_directory[pd]->page,0 ,PAGE_SIZE ,PCI_DMA_BIDIRECTIONAL ); ret =pci_dma_mapping_error (ppgtt->base.dev->pdev, pd_addr);if (ret)return ret; - ppgtt->pdp.page_directory[pd].daddr = pd_addr; + ppgtt->pdp.page_directory[pd]->daddr = pd_addr;return 0 ; }@@ -578,8 +680,8 @@ static int gen8_ppgtt_setup_page_tables(struct i915_hw_ppgtt *ppgtt, const int pt) {dma_addr_t pt_addr; -struct i915_page_directory_entry *pdir = &ppgtt->pdp.page_directory[pd]; -struct i915_page_table_entry *ptab = &pdir->page_tables[pt]; +struct i915_page_directory_entry *pdir = ppgtt->pdp.page_directory[pd]; +struct i915_page_table_entry *ptab = pdir->page_tables[pt];struct page *p = ptab->page;int ret;@@ -642,10 +744,12 @@ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size)* will never need to touch the PDEs again. */ for (i =0 ; i < max_pdp; i++) { +struct i915_page_directory_entry *pd = ppgtt->pdp.page_directory[i];gen8_ppgtt_pde_t *pd_vaddr; - pd_vaddr =kmap_atomic (ppgtt->pdp.page_directory[i].page); + pd_vaddr =kmap_atomic (ppgtt->pdp.page_directory[i]->page);for (j =0 ; j <GEN8_PDES_PER_PAGE ; j++) { -dma_addr_t addr = ppgtt->pdp.page_directory[i].page_tables[j].daddr; +struct i915_page_table_entry *pt = pd->page_tables[j]; +dma_addr_t addr = pt->daddr; pd_vaddr[j] =gen8_pde_encode (ppgtt->base.dev, addr,I915_CACHE_LLC ); }@@ -696,7 +800,7 @@ static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m)for (pde =0 ; pde < ppgtt->num_pd_entries; pde++) {u32 expected;gen6_gtt_pte_t *pt_vaddr; -dma_addr_t pt_addr = ppgtt->pd.page_tables[pde].daddr; +dma_addr_t pt_addr = ppgtt->pd.page_tables[pde]->daddr; pd_entry =readl (pd_addr + pde); expected = (GEN6_PDE_ADDR_ENCODE (pt_addr) |GEN6_PDE_VALID );@@ -707,7 +811,7 @@ static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m) expected);seq_printf (m,"\tPDE: %x\n" , pd_entry); - pt_vaddr =kmap_atomic (ppgtt->pd.page_tables[pde].page); + pt_vaddr =kmap_atomic (ppgtt->pd.page_tables[pde]->page);for (pte =0 ; pte <I915_PPGTT_PT_ENTRIES ; pte+=4 ) {unsigned long va = (pde * PAGE_SIZE * I915_PPGTT_PT_ENTRIES) +@@ -746,7 +850,7 @@ static void gen6_write_pdes(struct i915_hw_ppgtt *ppgtt)for (i =0 ; i < ppgtt->num_pd_entries; i++) {dma_addr_t pt_addr; - pt_addr = ppgtt->pd.page_tables[i].daddr; + pt_addr = ppgtt->pd.page_tables[i]->daddr; pd_entry =GEN6_PDE_ADDR_ENCODE (pt_addr); pd_entry |=GEN6_PDE_VALID ;@@ -922,7 +1026,7 @@ static void gen6_ppgtt_clear_range(struct i915_address_space *vm,if (last_pte >I915_PPGTT_PT_ENTRIES ) last_pte =I915_PPGTT_PT_ENTRIES ; - pt_vaddr =kmap_atomic (ppgtt->pd.page_tables[act_pt].page); + pt_vaddr =kmap_atomic (ppgtt->pd.page_tables[act_pt]->page);for (i = first_pte; i < last_pte; i++) pt_vaddr[i] = scratch_pte;@@ -951,7 +1055,7 @@ static void gen6_ppgtt_insert_entries(struct i915_address_space *vm, pt_vaddr =NULL ;for_each_sg_page (pages->sgl, &sg_iter, pages->nents,0 ) {if (pt_vaddr ==NULL ) - pt_vaddr =kmap_atomic (ppgtt->pd.page_tables[act_pt].page); + pt_vaddr =kmap_atomic (ppgtt->pd.page_tables[act_pt]->page); pt_vaddr[act_pte] = vm->pte_encode(sg_page_iter_dma_address(&sg_iter),@@ -974,7 +1078,7 @@ static void gen6_ppgtt_unmap_pages(struct i915_hw_ppgtt *ppgtt)for (i =0 ; i < ppgtt->num_pd_entries; i++)pci_unmap_page (ppgtt->base.dev->pdev, - ppgtt->pd.page_tables[i].daddr, + ppgtt->pd.page_tables[i]->daddr,4096 ,PCI_DMA_BIDIRECTIONAL ); }@@ -983,8 +1087,9 @@ static void gen6_ppgtt_free(struct i915_hw_ppgtt *ppgtt)int i;for (i =0 ; i < ppgtt->num_pd_entries; i++) -__free_page (ppgtt->pd.page_tables[i].page); -kfree (ppgtt->pd.page_tables); +unmap_and_free_pt (ppgtt->pd.page_tables[i]); + +unmap_and_free_pd (&ppgtt->pd); }static void gen6_ppgtt_cleanup (struct i915_address_space *vm)@@ -1039,27 +1144,6 @@ alloc:return 0 ; } -static int gen6_ppgtt_allocate_page_tables (struct i915_hw_ppgtt *ppgtt) -{ -struct i915_page_table_entry *pt; -int i; - - pt =kcalloc (ppgtt->num_pd_entries,sizeof (*pt),GFP_KERNEL ); -if (!pt) -return -ENOMEM ; - -for (i =0 ; i < ppgtt->num_pd_entries; i++) { - pt[i].page =alloc_page (GFP_KERNEL ); -if (!pt->page) { -gen6_ppgtt_free (ppgtt); -return -ENOMEM ; - } - } - - ppgtt->pd.page_tables = pt; -return 0 ; -} -static int gen6_ppgtt_alloc (struct i915_hw_ppgtt *ppgtt) {int ret;@@ -1068,7 +1152,7 @@ static int gen6_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt)if (ret)return ret; - ret =gen6_ppgtt_allocate_page_tables (ppgtt); + ret =alloc_pt_range (&ppgtt->pd,0 , ppgtt->num_pd_entries);if (ret) {drm_mm_remove_node (&ppgtt->node);return ret;@@ -1086,7 +1170,7 @@ static int gen6_ppgtt_setup_page_tables(struct i915_hw_ppgtt *ppgtt)struct page *page;dma_addr_t pt_addr; - page = ppgtt->pd.page_tables[i].page; + page = ppgtt->pd.page_tables[i]->page; pt_addr =pci_map_page (dev->pdev, page,0 ,4096 ,PCI_DMA_BIDIRECTIONAL );@@ -1095,7 +1179,7 @@ static int gen6_ppgtt_setup_page_tables(struct i915_hw_ppgtt *ppgtt)return -EIO ; } - ppgtt->pd.page_tables[i].daddr = pt_addr; + ppgtt->pd.page_tables[i]->daddr = pt_addr; }return 0 ;drivers/gpu/drm/i915/i915_gem_gtt.h +0
-0
drivers/gpu/drm/i915/i915_gem_gtt.h struct i915_page_directory_entry { · line 202
Quoted 5 lines
drivers/gpu/drm/i915/i915_gem_gtt.h +1
-1@@ -199,12 +199,12 @@ struct i915_page_directory_entry {dma_addr_t daddr; }; -struct i915_page_table_entry *page_tables; +struct i915_page_table_entry *page_tables[GEN6_PPGTT_PD_ENTRIES ];/* PDEs */
Would you consider changing the plural here in 'tables' so that we would lose the discrepancy against the page_directory below?
-Mika
Quoted 43 lines
drivers/gpu/drm/i915/i915_gem_gtt.h +1
-1};struct i915_page_directory_pointer_entry {/* struct page *page; */ -struct i915_page_directory_entry page_directory[GEN8_LEGACY_PDPES ]; +struct i915_page_directory_entry *page_directory[GEN8_LEGACY_PDPES ]; };struct i915_address_space {drivers/gpu/drm/i915/intel_lrc.c +8
-8@@ -1735,14 +1735,14 @@ populate_lr_context(struct intel_context *ctx, struct drm_i915_gem_object *ctx_o reg_state[CTX_PDP1_LDW ] =GEN8_RING_PDP_LDW (ring,1 ); reg_state[CTX_PDP0_UDW ] =GEN8_RING_PDP_UDW (ring,0 ); reg_state[CTX_PDP0_LDW ] =GEN8_RING_PDP_LDW (ring,0 ); - reg_state[CTX_PDP3_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[3 ].daddr); - reg_state[CTX_PDP3_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[3 ].daddr); - reg_state[CTX_PDP2_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[2 ].daddr); - reg_state[CTX_PDP2_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[2 ].daddr); - reg_state[CTX_PDP1_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[1 ].daddr); - reg_state[CTX_PDP1_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[1 ].daddr); - reg_state[CTX_PDP0_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[0 ].daddr); - reg_state[CTX_PDP0_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[0 ].daddr); + reg_state[CTX_PDP3_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[3 ]->daddr); + reg_state[CTX_PDP3_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[3 ]->daddr); + reg_state[CTX_PDP2_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[2 ]->daddr); + reg_state[CTX_PDP2_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[2 ]->daddr); + reg_state[CTX_PDP1_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[1 ]->daddr); + reg_state[CTX_PDP1_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[1 ]->daddr); + reg_state[CTX_PDP0_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[0 ]->daddr); + reg_state[CTX_PDP0_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[0 ]->daddr);if (ring->id ==RCS ) { reg_state[CTX_LRI_HEADER_2 ] =MI_LOAD_REGISTER_IMM (1 ); reg_state[CTX_R_PWR_CLK_STATE ] =0x20c8 ;Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
_______________________________________________
Intel-gfx mailing list
[email protected]
http://lists.freedesktop.org/mailman/listinfo/intel-gfx
Re: [PATCH v5 03/32] drm/i915: Create page table allocators
Quoted 19 lines · Mika Kuoppala
Quoted 11 lines · Michel Thierry
From: Ben Widawsky <[email protected]>
As we move toward dynamic page table allocation, it becomes much easier to manage our data structures if break do things less coarsely by breaking up all of our actions into individual tasks. This makes the code easier to write, read, and verify.
Aside from the dissection of the allocation functions, the patch statically allocates the page table structures without a page directory. This remains the same for all platforms,
Quoted 6 lines
The patch itself should not have much functional difference. The primary noticeable difference is the fact that page tables are no longer allocated, but rather statically declared as part of the page directory. This has non-zero overhead, but things gain non-trivial complexity as a result.
Quoted 4 lines
I don't quite understand the last sentence here. We gain overhead and complexity.
s/non-trivial/trivial?
I'll rephrase this.
Quoted 113 lines
Quoted 99 lines
This patch exists for a few reasons:
1. Splitting out the functions allows easily combining GEN6 and GEN8 code. Page tables have no difference based on GEN8. As we'll see in a future patch when we add the DMA mappings to the allocations, it requires only one small change to make work, and error handling should just fall into place.2. Unless we always want to allocate all page tables under a given PDE, we'll have to eventually break this up into an array of pointers (or pointer to pointer).
3. Having the discrete functions is easier to review, and understand. All allocations and frees now take place in just a couple of locations. Reviewing, and catching leaks should be easy.
4. Less important: the GFP flags are confined to one location, which makes playing around with such things trivial.
v2: Updated commit message to explain why this patch exists
v3: For lrc, s/pdp.page_directory[i].daddr/pdp.page_directory[i]->daddr/
v4: Renamed free_pt/pd_single functions to unmap_and_free_pt/pd (Daniel)
v5: Added additional safety checks in gen8 clear/free/unmap.
v6: Use
WARN_ONand return-EINVALinalloc_pt_range(Mika).Cc Mika Kuoppala <[email protected]>Signed-off-by Ben Widawsky <[email protected]>Signed-off-by Michel Thierry <[email protected]> (v3+)--- drivers/gpu/drm/i915/i915_gem_gtt.c | 252 ++++++++++++++++++++++++------------ drivers/gpu/drm/i915/i915_gem_gtt.h | 4 +- drivers/gpu/drm/i915/intel_lrc.c | 16 +-- 3 files changed, 178 insertions(+), 94 deletions(-)drivers/gpu/drm/i915/i915_gem_gtt.c +54
-0@@ -279,6 +279,98 @@ static gen6_gtt_pte_t iris_pte_encode(dma_addr_t addr,return pte; } +static void unmap_and_free_pt (struct i915_page_table_entry *pt) +{ +if (WARN_ON (!pt->page)) +return ; +__free_page (pt->page); +kfree (pt); +} + +static struct i915_page_table_entry *alloc_pt_single (void ) +{ +struct i915_page_table_entry *pt; + + pt =kzalloc (sizeof (*pt),GFP_KERNEL ); +if (!pt) +return ERR_PTR (-ENOMEM ); + + pt->page =alloc_page (GFP_KERNEL | __GFP_ZERO); +if (!pt->page) { +kfree (pt); +return ERR_PTR (-ENOMEM ); + } + +return pt; +} + +/** +* alloc_pt_range() - Allocate a multiple page tables +* @pd: The page directory which will have at least @count entries +* available to point to the allocated page tables. +* @pde: First page directory entry for which we are allocating. +* @count: Number of pages to allocate. +* +* Allocates multiple page table pages and sets the appropriate entries in the +* page table structure within the page directory. Function cleans up after +* itself on any failures. +* +* Return: 0 if allocation succeeded. +*/ +static int alloc_pt_range (struct i915_page_directory_entry *pd,uint16_t pde,size_t count) +{ +int i, ret; + +/* 512 is the max page tables per page_directory on any platform. */ +if (WARN_ON (pde + count >GEN6_PPGTT_PD_ENTRIES )) +return -EINVAL ; + +for (i = pde; i < pde + count; i++) { +struct i915_page_table_entry *pt =alloc_pt_single (); + +if (IS_ERR (pt)) { + ret =PTR_ERR (pt); + goto err_out; + } + WARN(pd->page_tables[i],drivers/gpu/drm/i915/i915_gem_gtt.c static gen6_gtt_pte_t iris_pte_encode(dma_addr_t addr, · line 345
Quoted 10 lines
drivers/gpu/drm/i915/i915_gem_gtt.c +10
-0+"Leaking page directory entry %d (%pa)\n" , + i, pd->page_tables[i]); + pd->page_tables[i] = pt; + } + +return 0 ; + +err_out: +while (i--) +unmap_and_free_pt (pd->page_tables[i]);This is suspicious as it is non symmetrical of how we allocate. If the plan is to free everything below pde, that should be mentioned in the comment above.
Quoted 2 lines
On this patch we call this with pde == 0, but I suspect later in the series there will be other usecases for this.
Actually is the other way around, later on this will be used only by aliasing ppgtt; others will iterate through a macro and call alloc_pt_single ("page table allocation rework" patch).
Anyway, it makes it clearer to have something like:
err_out:
- while (i--)
+ while (i-- > pde)
unmap_and_free_pt(pd->page_tables[i]);
return ret;Quoted 385 lines
Quoted 385 lines
drivers/gpu/drm/i915/i915_gem_gtt.c +104
-84+return ret; +} + +static void unmap_and_free_pd (struct i915_page_directory_entry *pd) +{ +if (pd->page) { +__free_page (pd->page); +kfree (pd); + } +} + +static struct i915_page_directory_entry *alloc_pd_single (void ) +{ +struct i915_page_directory_entry *pd; + + pd =kzalloc (sizeof (*pd),GFP_KERNEL ); +if (!pd) +return ERR_PTR (-ENOMEM ); + + pd->page =alloc_page (GFP_KERNEL | __GFP_ZERO); +if (!pd->page) { +kfree (pd); +return ERR_PTR (-ENOMEM ); + } + +return pd; +} +/* Broadwell Page Directory Pointer Descriptors */ static int gen8_write_pdp (struct intel_engine_cs *ring,unsigned entry,uint64_t val)@@ -311,7 +403,7 @@ static int gen8_mm_switch(struct i915_hw_ppgtt *ppgtt,int used_pd = ppgtt->num_pd_entries /GEN8_PDES_PER_PAGE ;for (i = used_pd -1 ; i >=0 ; i--) { -dma_addr_t addr = ppgtt->pdp.page_directory[i].daddr; +dma_addr_t addr = ppgtt->pdp.page_directory[i]->daddr; ret =gen8_write_pdp (ring, i, addr);if (ret)return ret;@@ -338,8 +430,24 @@ static void gen8_ppgtt_clear_range(struct i915_address_space *vm, I915_CACHE_LLC, use_scratch);while (num_entries) { -struct i915_page_directory_entry *pd = &ppgtt->pdp.page_directory[pdpe]; -struct page *page_table = pd->page_tables[pde].page; +struct i915_page_directory_entry *pd; +struct i915_page_table_entry *pt; +struct page *page_table; + +if (WARN_ON (!ppgtt->pdp.page_directory[pdpe])) +continue ; + + pd = ppgtt->pdp.page_directory[pdpe]; + +if (WARN_ON (!pd->page_tables[pde])) +continue ; + + pt = pd->page_tables[pde]; + +if (WARN_ON (!pt->page)) +continue ; + + page_table = pt->page; last_pte = pte + num_entries;if (last_pte >GEN8_PTES_PER_PAGE )@@ -384,8 +492,9 @@ static void gen8_ppgtt_insert_entries(struct i915_address_space *vm,break ;if (pt_vaddr ==NULL ) { -struct i915_page_directory_entry *pd = &ppgtt->pdp.page_directory[pdpe]; -struct page *page_table = pd->page_tables[pde].page; +struct i915_page_directory_entry *pd = ppgtt->pdp.page_directory[pdpe]; +struct i915_page_table_entry *pt = pd->page_tables[pde]; +struct page *page_table = pt->page; pt_vaddr =kmap_atomic (page_table); }@@ -416,19 +525,16 @@ static void gen8_free_page_tables(struct i915_page_directory_entry *pd) {int i; -if (pd->page_tables ==NULL ) +if (!pd->page)return ; -for (i =0 ; i <GEN8_PDES_PER_PAGE ; i++) -if (pd->page_tables[i].page) -__free_page (pd->page_tables[i].page); -} +for (i =0 ; i <GEN8_PDES_PER_PAGE ; i++) { +if (WARN_ON (!pd->page_tables[i])) +continue ; -static void gen8_free_page_directory (struct i915_page_directory_entry *pd) -{ -gen8_free_page_tables (pd); -kfree (pd->page_tables); -__free_page (pd->page); +unmap_and_free_pt (pd->page_tables[i]); + pd->page_tables[i] =NULL ; + } }static void gen8_ppgtt_free (struct i915_hw_ppgtt *ppgtt)@@ -436,7 +542,11 @@ static void gen8_ppgtt_free(struct i915_hw_ppgtt *ppgtt)int i;for (i =0 ; i < ppgtt->num_pd_pages; i++) { -gen8_free_page_directory (&ppgtt->pdp.page_directory[i]); +if (WARN_ON (!ppgtt->pdp.page_directory[i])) +continue ; + +gen8_free_page_tables (ppgtt->pdp.page_directory[i]); +unmap_and_free_pd (ppgtt->pdp.page_directory[i]); } }@@ -448,14 +558,23 @@ static void gen8_ppgtt_unmap_pages(struct i915_hw_ppgtt *ppgtt)for (i =0 ; i < ppgtt->num_pd_pages; i++) {/* TODO: In the future we'll support sparse mappings, so this * will have to change. */ -if (!ppgtt->pdp.page_directory[i].daddr) +if (!ppgtt->pdp.page_directory[i]->daddr)continue ; -pci_unmap_page (hwdev, ppgtt->pdp.page_directory[i].daddr,PAGE_SIZE , +pci_unmap_page (hwdev, ppgtt->pdp.page_directory[i]->daddr,PAGE_SIZE ,PCI_DMA_BIDIRECTIONAL );for (j =0 ; j <GEN8_PDES_PER_PAGE ; j++) { -dma_addr_t addr = ppgtt->pdp.page_directory[i].page_tables[j].daddr; +struct i915_page_directory_entry *pd = ppgtt->pdp.page_directory[i]; +struct i915_page_table_entry *pt; +dma_addr_t addr; + +if (WARN_ON (!pd->page_tables[j])) +continue ; + + pt = pd->page_tables[j]; + addr = pt->daddr; +if (addr)pci_unmap_page (hwdev, addr,PAGE_SIZE ,PCI_DMA_BIDIRECTIONAL );@@ -474,25 +593,20 @@ static void gen8_ppgtt_cleanup(struct i915_address_space *vm)static int gen8_ppgtt_allocate_page_tables (struct i915_hw_ppgtt *ppgtt) { -int i, j; +int i, ret;for (i =0 ; i < ppgtt->num_pd_pages; i++) { -struct i915_page_directory_entry *pd = &ppgtt->pdp.page_directory[i]; -for (j =0 ; j <GEN8_PDES_PER_PAGE ; j++) { -struct i915_page_table_entry *pt = &pd->page_tables[j]; - - pt->page =alloc_page (GFP_KERNEL | __GFP_ZERO); -if (!pt->page) - goto unwind_out; - - } + ret =alloc_pt_range (ppgtt->pdp.page_directory[i], +0 ,GEN8_PDES_PER_PAGE ); +if (ret) + goto unwind_out; }return 0 ; unwind_out:while (i--) -gen8_free_page_tables (&ppgtt->pdp.page_directory[i]); +gen8_free_page_tables (ppgtt->pdp.page_directory[i]);return -ENOMEM ; }@@ -503,19 +617,9 @@ static int gen8_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt,int i;for (i =0 ; i < max_pdp; i++) { -struct i915_page_table_entry *pt; - - pt =kcalloc (GEN8_PDES_PER_PAGE ,sizeof (*pt),GFP_KERNEL ); -if (!pt) + ppgtt->pdp.page_directory[i] =alloc_pd_single (); +if (IS_ERR (ppgtt->pdp.page_directory[i])) goto unwind_out; - - ppgtt->pdp.page_directory[i].page =alloc_page (GFP_KERNEL ); -if (!ppgtt->pdp.page_directory[i].page) { -kfree (pt); - goto unwind_out; - } - - ppgtt->pdp.page_directory[i].page_tables = pt; } ppgtt->num_pd_pages = max_pdp;@@ -524,10 +628,8 @@ static int gen8_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt,return 0 ; unwind_out: -while (i--) { -kfree (ppgtt->pdp.page_directory[i].page_tables); -__free_page (ppgtt->pdp.page_directory[i].page); - } +while (i--) +unmap_and_free_pd (ppgtt->pdp.page_directory[i]);return -ENOMEM ; }@@ -561,14 +663,14 @@ static int gen8_ppgtt_setup_page_directories(struct i915_hw_ppgtt *ppgtt,int ret; pd_addr =pci_map_page (ppgtt->base.dev->pdev, - ppgtt->pdp.page_directory[pd].page,0 , + ppgtt->pdp.page_directory[pd]->page,0 ,PAGE_SIZE ,PCI_DMA_BIDIRECTIONAL ); ret =pci_dma_mapping_error (ppgtt->base.dev->pdev, pd_addr);if (ret)return ret; - ppgtt->pdp.page_directory[pd].daddr = pd_addr; + ppgtt->pdp.page_directory[pd]->daddr = pd_addr;return 0 ; }@@ -578,8 +680,8 @@ static int gen8_ppgtt_setup_page_tables(struct i915_hw_ppgtt *ppgtt, const int pt) {dma_addr_t pt_addr; -struct i915_page_directory_entry *pdir = &ppgtt->pdp.page_directory[pd]; -struct i915_page_table_entry *ptab = &pdir->page_tables[pt]; +struct i915_page_directory_entry *pdir = ppgtt->pdp.page_directory[pd]; +struct i915_page_table_entry *ptab = pdir->page_tables[pt];struct page *p = ptab->page;int ret;@@ -642,10 +744,12 @@ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size)* will never need to touch the PDEs again. */ for (i =0 ; i < max_pdp; i++) { +struct i915_page_directory_entry *pd = ppgtt->pdp.page_directory[i];gen8_ppgtt_pde_t *pd_vaddr; - pd_vaddr =kmap_atomic (ppgtt->pdp.page_directory[i].page); + pd_vaddr =kmap_atomic (ppgtt->pdp.page_directory[i]->page);for (j =0 ; j <GEN8_PDES_PER_PAGE ; j++) { -dma_addr_t addr = ppgtt->pdp.page_directory[i].page_tables[j].daddr; +struct i915_page_table_entry *pt = pd->page_tables[j]; +dma_addr_t addr = pt->daddr; pd_vaddr[j] =gen8_pde_encode (ppgtt->base.dev, addr,I915_CACHE_LLC ); }@@ -696,7 +800,7 @@ static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m)for (pde =0 ; pde < ppgtt->num_pd_entries; pde++) {u32 expected;gen6_gtt_pte_t *pt_vaddr; -dma_addr_t pt_addr = ppgtt->pd.page_tables[pde].daddr; +dma_addr_t pt_addr = ppgtt->pd.page_tables[pde]->daddr; pd_entry =readl (pd_addr + pde); expected = (GEN6_PDE_ADDR_ENCODE (pt_addr) |GEN6_PDE_VALID );@@ -707,7 +811,7 @@ static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m) expected);seq_printf (m,"\tPDE: %x\n" , pd_entry); - pt_vaddr =kmap_atomic (ppgtt->pd.page_tables[pde].page); + pt_vaddr =kmap_atomic (ppgtt->pd.page_tables[pde]->page);for (pte =0 ; pte <I915_PPGTT_PT_ENTRIES ; pte+=4 ) {unsigned long va = (pde * PAGE_SIZE * I915_PPGTT_PT_ENTRIES) +@@ -746,7 +850,7 @@ static void gen6_write_pdes(struct i915_hw_ppgtt *ppgtt)for (i =0 ; i < ppgtt->num_pd_entries; i++) {dma_addr_t pt_addr; - pt_addr = ppgtt->pd.page_tables[i].daddr; + pt_addr = ppgtt->pd.page_tables[i]->daddr; pd_entry =GEN6_PDE_ADDR_ENCODE (pt_addr); pd_entry |=GEN6_PDE_VALID ;@@ -922,7 +1026,7 @@ static void gen6_ppgtt_clear_range(struct i915_address_space *vm,if (last_pte >I915_PPGTT_PT_ENTRIES ) last_pte =I915_PPGTT_PT_ENTRIES ; - pt_vaddr =kmap_atomic (ppgtt->pd.page_tables[act_pt].page); + pt_vaddr =kmap_atomic (ppgtt->pd.page_tables[act_pt]->page);for (i = first_pte; i < last_pte; i++) pt_vaddr[i] = scratch_pte;@@ -951,7 +1055,7 @@ static void gen6_ppgtt_insert_entries(struct i915_address_space *vm, pt_vaddr =NULL ;for_each_sg_page (pages->sgl, &sg_iter, pages->nents,0 ) {if (pt_vaddr ==NULL ) - pt_vaddr =kmap_atomic (ppgtt->pd.page_tables[act_pt].page); + pt_vaddr =kmap_atomic (ppgtt->pd.page_tables[act_pt]->page); pt_vaddr[act_pte] = vm->pte_encode(sg_page_iter_dma_address(&sg_iter),@@ -974,7 +1078,7 @@ static void gen6_ppgtt_unmap_pages(struct i915_hw_ppgtt *ppgtt)for (i =0 ; i < ppgtt->num_pd_entries; i++)pci_unmap_page (ppgtt->base.dev->pdev, - ppgtt->pd.page_tables[i].daddr, + ppgtt->pd.page_tables[i]->daddr,4096 ,PCI_DMA_BIDIRECTIONAL ); }@@ -983,8 +1087,9 @@ static void gen6_ppgtt_free(struct i915_hw_ppgtt *ppgtt)int i;for (i =0 ; i < ppgtt->num_pd_entries; i++) -__free_page (ppgtt->pd.page_tables[i].page); -kfree (ppgtt->pd.page_tables); +unmap_and_free_pt (ppgtt->pd.page_tables[i]); + +unmap_and_free_pd (&ppgtt->pd); }static void gen6_ppgtt_cleanup (struct i915_address_space *vm)@@ -1039,27 +1144,6 @@ alloc:return 0 ; } -static int gen6_ppgtt_allocate_page_tables (struct i915_hw_ppgtt *ppgtt) -{ -struct i915_page_table_entry *pt; -int i; - - pt =kcalloc (ppgtt->num_pd_entries,sizeof (*pt),GFP_KERNEL ); -if (!pt) -return -ENOMEM ; - -for (i =0 ; i < ppgtt->num_pd_entries; i++) { - pt[i].page =alloc_page (GFP_KERNEL ); -if (!pt->page) { -gen6_ppgtt_free (ppgtt); -return -ENOMEM ; - } - } - - ppgtt->pd.page_tables = pt; -return 0 ; -} -static int gen6_ppgtt_alloc (struct i915_hw_ppgtt *ppgtt) {int ret;@@ -1068,7 +1152,7 @@ static int gen6_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt)if (ret)return ret; - ret =gen6_ppgtt_allocate_page_tables (ppgtt); + ret =alloc_pt_range (&ppgtt->pd,0 , ppgtt->num_pd_entries);if (ret) {drm_mm_remove_node (&ppgtt->node);return ret;@@ -1086,7 +1170,7 @@ static int gen6_ppgtt_setup_page_tables(struct i915_hw_ppgtt *ppgtt)struct page *page;dma_addr_t pt_addr; - page = ppgtt->pd.page_tables[i].page; + page = ppgtt->pd.page_tables[i]->page; pt_addr =pci_map_page (dev->pdev, page,0 ,4096 ,PCI_DMA_BIDIRECTIONAL );@@ -1095,7 +1179,7 @@ static int gen6_ppgtt_setup_page_tables(struct i915_hw_ppgtt *ppgtt)return -EIO ; } - ppgtt->pd.page_tables[i].daddr = pt_addr; + ppgtt->pd.page_tables[i]->daddr = pt_addr; }return 0 ;drivers/gpu/drm/i915/i915_gem_gtt.h +0
-0@@ -199,12 +199,12 @@ struct i915_page_directory_entry {dma_addr_t daddr; };
Quoted 4 lines
drivers/gpu/drm/i915/i915_gem_gtt.h struct i915_page_directory_entry { · line 202
Quoted 2 lines
drivers/gpu/drm/i915/i915_gem_gtt.h +1
-1-struct i915_page_table_entry *page_tables; +struct i915_page_table_entry *page_tables[GEN6_PPGTT_PD_ENTRIES ];/* PDEs */ Would you consider changing the plural here in 'tables' so that we would lose the discrepancy against the
page_directorybelow?
Ok, I'll rename them, but it's cleaner to make this change in the patch that added it ("drm/i915: page table abstractions").
-Michel
Quoted 41 lines
-Mika
Quoted 39 lines
drivers/gpu/drm/i915/i915_gem_gtt.h +1
-1};struct i915_page_directory_pointer_entry {/* struct page *page; */ -struct i915_page_directory_entry page_directory[GEN8_LEGACY_PDPES ]; +struct i915_page_directory_entry *page_directory[GEN8_LEGACY_PDPES ]; };struct i915_address_space {drivers/gpu/drm/i915/intel_lrc.c +8
-8@@ -1735,14 +1735,14 @@ populate_lr_context(struct intel_context *ctx, struct drm_i915_gem_object *ctx_o reg_state[CTX_PDP1_LDW ] =GEN8_RING_PDP_LDW (ring,1 ); reg_state[CTX_PDP0_UDW ] =GEN8_RING_PDP_UDW (ring,0 ); reg_state[CTX_PDP0_LDW ] =GEN8_RING_PDP_LDW (ring,0 ); - reg_state[CTX_PDP3_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[3 ].daddr); - reg_state[CTX_PDP3_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[3 ].daddr); - reg_state[CTX_PDP2_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[2 ].daddr); - reg_state[CTX_PDP2_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[2 ].daddr); - reg_state[CTX_PDP1_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[1 ].daddr); - reg_state[CTX_PDP1_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[1 ].daddr); - reg_state[CTX_PDP0_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[0 ].daddr); - reg_state[CTX_PDP0_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[0 ].daddr); + reg_state[CTX_PDP3_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[3 ]->daddr); + reg_state[CTX_PDP3_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[3 ]->daddr); + reg_state[CTX_PDP2_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[2 ]->daddr); + reg_state[CTX_PDP2_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[2 ]->daddr); + reg_state[CTX_PDP1_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[1 ]->daddr); + reg_state[CTX_PDP1_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[1 ]->daddr); + reg_state[CTX_PDP0_UDW +1 ] =upper_32_bits (ppgtt->pdp.page_directory[0 ]->daddr); + reg_state[CTX_PDP0_LDW +1 ] =lower_32_bits (ppgtt->pdp.page_directory[0 ]->daddr);if (ring->id ==RCS ) { reg_state[CTX_LRI_HEADER_2 ] =MI_LOAD_REGISTER_IMM (1 ); reg_state[CTX_R_PWR_CLK_STATE ] =0x20c8 ;Signature · 1 line
2.1.1
Quoted 4 lines
Quoted 4 lines
_______________________________________________
Intel-gfx mailing list
[email protected]
http://lists.freedesktop.org/mailman/listinfo/intel-gfx
_______________________________________________
Intel-gfx mailing list
[email protected]
http://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH v5 4/32 drm/i915: Plumb drm_device through page tables operations
In reply to Michel ThierryThe next patch in the series will require it for alloc_pt_single.
--- drivers/gpu/drm/i915/i915_gem_gtt.c | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-)
drivers/gpu/drm/i915/i915_gem_gtt.c +16 -13
@@ -142,7 +142,6 @@ static int sanitize_enable_ppgtt(struct drm_device *dev, int enable_ppgtt)return has_aliasing_ppgtt ?1 :0 ; } -static void ppgtt_bind_vma (struct i915_vma *vma,enum i915_cache_level cache_level,u32 flags);
@@ -279,7 +278,7 @@ static gen6_gtt_pte_t iris_pte_encode(dma_addr_t addr,return pte; } -static void unmap_and_free_pt (struct i915_page_table_entry *pt) +static void unmap_and_free_pt (struct i915_page_table_entry *pt,struct drm_device *dev) {if (WARN_ON (!pt->page))return ;
@@ -287,7 +286,7 @@ static void unmap_and_free_pt(struct i915_page_table_entry *pt)kfree (pt); } -static struct i915_page_table_entry *alloc_pt_single (void ) +static struct i915_page_table_entry *alloc_pt_single (struct drm_device *dev) {struct i915_page_table_entry *pt;
@@ -317,7 +316,9 @@ static struct i915_page_table_entry *alloc_pt_single(void)* * Return: 0 if allocation succeeded. */ -static int alloc_pt_range (struct i915_page_directory_entry *pd,uint16_t pde,size_t count) +static int alloc_pt_range (struct i915_page_directory_entry *pd,uint16_t pde,size_t count, +struct drm_device *dev) + {int i, ret;
@@ -326,7 +327,7 @@ static int alloc_pt_range(struct i915_page_directory_entry *pd, uint16_t pde, sireturn -EINVAL ;for (i = pde; i < pde + count; i++) { -struct i915_page_table_entry *pt =alloc_pt_single (); +struct i915_page_table_entry *pt =alloc_pt_single (dev);if (IS_ERR (pt)) { ret =PTR_ERR (pt);
@@ -342,7 +343,7 @@ static int alloc_pt_range(struct i915_page_directory_entry *pd, uint16_t pde, si err_out:while (i--) -unmap_and_free_pt (pd->page_tables[i]); +unmap_and_free_pt (pd->page_tables[i], dev);return ret; }
@@ -521,7 +522,7 @@ static void gen8_ppgtt_insert_entries(struct i915_address_space *vm, } } -static void gen8_free_page_tables (struct i915_page_directory_entry *pd) +static void gen8_free_page_tables (struct i915_page_directory_entry *pd,struct drm_device *dev) {int i;
@@ -532,7 +533,7 @@ static void gen8_free_page_tables(struct i915_page_directory_entry *pd)if (WARN_ON (!pd->page_tables[i]))continue ; -unmap_and_free_pt (pd->page_tables[i]); +unmap_and_free_pt (pd->page_tables[i], dev); pd->page_tables[i] =NULL ; } }
@@ -545,7 +546,7 @@ static void gen8_ppgtt_free(struct i915_hw_ppgtt *ppgtt)if (WARN_ON (!ppgtt->pdp.page_directory[i]))continue ; -gen8_free_page_tables (ppgtt->pdp.page_directory[i]); +gen8_free_page_tables (ppgtt->pdp.page_directory[i], ppgtt->base.dev);unmap_and_free_pd (ppgtt->pdp.page_directory[i]); } }
@@ -597,7 +598,7 @@ static int gen8_ppgtt_allocate_page_tables(struct i915_hw_ppgtt *ppgtt)for (i =0 ; i < ppgtt->num_pd_pages; i++) { ret =alloc_pt_range (ppgtt->pdp.page_directory[i], -0 ,GEN8_PDES_PER_PAGE ); +0 ,GEN8_PDES_PER_PAGE , ppgtt->base.dev);if (ret) goto unwind_out; }
@@ -606,7 +607,7 @@ static int gen8_ppgtt_allocate_page_tables(struct i915_hw_ppgtt *ppgtt) unwind_out:while (i--) -gen8_free_page_tables (ppgtt->pdp.page_directory[i]); +gen8_free_page_tables (ppgtt->pdp.page_directory[i], ppgtt->base.dev);return -ENOMEM ; }
@@ -1087,7 +1088,7 @@ static void gen6_ppgtt_free(struct i915_hw_ppgtt *ppgtt)int i;for (i =0 ; i < ppgtt->num_pd_entries; i++) -unmap_and_free_pt (ppgtt->pd.page_tables[i]); +unmap_and_free_pt (ppgtt->pd.page_tables[i], ppgtt->base.dev);unmap_and_free_pd (&ppgtt->pd); }
@@ -1152,7 +1153,9 @@ static int gen6_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt)if (ret)return ret; - ret =alloc_pt_range (&ppgtt->pd,0 , ppgtt->num_pd_entries); + ret =alloc_pt_range (&ppgtt->pd,0 , ppgtt->num_pd_entries, + ppgtt->base.dev); +if (ret) {drm_mm_remove_node (&ppgtt->node);return ret;
Signature · 6 lines
2.1.1 _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx