Dynamic page table alloc, 64b, and GPU/CPU mirror

index 6e2145b..29dd825 100644
@@ -553,9 +553,7 @@ mi_set_context(struct intel_ring_buffer *ring,
 	int ret;
 
 	/* w/a: If Flush TLB Invalidation Mode is enabled, driver must do a TLB
-	 * invalidation prior to MI_SET_CONTEXT. On GEN6 we don't set the value
-	 * explicitly, so we rely on the value at ring init, stored in
-	 * itlb_before_ctx_switch.
+	 * invalidation prior to MI_SET_CONTEXT.
 	 */
 	if (IS_GEN6(ring->dev)) {
 		ret = ring->flush(ring, I915_GEM_GPU_DOMAINS, 0);

index d02c8de..d10ddcc 100644
@@ -994,7 +994,7 @@ static int i915_getparam(struct drm_device *dev, void *data,
 		value = HAS_WT(dev);
 		break;
 	case I915_PARAM_HAS_ALIASING_PPGTT:
-		value = dev_priv->mm.aliasing_ppgtt || USES_FULL_PPGTT(dev);
+		value = dev_priv->mm.aliasing_ppgtt ? 1 : 0;
 		break;
 	case I915_PARAM_HAS_WAIT_TIMEOUT:
 		value = 1;

@@ -1020,6 +1020,9 @@ static int i915_getparam(struct drm_device *dev, void *data,
 	case I915_PARAM_CMD_PARSER_VERSION:
 		value = i915_cmd_parser_get_version();
 		break;
+	case I915_PARAM_HAS_FULL_PPGTT:
+		value = USES_FULL_PPGTT(dev);
+		break;
 	default:
 		DRM_DEBUG("Unknown parameter %d\n", param->param);
 		return -EINVAL;

index 8a3e4ef00..6306a84 100644
@@ -338,6 +338,7 @@ typedef struct drm_i915_irq_wait {
 #define I915_PARAM_HAS_EXEC_HANDLE_LUT   26
 #define I915_PARAM_HAS_WT     	 	 27
 #define I915_PARAM_CMD_PARSER_VERSION	 28
+#define I915_PARAM_HAS_FULL_PPGTT	 29
 
 typedef struct drm_i915_getparam {
 	int param;

index d10ddcc..54a08a9 100644
@@ -1921,9 +1921,18 @@ void i915_driver_lastclose(struct drm_device * dev)
 
 void i915_driver_preclose(struct drm_device * dev, struct drm_file *file_priv)
 {
+	struct drm_i915_private *dev_priv = to_i915(dev);
+	bool was_interruptible;
+
 	mutex_lock(&dev->struct_mutex);
+	was_interruptible = dev_priv->mm.interruptible;
+	WARN_ON(!was_interruptible);
+	dev_priv->mm.interruptible = false;
+
 	i915_gem_context_close(dev, file_priv);
 	i915_gem_release(dev, file_priv);
+
+	dev_priv->mm.interruptible = was_interruptible;
 	mutex_unlock(&dev->struct_mutex);
 }
 

index 1a190a1..88d3d82 100644
@@ -2312,6 +2312,9 @@ bool i915_gem_obj_bound(struct drm_i915_gem_object *o,
 			struct i915_address_space *vm);
 unsigned long i915_gem_obj_size(struct drm_i915_gem_object *o,
 				struct i915_address_space *vm);
+void i915_gem_vma_bind(struct i915_vma *vma, enum i915_cache_level,
+		       unsigned flags);
+void i915_gem_vma_unbind(struct i915_vma *vma);
 struct i915_vma *i915_gem_obj_to_vma(struct drm_i915_gem_object *obj,
 				     struct i915_address_space *vm);
 struct i915_vma *

index 8fd1824..59b0e67 100644
@@ -2771,7 +2771,7 @@ int i915_vma_unbind(struct i915_vma *vma)
 
 	trace_i915_vma_unbind(vma);
 
-	vma->unbind_vma(vma);
+	i915_gem_vma_unbind(vma);
 
 	i915_gem_gtt_finish_object(obj);
 

@@ -3317,8 +3317,8 @@ search_free:
 	WARN_ON(flags & PIN_MAPPABLE && !obj->map_and_fenceable);
 
 	trace_i915_vma_bind(vma, flags);
-	vma->bind_vma(vma, obj->cache_level,
-		      flags & (PIN_MAPPABLE | PIN_GLOBAL) ? GLOBAL_BIND : 0);
+	i915_gem_vma_bind(vma, obj->cache_level,
+			  flags & (PIN_MAPPABLE | PIN_GLOBAL) ? GLOBAL_BIND : 0);
 
 	i915_gem_verify_gtt(dev);
 	return vma;

@@ -3522,8 +3522,8 @@ int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
 
 		list_for_each_entry(vma, &obj->vma_list, vma_link)
 			if (drm_mm_node_allocated(&vma->node))
-				vma->bind_vma(vma, cache_level,
-					      obj->has_global_gtt_mapping ? GLOBAL_BIND : 0);
+				i915_gem_vma_bind(vma, cache_level,
+						  obj->has_global_gtt_mapping ? GLOBAL_BIND : 0);
 	}
 
 	list_for_each_entry(vma, &obj->vma_list, vma_link)

@@ -3892,7 +3892,7 @@ i915_gem_object_pin(struct drm_i915_gem_object *obj,
 	}
 
 	if (flags & PIN_GLOBAL && !obj->has_global_gtt_mapping)
-		vma->bind_vma(vma, obj->cache_level, GLOBAL_BIND);
+		i915_gem_vma_bind(vma, obj->cache_level, GLOBAL_BIND);
 
 	vma->pin_count++;
 	if (flags & PIN_MAPPABLE)

index 29dd825..f2dc17a 100644
@@ -652,7 +652,7 @@ static int do_switch(struct intel_ring_buffer *ring,
 	if (!to->obj->has_global_gtt_mapping) {
 		struct i915_vma *vma = i915_gem_obj_to_vma(to->obj,
 							   &dev_priv->gtt.base);
-		vma->bind_vma(vma, to->obj->cache_level, GLOBAL_BIND);
+		i915_gem_vma_bind(vma, to->obj->cache_level, GLOBAL_BIND);
 	}
 
 	if (!to->is_initialized || i915_gem_context_is_default(to))

index 47fe8ec..cd9b932 100644
@@ -373,7 +373,8 @@ i915_gem_execbuffer_relocate_entry(struct drm_i915_gem_object *obj,
 		struct i915_vma *vma =
 			list_first_entry(&target_i915_obj->vma_list,
 					 typeof(*vma), vma_link);
-		vma->bind_vma(vma, target_i915_obj->cache_level, GLOBAL_BIND);
+		i915_gem_vma_bind(vma, target_i915_obj->cache_level,
+				  GLOBAL_BIND);
 	}
 
 	/* Validate that the target is in a valid r/w GPU domain */

@@ -1269,7 +1270,7 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
 		 * allocate space first */
 		struct i915_vma *vma = i915_gem_obj_to_ggtt(batch_obj);
 		BUG_ON(!vma);
-		vma->bind_vma(vma, batch_obj->cache_level, GLOBAL_BIND);
+		i915_gem_vma_bind(vma, batch_obj->cache_level, GLOBAL_BIND);
 	}
 
 	if (flags & I915_DISPATCH_SECURE)

index 33610fe..87d92d0 100644
@@ -1340,7 +1340,7 @@ void i915_gem_restore_gtt_mappings(struct drm_device *dev)
 		 * without telling our object about it. So we need to fake it.
 		 */
 		obj->has_global_gtt_mapping = 0;
-		vma->bind_vma(vma, obj->cache_level, GLOBAL_BIND);
+		i915_gem_vma_bind(vma, obj->cache_level, GLOBAL_BIND);
 	}
 
 

@@ -2041,6 +2041,17 @@ int i915_gem_gtt_init(struct drm_device *dev)
 	return 0;
 }
 
+void i915_gem_vma_bind(struct i915_vma *vma, enum i915_cache_level cache_level,
+		       unsigned flags)
+{
+	vma->bind_vma(vma, cache_level, flags);
+}
+
+void i915_gem_vma_unbind(struct i915_vma *vma)
+{
+	vma->unbind_vma(vma);
+}
+
 static struct i915_vma *__i915_gem_vma_create(struct drm_i915_gem_object *obj,
 					      struct i915_address_space *vm)
 {

index 88d3d82..62e1ecb 100644
@@ -2136,6 +2136,8 @@ void i915_gem_vma_destroy(struct i915_vma *vma);
 #define PIN_MAPPABLE 0x1
 #define PIN_NONBLOCK 0x2
 #define PIN_GLOBAL 0x4
+#define PIN_ALIASING 0x8
+#define PIN_GLOBAL_ALIASED (PIN_ALIASING | PIN_GLOBAL)
 int __must_check i915_gem_object_pin(struct drm_i915_gem_object *obj,
 				     struct i915_address_space *vm,
 				     uint32_t alignment,

@@ -2362,7 +2364,7 @@ i915_gem_obj_ggtt_pin(struct drm_i915_gem_object *obj,
 		      uint32_t alignment,
 		      unsigned flags)
 {
-	return i915_gem_object_pin(obj, obj_to_ggtt(obj), alignment, flags | PIN_GLOBAL);
+	return i915_gem_object_pin(obj, obj_to_ggtt(obj), alignment, flags | PIN_GLOBAL_ALIASED);
 }
 
 static inline int

index 59b0e67..e3ac643 100644
@@ -3231,8 +3231,12 @@ i915_gem_object_bind_to_vm(struct drm_i915_gem_object *obj,
 	size_t gtt_max =
 		flags & PIN_MAPPABLE ? dev_priv->gtt.mappable_end : vm->total;
 	struct i915_vma *vma;
+	u32 vma_bind_flags = 0;
 	int ret;
 
+	if (WARN_ON((flags & (PIN_MAPPABLE | PIN_GLOBAL)) == PIN_MAPPABLE))
+		flags |= PIN_GLOBAL;
+
 	fence_size = i915_gem_get_gtt_size(dev,
 					   obj->base.size,
 					   obj->tiling_mode);

@@ -3316,9 +3320,11 @@ search_free:
 
 	WARN_ON(flags & PIN_MAPPABLE && !obj->map_and_fenceable);
 
+	if (flags & PIN_GLOBAL_ALIASED)
+		vma_bind_flags = GLOBAL_BIND | ALIASING_BIND;
+
 	trace_i915_vma_bind(vma, flags);
-	i915_gem_vma_bind(vma, obj->cache_level,
-			  flags & (PIN_MAPPABLE | PIN_GLOBAL) ? GLOBAL_BIND : 0);
+	i915_gem_vma_bind(vma, obj->cache_level, vma_bind_flags);
 
 	i915_gem_verify_gtt(dev);
 	return vma;

@@ -3521,9 +3527,14 @@ int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
 		}
 
 		list_for_each_entry(vma, &obj->vma_list, vma_link)
-			if (drm_mm_node_allocated(&vma->node))
-				i915_gem_vma_bind(vma, cache_level,
-						  obj->has_global_gtt_mapping ? GLOBAL_BIND : 0);
+			if (drm_mm_node_allocated(&vma->node)) {
+				u32 bind_flags = 0;
+				if (obj->has_global_gtt_mapping)
+					bind_flags |= GLOBAL_BIND;
+				if (obj->has_aliasing_ppgtt_mapping)
+					bind_flags |= ALIASING_BIND;
+				i915_gem_vma_bind(vma, cache_level, bind_flags);
+			}
 	}
 
 	list_for_each_entry(vma, &obj->vma_list, vma_link)

@@ -3891,8 +3902,14 @@ i915_gem_object_pin(struct drm_i915_gem_object *obj,
 			return PTR_ERR(vma);
 	}
 
-	if (flags & PIN_GLOBAL && !obj->has_global_gtt_mapping)
-		i915_gem_vma_bind(vma, obj->cache_level, GLOBAL_BIND);
+	if (flags & PIN_GLOBAL_ALIASED) {
+		u32 bind_flags = 0;
+		if (flags & PIN_GLOBAL && !obj->has_global_gtt_mapping)
+			bind_flags |= GLOBAL_BIND;
+		if (flags & PIN_ALIASING && !obj->has_aliasing_ppgtt_mapping)
+			bind_flags |= ALIASING_BIND;
+		i915_gem_vma_bind(vma, obj->cache_level, bind_flags);
+	}
 
 	vma->pin_count++;
 	if (flags & PIN_MAPPABLE)

index cd9b932..7cad10f 100644
@@ -374,7 +374,7 @@ i915_gem_execbuffer_relocate_entry(struct drm_i915_gem_object *obj,
 			list_first_entry(&target_i915_obj->vma_list,
 					 typeof(*vma), vma_link);
 		i915_gem_vma_bind(vma, target_i915_obj->cache_level,
-				  GLOBAL_BIND);
+				  GLOBAL_BIND | ALIASING_BIND);
 	}
 
 	/* Validate that the target is in a valid r/w GPU domain */

@@ -561,6 +561,7 @@ i915_gem_execbuffer_reserve_vma(struct i915_vma *vma,
 	if (need_fence || need_reloc_mappable(vma))
 		flags |= PIN_MAPPABLE;
 
+	/* FIXME: What kind of bind does Chris want? */
 	if (entry->flags & EXEC_OBJECT_NEEDS_GTT)
 		flags |= PIN_GLOBAL;
 

@@ -1270,7 +1271,10 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
 		 * allocate space first */
 		struct i915_vma *vma = i915_gem_obj_to_ggtt(batch_obj);
 		BUG_ON(!vma);
-		i915_gem_vma_bind(vma, batch_obj->cache_level, GLOBAL_BIND);
+		/* FIXME: Current secure dispatch code actually uses PPGTT. We
+		 * need to fix this eventually */
+		i915_gem_vma_bind(vma, batch_obj->cache_level,
+				  GLOBAL_BIND | ALIASING_BIND);
 	}
 
 	if (flags & I915_DISPATCH_SECURE)

index 87d92d0..226afea 100644
@@ -1339,8 +1339,16 @@ void i915_gem_restore_gtt_mappings(struct drm_device *dev)
 		 * Unfortunately above, we've just wiped out the mappings
 		 * without telling our object about it. So we need to fake it.
 		 */
-		obj->has_global_gtt_mapping = 0;
-		i915_gem_vma_bind(vma, obj->cache_level, GLOBAL_BIND);
+		if (obj->has_global_gtt_mapping || obj->has_aliasing_ppgtt_mapping) {
+			u32 bind_flags = 0;
+			if (obj->has_global_gtt_mapping)
+				        bind_flags |= GLOBAL_BIND;
+			if (obj->has_aliasing_ppgtt_mapping)
+				        bind_flags |= ALIASING_BIND;
+			obj->has_global_gtt_mapping = 0;
+			obj->has_aliasing_ppgtt_mapping = 0;
+			i915_gem_vma_bind(vma, obj->cache_level, bind_flags);
+		}
 	}
 
 

index cfca023..5635c65 100644
@@ -153,7 +153,11 @@ struct i915_vma {
 	 * setting the valid PTE entries to a reserved scratch page. */
 	void (*unbind_vma)(struct i915_vma *vma);
 	/* Map an object into an address space with the given cache flags. */
+
+/* Only use this if you know you want a strictly global binding */
 #define GLOBAL_BIND (1<<0)
+/* Only use this if you know you want a strictly aliased binding */
+#define ALIASING_BIND (1<<1)
 	void (*bind_vma)(struct i915_vma *vma,
 			 enum i915_cache_level cache_level,
 			 u32 flags);

index 62e1ecb..29bf034 100644
@@ -2364,7 +2364,7 @@ i915_gem_obj_ggtt_pin(struct drm_i915_gem_object *obj,
 		      uint32_t alignment,
 		      unsigned flags)
 {
-	return i915_gem_object_pin(obj, obj_to_ggtt(obj), alignment, flags | PIN_GLOBAL_ALIASED);
+	return i915_gem_object_pin(obj, obj_to_ggtt(obj), alignment, flags | PIN_GLOBAL);
 }
 
 static inline int

index e3ac643..320d6b0 100644
@@ -3320,8 +3320,10 @@ search_free:
 
 	WARN_ON(flags & PIN_MAPPABLE && !obj->map_and_fenceable);
 
-	if (flags & PIN_GLOBAL_ALIASED)
-		vma_bind_flags = GLOBAL_BIND | ALIASING_BIND;
+	if (flags & PIN_ALIASING)
+		vma_bind_flags = ALIASING_BIND;
+	if (flags & PIN_GLOBAL)
+		vma_bind_flags = GLOBAL_BIND;
 
 	trace_i915_vma_bind(vma, flags);
 	i915_gem_vma_bind(vma, obj->cache_level, vma_bind_flags);

index 7cad10f..3c3aba7 100644
@@ -549,10 +549,11 @@ i915_gem_execbuffer_reserve_vma(struct i915_vma *vma,
 	struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
 	bool has_fenced_gpu_access = INTEL_INFO(ring->dev)->gen < 4;
 	bool need_fence;
-	unsigned flags;
+	unsigned flags = 0;
 	int ret;
 
-	flags = 0;
+	if (i915_is_ggtt(vma->vm))
+		flags = PIN_ALIASING;
 
 	need_fence =
 		has_fenced_gpu_access &&

index 226afea..bec637b 100644
@@ -1595,6 +1595,9 @@ static void ggtt_bind_vma(struct i915_vma *vma,
 		}
 	}
 
+	if (!(flags & ALIASING_BIND))
+		return;
+
 	if (dev_priv->mm.aliasing_ppgtt &&
 	    (!obj->has_aliasing_ppgtt_mapping ||
 	     (cache_level != obj->cache_level))) {

index bec637b..33cac92 100644
@@ -73,6 +73,11 @@ static void ppgtt_bind_vma(struct i915_vma *vma,
 static void ppgtt_unbind_vma(struct i915_vma *vma);
 static int gen8_ppgtt_enable(struct i915_hw_ppgtt *ppgtt);
 
+static size_t gtt_total_entries(struct i915_gtt *gtt)
+{
+	return gtt->base.total >> PAGE_SHIFT;
+}
+
 static inline gen8_gtt_pte_t gen8_pte_encode(dma_addr_t addr,
 					     enum i915_cache_level level,
 					     bool valid)

@@ -1491,7 +1496,7 @@ static void gen8_ggtt_clear_range(struct i915_address_space *vm,
 	unsigned num_entries = length >> PAGE_SHIFT;
 	gen8_gtt_pte_t scratch_pte, __iomem *gtt_base =
 		(gen8_gtt_pte_t __iomem *) dev_priv->gtt.gsm + first_entry;
-	const int max_entries = gtt_total_entries(dev_priv->gtt) - first_entry;
+	const int max_entries = gtt_total_entries(&dev_priv->gtt) - first_entry;
 	int i;
 
 	if (WARN(num_entries > max_entries,

@@ -1517,7 +1522,7 @@ static void gen6_ggtt_clear_range(struct i915_address_space *vm,
 	unsigned num_entries = length >> PAGE_SHIFT;
 	gen6_gtt_pte_t scratch_pte, __iomem *gtt_base =
 		(gen6_gtt_pte_t __iomem *) dev_priv->gtt.gsm + first_entry;
-	const int max_entries = gtt_total_entries(dev_priv->gtt) - first_entry;
+	const int max_entries = gtt_total_entries(&dev_priv->gtt) - first_entry;
 	int i;
 
 	if (WARN(num_entries > max_entries,

index 5635c65..ad68079 100644
@@ -38,8 +38,6 @@ typedef uint32_t gen6_gtt_pte_t;
 typedef uint64_t gen8_gtt_pte_t;
 typedef gen8_gtt_pte_t gen8_ppgtt_pde_t;
 
-#define gtt_total_entries(gtt) ((gtt).base.total >> PAGE_SHIFT)
-
 #define I915_PPGTT_PT_ENTRIES		(PAGE_SIZE / sizeof(gen6_gtt_pte_t))
 /* gen6-hsw has bit 11-4 for physical addr bit 39-32 */
 #define GEN6_GTT_ADDR_ENCODE(addr)	((addr) | (((addr) >> 28) & 0xff0))

index 33cac92..d3c52b1 100644
@@ -319,7 +319,7 @@ static void gen8_ppgtt_insert_entries(struct i915_address_space *vm,
 	pt_vaddr = NULL;
 
 	for_each_sg_page(pages->sgl, &sg_iter, pages->nents, 0) {
-		if (WARN_ON(pdpe >= GEN8_LEGACY_PDPS))
+		if (WARN_ON(pdpe >= GEN8_LEGACY_PDPES))
 			break;
 
 		if (pt_vaddr == NULL)

@@ -433,7 +433,7 @@ bail:
 static int gen8_ppgtt_allocate_page_tables(struct i915_hw_ppgtt *ppgtt,
 					   const int max_pdp)
 {
-	struct page **pt_pages[GEN8_LEGACY_PDPS];
+	struct page **pt_pages[GEN8_LEGACY_PDPES];
 	int i, ret;
 
 	for (i = 0; i < max_pdp; i++) {

@@ -485,7 +485,7 @@ static int gen8_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt,
 		return -ENOMEM;
 
 	ppgtt->num_pd_pages = 1 << get_order(max_pdp << PAGE_SHIFT);
-	BUG_ON(ppgtt->num_pd_pages > GEN8_LEGACY_PDPS);
+	BUG_ON(ppgtt->num_pd_pages > GEN8_LEGACY_PDPES);
 
 	return 0;
 }

index ad68079..7c06c43 100644
@@ -84,7 +84,7 @@ typedef gen8_gtt_pte_t gen8_ppgtt_pde_t;
 #define GEN8_PDE_MASK			0x1ff
 #define GEN8_PTE_SHIFT			12
 #define GEN8_PTE_MASK			0x1ff
-#define GEN8_LEGACY_PDPS		4
+#define GEN8_LEGACY_PDPES		4
 #define GEN8_PTES_PER_PAGE		(PAGE_SIZE / sizeof(gen8_gtt_pte_t))
 #define GEN8_PDES_PER_PAGE		(PAGE_SIZE / sizeof(gen8_ppgtt_pde_t))
 

@@ -247,12 +247,12 @@ struct i915_hw_ppgtt {
 	unsigned num_pd_pages; /* gen8+ */
 	union {
 		struct page **pt_pages;
-		struct page **gen8_pt_pages[GEN8_LEGACY_PDPS];
+		struct page **gen8_pt_pages[GEN8_LEGACY_PDPES];
 	};
 	struct page *pd_pages;
 	union {
 		uint32_t pd_offset;
-		dma_addr_t pd_dma_addr[GEN8_LEGACY_PDPS];
+		dma_addr_t pd_dma_addr[GEN8_LEGACY_PDPES];
 	};
 	union {
 		dma_addr_t *pt_dma_addr;

index d9c1414..4a0b1c8 100644
@@ -1812,18 +1812,13 @@ static int i915_swizzle_info(struct seq_file *m, void *data)
 	return 0;
 }
 
-static int per_file_ctx(int id, void *ptr, void *data)
+static void print_ppgtt(struct seq_file *m, struct i915_hw_ppgtt *ppgtt, const char *name)
 {
-	struct i915_hw_context *ctx = ptr;
-	struct seq_file *m = data;
-	struct i915_hw_ppgtt *ppgtt = ctx_to_ppgtt(ctx);
-
-	ppgtt->debug_dump(ppgtt, m);
-
-	return 0;
+	seq_printf(m, "%s:\n", name);
+	seq_printf(m, "pd gtt offset: 0x%08x\n", ppgtt->pd_offset);
 }
 
-static void gen8_ppgtt_info(struct seq_file *m, struct drm_device *dev)
+static void gen8_ppgtt_info(struct seq_file *m, struct drm_device *dev, int verbose)
 {
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	struct intel_ring_buffer *ring;

@@ -1847,7 +1842,21 @@ static void gen8_ppgtt_info(struct seq_file *m, struct drm_device *dev)
 	}
 }
 
-static void gen6_ppgtt_info(struct seq_file *m, struct drm_device *dev)
+static int per_file_ctx(int id, void *ptr, void *data)
+{
+	struct i915_hw_context *ctx = ptr;
+	struct seq_file *m = data;
+	bool verbose = (unsigned long)data & 1;
+	struct i915_hw_ppgtt *ppgtt = ctx_to_ppgtt(ctx);
+
+	print_ppgtt(m, ppgtt, ctx->id == DEFAULT_CONTEXT_ID ? "Default context" : "User context");
+	if (verbose)
+		ppgtt->debug_dump(ppgtt, m);
+
+	return 0;
+}
+
+static void gen6_ppgtt_info(struct seq_file *m, struct drm_device *dev, bool verbose)
 {
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	struct intel_ring_buffer *ring;

@@ -1868,10 +1877,9 @@ static void gen6_ppgtt_info(struct seq_file *m, struct drm_device *dev)
 	if (dev_priv->mm.aliasing_ppgtt) {
 		struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
 
-		seq_puts(m, "aliasing PPGTT:\n");
-		seq_printf(m, "pd gtt offset: 0x%08x\n", ppgtt->pd_offset);
-
-		ppgtt->debug_dump(ppgtt, m);
+		print_ppgtt(m, ppgtt, "Aliasing PPGTT");
+		if (verbose)
+			ppgtt->debug_dump(ppgtt, m);
 	} else
 		return;
 

@@ -1880,10 +1888,11 @@ static void gen6_ppgtt_info(struct seq_file *m, struct drm_device *dev)
 		struct i915_hw_ppgtt *pvt_ppgtt;
 
 		pvt_ppgtt = ctx_to_ppgtt(file_priv->private_default_ctx);
-		seq_printf(m, "proc: %s\n",
+		seq_printf(m, "\nproc: %s\n",
 			   get_pid_task(file->pid, PIDTYPE_PID)->comm);
-		seq_puts(m, "  default context:\n");
-		idr_for_each(&file_priv->context_idr, per_file_ctx, m);
+		print_ppgtt(m, pvt_ppgtt, "Default context");
+		idr_for_each(&file_priv->context_idr, per_file_ctx,
+			     (void *)((unsigned long)m | verbose));
 	}
 	seq_printf(m, "ECOCHK: 0x%08x\n", I915_READ(GAM_ECOCHK));
 }

@@ -1893,6 +1902,7 @@ static int i915_ppgtt_info(struct seq_file *m, void *data)
 	struct drm_info_node *node = (struct drm_info_node *) m->private;
 	struct drm_device *dev = node->minor->dev;
 	struct drm_i915_private *dev_priv = dev->dev_private;
+	bool verbose = node->info_ent->data ? true : false;
 
 	int ret = mutex_lock_interruptible(&dev->struct_mutex);
 	if (ret)

@@ -1900,9 +1910,9 @@ static int i915_ppgtt_info(struct seq_file *m, void *data)
 	intel_runtime_pm_get(dev_priv);
 
 	if (INTEL_INFO(dev)->gen >= 8)
-		gen8_ppgtt_info(m, dev);
+		gen8_ppgtt_info(m, dev, verbose);
 	else if (INTEL_INFO(dev)->gen >= 6)
-		gen6_ppgtt_info(m, dev);
+		gen6_ppgtt_info(m, dev, verbose);
 
 	intel_runtime_pm_put(dev_priv);
 	mutex_unlock(&dev->struct_mutex);

@@ -3843,6 +3853,7 @@ static const struct drm_info_list i915_debugfs_list[] = {
 	{"i915_gen6_forcewake_count", i915_gen6_forcewake_count_info, 0},
 	{"i915_swizzle_info", i915_swizzle_info, 0},
 	{"i915_ppgtt_info", i915_ppgtt_info, 0},
+	{"i915_ppgtt_verbose_info", i915_ppgtt_info, 0, (void *)1},
 	{"i915_llc", i915_llc, 0},
 	{"i915_edp_psr_status", i915_edp_psr_status, 0},
 	{"i915_sink_crc_eDP1", i915_sink_crc, 0},

index d3c52b1..0869e54 100644
@@ -515,40 +515,40 @@ static int gen8_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt,
 }
 
 static int gen8_ppgtt_setup_page_directories(struct i915_hw_ppgtt *ppgtt,
-					     const int pd)
+					     const int pdpe)
 {
 	dma_addr_t pd_addr;
 	int ret;
 
 	pd_addr = pci_map_page(ppgtt->base.dev->pdev,
-			       &ppgtt->pd_pages[pd], 0,
+			       &ppgtt->pd_pages[pdpe], 0,
 			       PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
 
 	ret = pci_dma_mapping_error(ppgtt->base.dev->pdev, pd_addr);
 	if (ret)
 		return ret;
 
-	ppgtt->pd_dma_addr[pd] = pd_addr;
+	ppgtt->pd_dma_addr[pdpe] = pd_addr;
 
 	return 0;
 }
 
 static int gen8_ppgtt_setup_page_tables(struct i915_hw_ppgtt *ppgtt,
-					const int pd,
-					const int pt)
+					const int pdpe,
+					const int pde)
 {
 	dma_addr_t pt_addr;
 	struct page *p;
 	int ret;
 
-	p = ppgtt->gen8_pt_pages[pd][pt];
+	p = ppgtt->gen8_pt_pages[pdpe][pde];
 	pt_addr = pci_map_page(ppgtt->base.dev->pdev,
 			       p, 0, PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
 	ret = pci_dma_mapping_error(ppgtt->base.dev->pdev, pt_addr);
 	if (ret)
 		return ret;
 
-	ppgtt->gen8_pt_dma_addr[pd][pt] = pt_addr;
+	ppgtt->gen8_pt_dma_addr[pdpe][pde] = pt_addr;
 
 	return 0;
 }

index 0869e54..d772577 100644
@@ -372,7 +372,7 @@ static void gen8_ppgtt_free(const struct i915_hw_ppgtt *ppgtt)
 	__free_pages(ppgtt->pd_pages, get_order(ppgtt->num_pd_pages << PAGE_SHIFT));
 }
 
-static void gen8_ppgtt_unmap_pages(struct i915_hw_ppgtt *ppgtt)
+static void gen8_ppgtt_dma_unmap_pages(struct i915_hw_ppgtt *ppgtt)
 {
 	struct pci_dev *hwdev = ppgtt->base.dev->pdev;
 	int i, j;

@@ -403,7 +403,7 @@ static void gen8_ppgtt_cleanup(struct i915_address_space *vm)
 	list_del(&vm->global_link);
 	drm_mm_takedown(&vm->mm);
 
-	gen8_ppgtt_unmap_pages(ppgtt);
+	gen8_ppgtt_dma_unmap_pages(ppgtt);
 	gen8_ppgtt_free(ppgtt);
 }
 

@@ -631,7 +631,7 @@ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size)
 	return 0;
 
 bail:
-	gen8_ppgtt_unmap_pages(ppgtt);
+	gen8_ppgtt_dma_unmap_pages(ppgtt);
 	gen8_ppgtt_free(ppgtt);
 	return ret;
 }

@@ -999,7 +999,7 @@ static void gen6_ppgtt_insert_entries(struct i915_address_space *vm,
 		kunmap_atomic(pt_vaddr);
 }
 
-static void gen6_ppgtt_unmap_pages(struct i915_hw_ppgtt *ppgtt)
+static void gen6_ppgtt_dma_unmap_pages(struct i915_hw_ppgtt *ppgtt)
 {
 	int i;
 

@@ -1030,7 +1030,7 @@ static void gen6_ppgtt_cleanup(struct i915_address_space *vm)
 	drm_mm_takedown(&ppgtt->base.mm);
 	drm_mm_remove_node(&ppgtt->node);
 
-	gen6_ppgtt_unmap_pages(ppgtt);
+	gen6_ppgtt_dma_unmap_pages(ppgtt);
 	gen6_ppgtt_free(ppgtt);
 }
 

@@ -1128,7 +1128,7 @@ static int gen6_ppgtt_setup_page_tables(struct i915_hw_ppgtt *ppgtt)
 				       PCI_DMA_BIDIRECTIONAL);
 
 		if (pci_dma_mapping_error(dev->pdev, pt_addr)) {
-			gen6_ppgtt_unmap_pages(ppgtt);
+			gen6_ppgtt_dma_unmap_pages(ppgtt);
 			return -EIO;
 		}
 

index d772577..5ca8208 100644
@@ -1063,11 +1063,14 @@ alloc:
 		goto alloc;
 	}
 
+	if (ret)
+		return ret;
+
 	if (ppgtt->node.start < dev_priv->gtt.mappable_end)
 		DRM_DEBUG("Forced to use aperture for PDEs\n");
 
 	ppgtt->num_pd_entries = GEN6_PPGTT_PD_ENTRIES;
-	return ret;
+	return 0;
 }
 
 static int gen6_ppgtt_allocate_page_tables(struct i915_hw_ppgtt *ppgtt)

index 5ca8208..08b1b25 100644
@@ -1180,6 +1180,8 @@ static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
 	ppgtt->pd_offset =
 		ppgtt->node.start / PAGE_SIZE * sizeof(gen6_gtt_pte_t);
 
+	gen6_write_pdes(ppgtt);
+
 	ppgtt->base.clear_range(&ppgtt->base, 0, ppgtt->base.total, true);
 
 	DRM_DEBUG_DRIVER("Allocated pde space (%ldM) at GTT entry: %lx\n",

@@ -1204,20 +1206,14 @@ int i915_gem_init_ppgtt(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt)
 	else
 		BUG();
 
-	if (!ret) {
-		struct drm_i915_private *dev_priv = dev->dev_private;
-		kref_init(&ppgtt->ref);
-		drm_mm_init(&ppgtt->base.mm, ppgtt->base.start,
-			    ppgtt->base.total);
-		i915_init_vm(dev_priv, &ppgtt->base);
-		if (INTEL_INFO(dev)->gen < 8) {
-			gen6_write_pdes(ppgtt);
-			DRM_DEBUG("Adding PPGTT at offset %x\n",
-				  ppgtt->pd_offset << 10);
-		}
-	}
+	if (ret)
+		return ret;
 
-	return ret;
+	kref_init(&ppgtt->ref);
+	drm_mm_init(&ppgtt->base.mm, ppgtt->base.start, ppgtt->base.total);
+	i915_init_vm(dev_priv, &ppgtt->base);
+
+	return 0;
 }
 
 static void

index 7c06c43..2002393 100644
@@ -256,7 +256,7 @@ struct i915_hw_ppgtt {
 	};
 	union {
 		dma_addr_t *pt_dma_addr;
-		dma_addr_t *gen8_pt_dma_addr[4];
+		dma_addr_t *gen8_pt_dma_addr[GEN8_LEGACY_PDPES];
 	};
 
 	struct i915_hw_context *ctx;

index 08b1b25..bfa9811 100644
@@ -692,26 +692,33 @@ static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m)
 	}
 }
 
-static void gen6_write_pdes(struct i915_hw_ppgtt *ppgtt)
+static void gen6_map_single(struct i915_hw_ppgtt *ppgtt,
+			    const unsigned pde_index,
+			    dma_addr_t daddr)
 {
 	struct drm_i915_private *dev_priv = ppgtt->base.dev->dev_private;
-	gen6_gtt_pte_t __iomem *pd_addr;
 	uint32_t pd_entry;
+	gen6_gtt_pte_t __iomem *pd_addr =
+		(gen6_gtt_pte_t __iomem*)dev_priv->gtt.gsm + ppgtt->pd_offset / sizeof(gen6_gtt_pte_t);
+
+	pd_entry = GEN6_PDE_ADDR_ENCODE(daddr);
+	pd_entry |= GEN6_PDE_VALID;
+
+	writel(pd_entry, pd_addr + pde_index);
+}
+
+/* Map all the page tables found in the ppgtt structure to incrementing page
+ * directories. */
+static void gen6_map_page_tables(struct i915_hw_ppgtt *ppgtt)
+{
+	struct drm_i915_private *dev_priv = ppgtt->base.dev->dev_private;
 	int i;
 
 	WARN_ON(ppgtt->pd_offset & 0x3f);
-	pd_addr = (gen6_gtt_pte_t __iomem*)dev_priv->gtt.gsm +
-		ppgtt->pd_offset / sizeof(gen6_gtt_pte_t);
-	for (i = 0; i < ppgtt->num_pd_entries; i++) {
-		dma_addr_t pt_addr;
-
-		pt_addr = ppgtt->pt_dma_addr[i];
-		pd_entry = GEN6_PDE_ADDR_ENCODE(pt_addr);
-		pd_entry |= GEN6_PDE_VALID;
+	for (i = 0; i < ppgtt->num_pd_entries; i++)
+		gen6_map_single(ppgtt, i, ppgtt->pt_dma_addr[i]);
 
-		writel(pd_entry, pd_addr + i);
-	}
-	readl(pd_addr);
+	readl(dev_priv->gtt.gsm);
 }
 
 static uint32_t get_pd_offset(struct i915_hw_ppgtt *ppgtt)

@@ -1180,7 +1187,7 @@ static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
 	ppgtt->pd_offset =
 		ppgtt->node.start / PAGE_SIZE * sizeof(gen6_gtt_pte_t);
 
-	gen6_write_pdes(ppgtt);
+	gen6_map_page_tables(ppgtt);
 
 	ppgtt->base.clear_range(&ppgtt->base, 0, ppgtt->base.total, true);
 

@@ -1369,11 +1376,11 @@ void i915_gem_restore_gtt_mappings(struct drm_device *dev)
 		/* TODO: Perhaps it shouldn't be gen6 specific */
 		if (i915_is_ggtt(vm)) {
 			if (dev_priv->mm.aliasing_ppgtt)
-				gen6_write_pdes(dev_priv->mm.aliasing_ppgtt);
+				gen6_map_page_tables(dev_priv->mm.aliasing_ppgtt);
 			continue;
 		}
 
-		gen6_write_pdes(container_of(vm, struct i915_hw_ppgtt, base));
+		gen6_map_page_tables(container_of(vm, struct i915_hw_ppgtt, base));
 	}
 
 	i915_gem_chipset_flush(dev);

index bfa9811..086c533 100644
@@ -621,8 +621,6 @@ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size)
 	ppgtt->base.start = 0;
 	ppgtt->base.total = ppgtt->num_pd_entries * GEN8_PTES_PER_PAGE * PAGE_SIZE;
 
-	ppgtt->base.clear_range(&ppgtt->base, 0, ppgtt->base.total, true);
-
 	DRM_DEBUG_DRIVER("Allocated %d pages for page directories (%d wasted)\n",
 			 ppgtt->num_pd_pages, ppgtt->num_pd_pages - max_pdp);
 	DRM_DEBUG_DRIVER("Allocated %d pages for page tables (%lld wasted)\n",

@@ -1189,8 +1187,6 @@ static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
 
 	gen6_map_page_tables(ppgtt);
 
-	ppgtt->base.clear_range(&ppgtt->base, 0, ppgtt->base.total, true);
-
 	DRM_DEBUG_DRIVER("Allocated pde space (%ldM) at GTT entry: %lx\n",
 			 ppgtt->node.size >> 20,
 			 ppgtt->node.start / PAGE_SIZE);

@@ -1218,6 +1214,7 @@ int i915_gem_init_ppgtt(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt)
 
 	kref_init(&ppgtt->ref);
 	drm_mm_init(&ppgtt->base.mm, ppgtt->base.start, ppgtt->base.total);
+	ppgtt->base.clear_range(&ppgtt->base, 0, ppgtt->base.total, true);
 	i915_init_vm(dev_priv, &ppgtt->base);
 
 	return 0;

index 086c533..a8eb077 100644
@@ -248,7 +248,7 @@ static int gen8_mm_switch(struct i915_hw_ppgtt *ppgtt,
 	int i, ret;
 
 	/* bit of a hack to find the actual last used pd */
-	int used_pd = ppgtt->num_pd_entries / GEN8_PDES_PER_PAGE;
+	int used_pd = ppgtt->num_pd_entries / I915_PDES_PER_PD;
 
 	for (i = used_pd - 1; i >= 0; i--) {
 		dma_addr_t addr = ppgtt->pd_dma_addr[i];

@@ -268,9 +268,9 @@ static void gen8_ppgtt_clear_range(struct i915_address_space *vm,
 	struct i915_hw_ppgtt *ppgtt =
 		container_of(vm, struct i915_hw_ppgtt, base);
 	gen8_gtt_pte_t *pt_vaddr, scratch_pte;
-	unsigned pdpe = start >> GEN8_PDPE_SHIFT & GEN8_PDPE_MASK;
-	unsigned pde = start >> GEN8_PDE_SHIFT & GEN8_PDE_MASK;
-	unsigned pte = start >> GEN8_PTE_SHIFT & GEN8_PTE_MASK;
+	unsigned pdpe = gen8_pdpe_index(start);
+	unsigned pde = gen8_pde_index(start);
+	unsigned pte = gen8_pte_index(start);
 	unsigned num_entries = length >> PAGE_SHIFT;
 	unsigned last_pte, i;
 

@@ -281,8 +281,8 @@ static void gen8_ppgtt_clear_range(struct i915_address_space *vm,
 		struct page *page_table = ppgtt->gen8_pt_pages[pdpe][pde];
 
 		last_pte = pte + num_entries;
-		if (last_pte > GEN8_PTES_PER_PAGE)
-			last_pte = GEN8_PTES_PER_PAGE;
+		if (last_pte > GEN8_PTES_PER_PT)
+			last_pte = GEN8_PTES_PER_PT;
 
 		pt_vaddr = kmap_atomic(page_table);
 

@@ -296,7 +296,7 @@ static void gen8_ppgtt_clear_range(struct i915_address_space *vm,
 		kunmap_atomic(pt_vaddr);
 
 		pte = 0;
-		if (++pde == GEN8_PDES_PER_PAGE) {
+		if (++pde == I915_PDES_PER_PD) {
 			pdpe++;
 			pde = 0;
 		}

@@ -311,9 +311,9 @@ static void gen8_ppgtt_insert_entries(struct i915_address_space *vm,
 	struct i915_hw_ppgtt *ppgtt =
 		container_of(vm, struct i915_hw_ppgtt, base);
 	gen8_gtt_pte_t *pt_vaddr;
-	unsigned pdpe = start >> GEN8_PDPE_SHIFT & GEN8_PDPE_MASK;
-	unsigned pde = start >> GEN8_PDE_SHIFT & GEN8_PDE_MASK;
-	unsigned pte = start >> GEN8_PTE_SHIFT & GEN8_PTE_MASK;
+	unsigned pdpe = gen8_pdpe_index(start);
+	unsigned pde = gen8_pde_index(start);
+	unsigned pte = gen8_pte_index(start);
 	struct sg_page_iter sg_iter;
 
 	pt_vaddr = NULL;

@@ -328,12 +328,12 @@ static void gen8_ppgtt_insert_entries(struct i915_address_space *vm,
 		pt_vaddr[pte] =
 			gen8_pte_encode(sg_page_iter_dma_address(&sg_iter),
 					cache_level, true);
-		if (++pte == GEN8_PTES_PER_PAGE) {
+		if (++pte == GEN8_PTES_PER_PT) {
 			if (!HAS_LLC(ppgtt->base.dev))
 				drm_clflush_virt_range(pt_vaddr, PAGE_SIZE);
 			kunmap_atomic(pt_vaddr);
 			pt_vaddr = NULL;
-			if (++pde == GEN8_PDES_PER_PAGE) {
+			if (++pde == I915_PDES_PER_PD) {
 				pdpe++;
 				pde = 0;
 			}

@@ -354,7 +354,7 @@ static void gen8_free_page_tables(struct page **pt_pages)
 	if (pt_pages == NULL)
 		return;
 
-	for (i = 0; i < GEN8_PDES_PER_PAGE; i++)
+	for (i = 0; i < I915_PDES_PER_PD; i++)
 		if (pt_pages[i])
 			__free_pages(pt_pages[i], 0);
 }

@@ -386,7 +386,7 @@ static void gen8_ppgtt_dma_unmap_pages(struct i915_hw_ppgtt *ppgtt)
 		pci_unmap_page(hwdev, ppgtt->pd_dma_addr[i], PAGE_SIZE,
 			       PCI_DMA_BIDIRECTIONAL);
 
-		for (j = 0; j < GEN8_PDES_PER_PAGE; j++) {
+		for (j = 0; j < I915_PDES_PER_PD; j++) {
 			dma_addr_t addr = ppgtt->gen8_pt_dma_addr[i][j];
 			if (addr)
 				pci_unmap_page(hwdev, addr, PAGE_SIZE,

@@ -412,11 +412,11 @@ static struct page **__gen8_alloc_page_tables(void)
 	struct page **pt_pages;
 	int i;
 
-	pt_pages = kcalloc(GEN8_PDES_PER_PAGE, sizeof(struct page *), GFP_KERNEL);
+	pt_pages = kcalloc(I915_PDES_PER_PD, sizeof(struct page *), GFP_KERNEL);
 	if (!pt_pages)
 		return ERR_PTR(-ENOMEM);
 
-	for (i = 0; i < GEN8_PDES_PER_PAGE; i++) {
+	for (i = 0; i < I915_PDES_PER_PD; i++) {
 		pt_pages[i] = alloc_page(GFP_KERNEL | __GFP_ZERO);
 		if (!pt_pages[i])
 			goto bail;

@@ -466,7 +466,7 @@ static int gen8_ppgtt_allocate_dma(struct i915_hw_ppgtt *ppgtt)
 	int i;
 
 	for (i = 0; i < ppgtt->num_pd_pages; i++) {
-		ppgtt->gen8_pt_dma_addr[i] = kcalloc(GEN8_PDES_PER_PAGE,
+		ppgtt->gen8_pt_dma_addr[i] = kcalloc(I915_PDES_PER_PD,
 						     sizeof(dma_addr_t),
 						     GFP_KERNEL);
 		if (!ppgtt->gen8_pt_dma_addr[i])

@@ -505,7 +505,7 @@ static int gen8_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt,
 		return ret;
 	}
 
-	ppgtt->num_pd_entries = max_pdp * GEN8_PDES_PER_PAGE;
+	ppgtt->num_pd_entries = max_pdp * I915_PDES_PER_PD;
 
 	ret = gen8_ppgtt_allocate_dma(ppgtt);
 	if (ret)

@@ -566,7 +566,7 @@ static int gen8_ppgtt_setup_page_tables(struct i915_hw_ppgtt *ppgtt,
 static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size)
 {
 	const int max_pdp = DIV_ROUND_UP(size, 1 << 30);
-	const int min_pt_pages = GEN8_PDES_PER_PAGE * max_pdp;
+	const int min_pt_pages = I915_PDES_PER_PD * max_pdp;
 	int i, j, ret;
 
 	if (size % (1<<30))

@@ -585,7 +585,7 @@ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size)
 		if (ret)
 			goto bail;
 
-		for (j = 0; j < GEN8_PDES_PER_PAGE; j++) {
+		for (j = 0; j < I915_PDES_PER_PD; j++) {
 			ret = gen8_ppgtt_setup_page_tables(ppgtt, i, j);
 			if (ret)
 				goto bail;

@@ -603,7 +603,7 @@ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size)
 	for (i = 0; i < max_pdp; i++) {
 		gen8_ppgtt_pde_t *pd_vaddr;
 		pd_vaddr = kmap_atomic(&ppgtt->pd_pages[i]);
-		for (j = 0; j < GEN8_PDES_PER_PAGE; j++) {
+		for (j = 0; j < I915_PDES_PER_PD; j++) {
 			dma_addr_t addr = ppgtt->gen8_pt_dma_addr[i][j];
 			pd_vaddr[j] = gen8_pde_encode(ppgtt->base.dev, addr,
 						      I915_CACHE_LLC);

@@ -619,7 +619,7 @@ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size)
 	ppgtt->base.insert_entries = gen8_ppgtt_insert_entries;
 	ppgtt->base.cleanup = gen8_ppgtt_cleanup;
 	ppgtt->base.start = 0;
-	ppgtt->base.total = ppgtt->num_pd_entries * GEN8_PTES_PER_PAGE * PAGE_SIZE;
+	ppgtt->base.total = ppgtt->num_pd_entries * GEN8_PTES_PER_PT * PAGE_SIZE;
 
 	DRM_DEBUG_DRIVER("Allocated %d pages for page directories (%d wasted)\n",
 			 ppgtt->num_pd_pages, ppgtt->num_pd_pages - max_pdp);

@@ -665,9 +665,9 @@ static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m)
 		seq_printf(m, "\tPDE: %x\n", pd_entry);
 
 		pt_vaddr = kmap_atomic(ppgtt->pt_pages[pde]);
-		for (pte = 0; pte < I915_PPGTT_PT_ENTRIES; pte+=4) {
+		for (pte = 0; pte < GEN6_PTES_PER_PT; pte+=4) {
 			unsigned long va =
-				(pde * PAGE_SIZE * I915_PPGTT_PT_ENTRIES) +
+				(pde * PAGE_SIZE * GEN6_PTES_PER_PT) +
 				(pte * PAGE_SIZE);
 			int i;
 			bool found = false;

@@ -946,29 +946,28 @@ static void gen6_ppgtt_clear_range(struct i915_address_space *vm,
 	struct i915_hw_ppgtt *ppgtt =
 		container_of(vm, struct i915_hw_ppgtt, base);
 	gen6_gtt_pte_t *pt_vaddr, scratch_pte;
-	unsigned first_entry = start >> PAGE_SHIFT;
+	unsigned pde = gen6_pde_index(start);
 	unsigned num_entries = length >> PAGE_SHIFT;
-	unsigned act_pt = first_entry / I915_PPGTT_PT_ENTRIES;
-	unsigned first_pte = first_entry % I915_PPGTT_PT_ENTRIES;
+	unsigned pte = gen6_pte_index(start);
 	unsigned last_pte, i;
 
 	scratch_pte = vm->pte_encode(vm->scratch.addr, I915_CACHE_LLC, true);
 
 	while (num_entries) {
-		last_pte = first_pte + num_entries;
-		if (last_pte > I915_PPGTT_PT_ENTRIES)
-			last_pte = I915_PPGTT_PT_ENTRIES;
+		last_pte = pte + num_entries;
+		if (last_pte > GEN6_PTES_PER_PT)
+			last_pte = GEN6_PTES_PER_PT;
 
-		pt_vaddr = kmap_atomic(ppgtt->pt_pages[act_pt]);
+		pt_vaddr = kmap_atomic(ppgtt->pt_pages[pde]);
 
-		for (i = first_pte; i < last_pte; i++)
+		for (i = pte; i < last_pte; i++)
 			pt_vaddr[i] = scratch_pte;
 
 		kunmap_atomic(pt_vaddr);
 
-		num_entries -= last_pte - first_pte;
-		first_pte = 0;
-		act_pt++;
+		num_entries -= last_pte - pte;
+		pte = 0;
+		pde++;
 	}
 }
 

@@ -980,24 +979,23 @@ static void gen6_ppgtt_insert_entries(struct i915_address_space *vm,
 	struct i915_hw_ppgtt *ppgtt =
 		container_of(vm, struct i915_hw_ppgtt, base);
 	gen6_gtt_pte_t *pt_vaddr;
-	unsigned first_entry = start >> PAGE_SHIFT;
-	unsigned act_pt = first_entry / I915_PPGTT_PT_ENTRIES;
-	unsigned act_pte = first_entry % I915_PPGTT_PT_ENTRIES;
+	unsigned pde = gen6_pde_index(start);
+	unsigned pte = gen6_pte_index(start);
 	struct sg_page_iter sg_iter;
 
 	pt_vaddr = NULL;
 	for_each_sg_page(pages->sgl, &sg_iter, pages->nents, 0) {
 		if (pt_vaddr == NULL)
-			pt_vaddr = kmap_atomic(ppgtt->pt_pages[act_pt]);
+			pt_vaddr = kmap_atomic(ppgtt->pt_pages[pde]);
 
-		pt_vaddr[act_pte] =
+		pt_vaddr[pte] =
 			vm->pte_encode(sg_page_iter_dma_address(&sg_iter),
 				       cache_level, true);
-		if (++act_pte == I915_PPGTT_PT_ENTRIES) {
+		if (++pte == GEN6_PTES_PER_PT) {
 			kunmap_atomic(pt_vaddr);
 			pt_vaddr = NULL;
-			act_pt++;
-			act_pte = 0;
+			pde++;
+			pte = 0;
 		}
 	}
 	if (pt_vaddr)

@@ -1074,7 +1072,7 @@ alloc:
 	if (ppgtt->node.start < dev_priv->gtt.mappable_end)
 		DRM_DEBUG("Forced to use aperture for PDEs\n");
 
-	ppgtt->num_pd_entries = GEN6_PPGTT_PD_ENTRIES;
+	ppgtt->num_pd_entries = I915_PDES_PER_PD;
 	return 0;
 }
 

@@ -1179,7 +1177,7 @@ static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
 	ppgtt->base.insert_entries = gen6_ppgtt_insert_entries;
 	ppgtt->base.cleanup = gen6_ppgtt_cleanup;
 	ppgtt->base.start = 0;
-	ppgtt->base.total =  ppgtt->num_pd_entries * I915_PPGTT_PT_ENTRIES * PAGE_SIZE;
+	ppgtt->base.total =  ppgtt->num_pd_entries * GEN6_PTES_PER_PT * PAGE_SIZE;
 	ppgtt->debug_dump = gen6_dump_ppgtt;
 
 	ppgtt->pd_offset =

158 lines not on this page · raw message

index a8eb077..f2478c9 100644
@@ -278,7 +278,8 @@ static void gen8_ppgtt_clear_range(struct i915_address_space *vm,
 				      I915_CACHE_LLC, use_scratch);
 
 	while (num_entries) {
-		struct page *page_table = ppgtt->gen8_pt_pages[pdpe][pde];
+		struct i915_pagedir *pd = &ppgtt->pdp.pagedir[pdpe];
+		struct page *page_table = pd->page_tables[pde].page;
 
 		last_pte = pte + num_entries;
 		if (last_pte > GEN8_PTES_PER_PT)

@@ -322,8 +323,11 @@ static void gen8_ppgtt_insert_entries(struct i915_address_space *vm,
 		if (WARN_ON(pdpe >= GEN8_LEGACY_PDPES))
 			break;
 
-		if (pt_vaddr == NULL)
-			pt_vaddr = kmap_atomic(ppgtt->gen8_pt_pages[pdpe][pde]);
+		if (pt_vaddr == NULL) {
+			struct i915_pagedir *pd = &ppgtt->pdp.pagedir[pdpe];
+			struct page *page_table = pd->page_tables[pde].page;
+			pt_vaddr = kmap_atomic(page_table);
+		}
 
 		pt_vaddr[pte] =
 			gen8_pte_encode(sg_page_iter_dma_address(&sg_iter),

@@ -347,29 +351,33 @@ static void gen8_ppgtt_insert_entries(struct i915_address_space *vm,
 	}
 }
 
-static void gen8_free_page_tables(struct page **pt_pages)
+static void gen8_free_page_tables(struct i915_pagedir *pd)
 {
 	int i;
 
-	if (pt_pages == NULL)
+	if (pd->page_tables == NULL)
 		return;
 
 	for (i = 0; i < I915_PDES_PER_PD; i++)
-		if (pt_pages[i])
-			__free_pages(pt_pages[i], 0);
+		if (pd->page_tables[i].page)
+			__free_page(pd->page_tables[i].page);
+}
+
+static void gen8_free_page_directories(struct i915_pagedir *pd)
+{
+	kfree(pd->page_tables);
+	__free_page(pd->page);
 }
 
-static void gen8_ppgtt_free(const struct i915_hw_ppgtt *ppgtt)
+static void gen8_ppgtt_free(struct i915_hw_ppgtt *ppgtt)
 {
 	int i;
 
 	for (i = 0; i < ppgtt->num_pd_pages; i++) {
-		gen8_free_page_tables(ppgtt->gen8_pt_pages[i]);
-		kfree(ppgtt->gen8_pt_pages[i]);
+		gen8_free_page_tables(&ppgtt->pdp.pagedir[i]);
+		gen8_free_page_directories(&ppgtt->pdp.pagedir[i]);
 		kfree(ppgtt->gen8_pt_dma_addr[i]);
 	}
-
-	__free_pages(ppgtt->pd_pages, get_order(ppgtt->num_pd_pages << PAGE_SHIFT));
 }
 
 static void gen8_ppgtt_dma_unmap_pages(struct i915_hw_ppgtt *ppgtt)

@@ -407,87 +415,73 @@ static void gen8_ppgtt_cleanup(struct i915_address_space *vm)
 	gen8_ppgtt_free(ppgtt);
 }
 
-static struct page **__gen8_alloc_page_tables(void)
+static int gen8_ppgtt_allocate_dma(struct i915_hw_ppgtt *ppgtt)
 {
-	struct page **pt_pages;
 	int i;
 
-	pt_pages = kcalloc(I915_PDES_PER_PD, sizeof(struct page *), GFP_KERNEL);
-	if (!pt_pages)
-		return ERR_PTR(-ENOMEM);
-
-	for (i = 0; i < I915_PDES_PER_PD; i++) {
-		pt_pages[i] = alloc_page(GFP_KERNEL | __GFP_ZERO);
-		if (!pt_pages[i])
-			goto bail;
+	for (i = 0; i < ppgtt->num_pd_pages; i++) {
+		ppgtt->gen8_pt_dma_addr[i] = kcalloc(I915_PDES_PER_PD,
+						     sizeof(dma_addr_t),
+						     GFP_KERNEL);
+		if (!ppgtt->gen8_pt_dma_addr[i])
+			return -ENOMEM;
 	}
 
-	return pt_pages;
-
-bail:
-	gen8_free_page_tables(pt_pages);
-	kfree(pt_pages);
-	return ERR_PTR(-ENOMEM);
+	return 0;
 }
 
-static int gen8_ppgtt_allocate_page_tables(struct i915_hw_ppgtt *ppgtt,
-					   const int max_pdp)
+static int gen8_ppgtt_allocate_page_tables(struct i915_hw_ppgtt *ppgtt)
 {
-	struct page **pt_pages[GEN8_LEGACY_PDPES];
-	int i, ret;
+	int i, j;
 
-	for (i = 0; i < max_pdp; i++) {
-		pt_pages[i] = __gen8_alloc_page_tables();
-		if (IS_ERR(pt_pages[i])) {
-			ret = PTR_ERR(pt_pages[i]);
-			goto unwind_out;
+	for (i = 0; i < ppgtt->num_pd_pages; i++) {
+		for (j = 0; j < I915_PDES_PER_PD; j++) {
+			struct i915_pagetab *pt = &ppgtt->pdp.pagedir[i].page_tables[j];
+			pt->page = alloc_page(GFP_KERNEL | __GFP_ZERO);
+			if (!pt->page)
+				goto unwind_out;
 		}
 	}
 
-	/* NB: Avoid touching gen8_pt_pages until last to keep the allocation,
-	 * "atomic" - for cleanup purposes.
-	 */
-	for (i = 0; i < max_pdp; i++)
-		ppgtt->gen8_pt_pages[i] = pt_pages[i];
-
 	return 0;
 
 unwind_out:
-	while (i--) {
-		gen8_free_page_tables(pt_pages[i]);
-		kfree(pt_pages[i]);
-	}
+	while (i--)
+		gen8_free_page_tables(&ppgtt->pdp.pagedir[i]);
 
-	return ret;
+	return -ENOMEM;
 }
 
-static int gen8_ppgtt_allocate_dma(struct i915_hw_ppgtt *ppgtt)
+static int gen8_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt,
+						const int max_pdp)
 {
 	int i;
 
-	for (i = 0; i < ppgtt->num_pd_pages; i++) {
-		ppgtt->gen8_pt_dma_addr[i] = kcalloc(I915_PDES_PER_PD,
-						     sizeof(dma_addr_t),
-						     GFP_KERNEL);
-		if (!ppgtt->gen8_pt_dma_addr[i])
-			return -ENOMEM;
-	}
+	for (i = 0; i < max_pdp; i++) {
+		struct i915_pagetab *pt;
+		pt = kcalloc(I915_PDES_PER_PD, sizeof(*pt), GFP_KERNEL);
+		if (!pt)
+			goto unwind_out;
 
-	return 0;
-}
+		ppgtt->pdp.pagedir[i].page = alloc_page(GFP_KERNEL | __GFP_ZERO);
+		if (!ppgtt->pdp.pagedir[i].page)
+			goto unwind_out;
 
-static int gen8_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt,
-						const int max_pdp)
-{
-	ppgtt->pd_pages = alloc_pages(GFP_KERNEL | __GFP_ZERO,
-				      get_order(max_pdp << PAGE_SHIFT));
-	if (!ppgtt->pd_pages)
-		return -ENOMEM;
+		ppgtt->pdp.pagedir[i].page_tables = pt;
+	}
 
-	ppgtt->num_pd_pages = 1 << get_order(max_pdp << PAGE_SHIFT);
+	ppgtt->num_pd_pages = max_pdp;
 	BUG_ON(ppgtt->num_pd_pages > GEN8_LEGACY_PDPES);
 
 	return 0;
+
+unwind_out:
+	while (i--) {
+		kfree(ppgtt->pdp.pagedir[i].page_tables);
+		__free_page(ppgtt->pdp.pagedir[i].page);
+	}
+
+	return -ENOMEM;
 }
 
 static int gen8_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt,

@@ -499,18 +493,19 @@ static int gen8_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt,
 	if (ret)
 		return ret;
 
-	ret = gen8_ppgtt_allocate_page_tables(ppgtt, max_pdp);
-	if (ret) {
-		__free_pages(ppgtt->pd_pages, get_order(max_pdp << PAGE_SHIFT));
-		return ret;
-	}
+	ret = gen8_ppgtt_allocate_page_tables(ppgtt);
+	if (ret)
+		goto err_out;
 
 	ppgtt->num_pd_entries = max_pdp * I915_PDES_PER_PD;
 
 	ret = gen8_ppgtt_allocate_dma(ppgtt);
-	if (ret)
-		gen8_ppgtt_free(ppgtt);
+	if (!ret)
+		return ret;
 
+	/* TODO: Check this for all cases */
+err_out:
+	gen8_ppgtt_free(ppgtt);
 	return ret;
 }
 

@@ -521,7 +516,7 @@ static int gen8_ppgtt_setup_page_directories(struct i915_hw_ppgtt *ppgtt,
 	int ret;
 
 	pd_addr = pci_map_page(ppgtt->base.dev->pdev,
-			       &ppgtt->pd_pages[pdpe], 0,
+			       ppgtt->pdp.pagedir[pdpe].page, 0,
 			       PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
 
 	ret = pci_dma_mapping_error(ppgtt->base.dev->pdev, pd_addr);

@@ -541,7 +536,7 @@ static int gen8_ppgtt_setup_page_tables(struct i915_hw_ppgtt *ppgtt,
 	struct page *p;
 	int ret;
 
-	p = ppgtt->gen8_pt_pages[pdpe][pde];
+	p = ppgtt->pdp.pagedir[pdpe].page_tables[pde].page;
 	pt_addr = pci_map_page(ppgtt->base.dev->pdev,
 			       p, 0, PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
 	ret = pci_dma_mapping_error(ppgtt->base.dev->pdev, pt_addr);

@@ -602,7 +597,7 @@ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size)
 	 */
 	for (i = 0; i < max_pdp; i++) {
 		gen8_ppgtt_pde_t *pd_vaddr;
-		pd_vaddr = kmap_atomic(&ppgtt->pd_pages[i]);
+		pd_vaddr = kmap_atomic(ppgtt->pdp.pagedir[i].page);
 		for (j = 0; j < I915_PDES_PER_PD; j++) {
 			dma_addr_t addr = ppgtt->gen8_pt_dma_addr[i][j];
 			pd_vaddr[j] = gen8_pde_encode(ppgtt->base.dev, addr,

@@ -664,7 +659,7 @@ static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m)
 				   expected);
 		seq_printf(m, "\tPDE: %x\n", pd_entry);
 
-		pt_vaddr = kmap_atomic(ppgtt->pt_pages[pde]);
+		pt_vaddr = kmap_atomic(ppgtt->pd.page_tables[pde].page);
 		for (pte = 0; pte < GEN6_PTES_PER_PT; pte+=4) {
 			unsigned long va =
 				(pde * PAGE_SIZE * GEN6_PTES_PER_PT) +

@@ -958,7 +953,7 @@ static void gen6_ppgtt_clear_range(struct i915_address_space *vm,
 		if (last_pte > GEN6_PTES_PER_PT)
 			last_pte = GEN6_PTES_PER_PT;
 
-		pt_vaddr = kmap_atomic(ppgtt->pt_pages[pde]);
+		pt_vaddr = kmap_atomic(ppgtt->pd.page_tables[pde].page);
 
 		for (i = pte; i < last_pte; i++)
 			pt_vaddr[i] = scratch_pte;

@@ -986,7 +981,7 @@ static void gen6_ppgtt_insert_entries(struct i915_address_space *vm,
 	pt_vaddr = NULL;
 	for_each_sg_page(pages->sgl, &sg_iter, pages->nents, 0) {
 		if (pt_vaddr == NULL)
-			pt_vaddr = kmap_atomic(ppgtt->pt_pages[pde]);
+			pt_vaddr = kmap_atomic(ppgtt->pd.page_tables[pde].page);
 
 		pt_vaddr[pte] =
 			vm->pte_encode(sg_page_iter_dma_address(&sg_iter),

@@ -1020,8 +1015,8 @@ static void gen6_ppgtt_free(struct i915_hw_ppgtt *ppgtt)
 
 	kfree(ppgtt->pt_dma_addr);
 	for (i = 0; i < ppgtt->num_pd_entries; i++)
-		__free_page(ppgtt->pt_pages[i]);
-	kfree(ppgtt->pt_pages);
+		__free_page(ppgtt->pd.page_tables[i].page);
+	kfree(ppgtt->pd.page_tables);
 }
 
 static void gen6_ppgtt_cleanup(struct i915_address_space *vm)

@@ -1078,22 +1073,22 @@ alloc:
 
 static int gen6_ppgtt_allocate_page_tables(struct i915_hw_ppgtt *ppgtt)
 {
+	struct i915_pagetab *pt;
 	int i;
 
-	ppgtt->pt_pages = kcalloc(ppgtt->num_pd_entries, sizeof(struct page *),
-				  GFP_KERNEL);
-
-	if (!ppgtt->pt_pages)
+	pt = kcalloc(ppgtt->num_pd_entries, sizeof(*pt), GFP_KERNEL);
+	if (!pt)
 		return -ENOMEM;
 
 	for (i = 0; i < ppgtt->num_pd_entries; i++) {
-		ppgtt->pt_pages[i] = alloc_page(GFP_KERNEL | __GFP_ZERO);
-		if (!ppgtt->pt_pages[i]) {
+		pt[i].page = alloc_page(GFP_KERNEL | __GFP_ZERO);
+		if (!pt->page) {
 			gen6_ppgtt_free(ppgtt);
 			return -ENOMEM;
 		}
 	}
 
+	ppgtt->pd.page_tables = pt;
 	return 0;
 }
 

@@ -1128,9 +1123,11 @@ static int gen6_ppgtt_setup_page_tables(struct i915_hw_ppgtt *ppgtt)
 	int i;
 
 	for (i = 0; i < ppgtt->num_pd_entries; i++) {
+		struct page *page;
 		dma_addr_t pt_addr;
 
-		pt_addr = pci_map_page(dev->pdev, ppgtt->pt_pages[i], 0, 4096,
+		page = ppgtt->pd.page_tables[i].page;
+		pt_addr = pci_map_page(dev->pdev, page, 0, 4096,
 				       PCI_DMA_BIDIRECTIONAL);
 
 		if (pci_dma_mapping_error(dev->pdev, pt_addr)) {

@@ -1177,7 +1174,7 @@ static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
 	ppgtt->base.insert_entries = gen6_ppgtt_insert_entries;
 	ppgtt->base.cleanup = gen6_ppgtt_cleanup;
 	ppgtt->base.start = 0;
-	ppgtt->base.total =  ppgtt->num_pd_entries * GEN6_PTES_PER_PT * PAGE_SIZE;
+	ppgtt->base.total = ppgtt->num_pd_entries * GEN6_PTES_PER_PT * PAGE_SIZE;
 	ppgtt->debug_dump = gen6_dump_ppgtt;
 
 	ppgtt->pd_offset =

index 3d3337b..cddd1e8 100644
@@ -252,6 +252,20 @@ struct i915_gtt {
 			  unsigned long *mappable_end);
 };
 
+struct i915_pagetab {
+	struct page *page;
+};
+
+struct i915_pagedir {
+	struct page *page; /* NULL for GEN6-GEN7 */
+	struct i915_pagetab *page_tables;
+};
+
+struct i915_pagedirpo {
+	/* struct page *page; */
+	struct i915_pagedir pagedir[GEN8_LEGACY_PDPES];
+};
+
 struct i915_hw_ppgtt {
 	struct i915_address_space base;
 	struct kref ref;

@@ -259,11 +273,6 @@ struct i915_hw_ppgtt {
 	unsigned num_pd_entries;
 	unsigned num_pd_pages; /* gen8+ */
 	union {
-		struct page **pt_pages;
-		struct page **gen8_pt_pages[GEN8_LEGACY_PDPES];
-	};
-	struct page *pd_pages;
-	union {
 		uint32_t pd_offset;
 		dma_addr_t pd_dma_addr[GEN8_LEGACY_PDPES];
 	};

@@ -271,7 +280,10 @@ struct i915_hw_ppgtt {
 		dma_addr_t *pt_dma_addr;
 		dma_addr_t *gen8_pt_dma_addr[GEN8_LEGACY_PDPES];
 	};
-
+	union {
+		struct i915_pagedirpo pdp;
+		struct i915_pagedir pd;
+	};
 	struct i915_hw_context *ctx;
 
 	int (*enable)(struct i915_hw_ppgtt *ppgtt);

index 4a0b1c8..64051b0 100644
@@ -1815,7 +1815,7 @@ static int i915_swizzle_info(struct seq_file *m, void *data)
 static void print_ppgtt(struct seq_file *m, struct i915_hw_ppgtt *ppgtt, const char *name)
 {
 	seq_printf(m, "%s:\n", name);
-	seq_printf(m, "pd gtt offset: 0x%08x\n", ppgtt->pd_offset);
+	seq_printf(m, "pd gtt offset: 0x%08x\n", ppgtt->pd.pd_offset);
 }
 
 static void gen8_ppgtt_info(struct seq_file *m, struct drm_device *dev, int verbose)

216 lines not on this page · raw message

index cddd1e8..07a5cd4 100644
@@ -254,10 +254,16 @@ struct i915_gtt {
 
 struct i915_pagetab {
 	struct page *page;
+	dma_addr_t daddr;
 };
 
 struct i915_pagedir {
 	struct page *page; /* NULL for GEN6-GEN7 */
+	union {
+		uint32_t pd_offset;
+		dma_addr_t daddr;
+	};
+
 	struct i915_pagetab *page_tables;
 };
 

@@ -273,17 +279,10 @@ struct i915_hw_ppgtt {
 	unsigned num_pd_entries;
 	unsigned num_pd_pages; /* gen8+ */
 	union {
-		uint32_t pd_offset;
-		dma_addr_t pd_dma_addr[GEN8_LEGACY_PDPES];
-	};
-	union {
-		dma_addr_t *pt_dma_addr;
-		dma_addr_t *gen8_pt_dma_addr[GEN8_LEGACY_PDPES];
-	};
-	union {
 		struct i915_pagedirpo pdp;
 		struct i915_pagedir pd;
 	};
+
 	struct i915_hw_context *ctx;
 
 	int (*enable)(struct i915_hw_ppgtt *ppgtt);

index 50d2af8..5d691cd 100644
@@ -756,7 +756,6 @@ static void i915_gem_record_fences(struct drm_device *dev,
 	}
 }
 
-
 static void gen8_record_semaphore_state(struct drm_i915_private *dev_priv,
 					struct drm_i915_error_state *error,
 					struct intel_ring_buffer *ring,

index 1f186d3..35370eb 100644
@@ -211,6 +211,102 @@ static gen6_gtt_pte_t iris_pte_encode(dma_addr_t addr,
 	return pte;
 }
 
+static void free_pt_single(struct i915_pagetab *pt)
+{
+	if (WARN_ON(!pt->page))
+		return;
+	__free_page(pt->page);
+	kfree(pt);
+}
+
+static struct i915_pagetab *alloc_pt_single(void)
+{
+	struct i915_pagetab *pt;
+
+	pt = kzalloc(sizeof(*pt), GFP_KERNEL);
+	if (!pt)
+		return ERR_PTR(-ENOMEM);
+
+	pt->page = alloc_page(GFP_KERNEL | __GFP_ZERO);
+	if (!pt->page) {
+		kfree(pt);
+		return ERR_PTR(-ENOMEM);
+	}
+
+	return pt;
+}
+
+/**
+ * alloc_pt_range() - Allocate a multiple page tables
+ * @pd:		The page directory which will have at least @count entries
+ *		available to point to the allocated page tables.
+ * @pde:	First page directory entry for which we are allocating.
+ * @count:	Number of pages to allocate.
+ *
+ * Allocates multiple page table pages and sets the appropriate entries in the
+ * page table structure within the page directory. Function cleans up after
+ * itself on any failures.
+ *
+ * Return: 0 if allocation succeeded.
+ */
+static int alloc_pt_range(struct i915_pagedir *pd, uint16_t pde, size_t count)
+{
+	int i, ret;
+
+	/* 512 is the max page tables per pagedir on any platform.
+	 * TODO: make WARN after patch series is done
+	 */
+	BUG_ON(pde + count > I915_PDES_PER_PD);
+
+	for (i = pde; i < pde + count; i++) {
+		struct i915_pagetab *pt = alloc_pt_single();
+		if (IS_ERR(pt)) {
+			ret = PTR_ERR(pt);
+			goto err_out;
+		}
+		WARN(pd->page_tables[i],
+		     "Leaking page directory entry %d (%pa)\n",
+		     i, pd->page_tables[i]);
+		pd->page_tables[i] = pt;
+	}
+
+	return 0;
+
+err_out:
+	while (i--)
+		free_pt_single(pd->page_tables[i]);
+	return ret;
+}
+
+static void __free_pd_single(struct i915_pagedir *pd)
+{
+	__free_page(pd->page);
+	kfree(pd);
+}
+
+#define free_pd_single(pd) do { \
+	if ((pd)->page) { \
+		__free_pd_single(pd); \
+	} \
+} while (0)
+
+static struct i915_pagedir *alloc_pd_single(void)
+{
+	struct i915_pagedir *pd;
+
+	pd = kzalloc(sizeof(*pd), GFP_KERNEL);
+	if (!pd)
+		return ERR_PTR(-ENOMEM);
+
+	pd->page = alloc_page(GFP_KERNEL | __GFP_ZERO);
+	if (!pd->page) {
+		kfree(pd);
+		return ERR_PTR(-ENOMEM);
+	}
+
+	return pd;
+}
+
 /* Broadwell Page Directory Pointer Descriptors */
 static int gen8_write_pdp(struct intel_ring_buffer *ring, unsigned entry,
 			   uint64_t val, bool synchronous)

@@ -251,7 +347,7 @@ static int gen8_mm_switch(struct i915_hw_ppgtt *ppgtt,
 	int used_pd = ppgtt->num_pd_entries / I915_PDES_PER_PD;
 
 	for (i = used_pd - 1; i >= 0; i--) {
-		dma_addr_t addr = ppgtt->pdp.pagedir[i].daddr;
+		dma_addr_t addr = ppgtt->pdp.pagedir[i]->daddr;
 		ret = gen8_write_pdp(ring, i, addr, synchronous);
 		if (ret)
 			return ret;

@@ -278,8 +374,9 @@ static void gen8_ppgtt_clear_range(struct i915_address_space *vm,
 				      I915_CACHE_LLC, use_scratch);
 
 	while (num_entries) {
-		struct i915_pagedir *pd = &ppgtt->pdp.pagedir[pdpe];
-		struct page *page_table = pd->page_tables[pde].page;
+		struct i915_pagedir *pd = ppgtt->pdp.pagedir[pdpe];
+		struct i915_pagetab *pt = pd->page_tables[pde];
+		struct page *page_table = pt->page;
 
 		last_pte = pte + num_entries;
 		if (last_pte > GEN8_PTES_PER_PT)

@@ -324,8 +421,9 @@ static void gen8_ppgtt_insert_entries(struct i915_address_space *vm,
 			break;
 
 		if (pt_vaddr == NULL) {
-			struct i915_pagedir *pd = &ppgtt->pdp.pagedir[pdpe];
-			struct page *page_table = pd->page_tables[pde].page;
+			struct i915_pagedir *pd = ppgtt->pdp.pagedir[pdpe];
+			struct i915_pagetab *pt = pd->page_tables[pde];
+			struct page *page_table = pt->page;
 			pt_vaddr = kmap_atomic(page_table);
 		}
 

@@ -355,18 +453,13 @@ static void gen8_free_page_tables(struct i915_pagedir *pd)
 {
 	int i;
 
-	if (pd->page_tables == NULL)
+	if (!pd->page)
 		return;
 
-	for (i = 0; i < I915_PDES_PER_PD; i++)
-		if (pd->page_tables[i].page)
-			__free_page(pd->page_tables[i].page);
-}
-
-static void gen8_free_page_directories(struct i915_pagedir *pd)
-{
-	kfree(pd->page_tables);
-	__free_page(pd->page);
+	for (i = 0; i < I915_PDES_PER_PD; i++) {
+		free_pt_single(pd->page_tables[i]);
+		pd->page_tables[i] = NULL;
+	}
 }
 
 static void gen8_ppgtt_free(struct i915_hw_ppgtt *ppgtt)

@@ -374,8 +467,8 @@ static void gen8_ppgtt_free(struct i915_hw_ppgtt *ppgtt)
 	int i;
 
 	for (i = 0; i < ppgtt->num_pd_pages; i++) {
-		gen8_free_page_tables(&ppgtt->pdp.pagedir[i]);
-		gen8_free_page_directories(&ppgtt->pdp.pagedir[i]);
+		gen8_free_page_tables(ppgtt->pdp.pagedir[i]);
+		free_pd_single(ppgtt->pdp.pagedir[i]);
 	}
 }
 

@@ -387,14 +480,16 @@ static void gen8_ppgtt_dma_unmap_pages(struct i915_hw_ppgtt *ppgtt)
 	for (i = 0; i < ppgtt->num_pd_pages; i++) {
 		/* TODO: In the future we'll support sparse mappings, so this
 		 * will have to change. */
-		if (!ppgtt->pdp.pagedir[i].daddr)
+		if (!ppgtt->pdp.pagedir[i]->daddr)
 			continue;
 
-		pci_unmap_page(hwdev, ppgtt->pdp.pagedir[i].daddr, PAGE_SIZE,
+		pci_unmap_page(hwdev, ppgtt->pdp.pagedir[i]->daddr, PAGE_SIZE,
 			       PCI_DMA_BIDIRECTIONAL);
 
 		for (j = 0; j < I915_PDES_PER_PD; j++) {
-			dma_addr_t addr = ppgtt->pdp.pagedir[i].page_tables[j].daddr;
+			struct i915_pagedir *pd = ppgtt->pdp.pagedir[i];
+			struct i915_pagetab *pt =  pd->page_tables[j];
+			dma_addr_t addr = pt->daddr;
 			if (addr)
 				pci_unmap_page(hwdev, addr, PAGE_SIZE,
 					       PCI_DMA_BIDIRECTIONAL);

@@ -416,24 +511,20 @@ static void gen8_ppgtt_cleanup(struct i915_address_space *vm)
 
 static int gen8_ppgtt_allocate_page_tables(struct i915_hw_ppgtt *ppgtt)
 {
-	int i, j;
+	int i, ret;
 
 	for (i = 0; i < ppgtt->num_pd_pages; i++) {
-		struct i915_pagedir *pd = &ppgtt->pdp.pagedir[i];
-		for (j = 0; j < I915_PDES_PER_PD; j++) {
-			struct i915_pagetab *pt = &pd->page_tables[j];
-			pt->page = alloc_page(GFP_KERNEL | __GFP_ZERO);
-			if (!pt->page)
-				goto unwind_out;
-
-		}
+		ret = alloc_pt_range(ppgtt->pdp.pagedir[i],
+				     0, I915_PDES_PER_PD);
+		if (ret)
+			goto unwind_out;
 	}
 
 	return 0;
 
 unwind_out:
 	while (i--)
-		gen8_free_page_tables(&ppgtt->pdp.pagedir[i]);
+		gen8_free_page_tables(ppgtt->pdp.pagedir[i]);
 
 	return -ENOMEM;
 }

@@ -444,16 +535,9 @@ static int gen8_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt,
 	int i;
 
 	for (i = 0; i < max_pdp; i++) {
-		struct i915_pagetab *pt;
-		pt = kcalloc(I915_PDES_PER_PD, sizeof(*pt), GFP_KERNEL);
-		if (!pt)
+		ppgtt->pdp.pagedir[i] = alloc_pd_single();
+		if (IS_ERR(ppgtt->pdp.pagedir[i]))
 			goto unwind_out;
-
-		ppgtt->pdp.pagedir[i].page = alloc_page(GFP_KERNEL | __GFP_ZERO);
-		if (!ppgtt->pdp.pagedir[i].page)
-			goto unwind_out;
-
-		ppgtt->pdp.pagedir[i].page_tables = pt;
 	}
 
 	ppgtt->num_pd_pages = max_pdp;

@@ -462,10 +546,8 @@ static int gen8_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt,
 	return 0;
 
 unwind_out:
-	while (i--) {
-		kfree(ppgtt->pdp.pagedir[i].page_tables);
-		__free_page(ppgtt->pdp.pagedir[i].page);
-	}
+	while (i--)
+		free_pd_single(ppgtt->pdp.pagedir[i]);
 
 	return -ENOMEM;
 }

@@ -500,14 +582,14 @@ static int gen8_ppgtt_setup_page_directories(struct i915_hw_ppgtt *ppgtt,
 	int ret;
 
 	pd_addr = pci_map_page(ppgtt->base.dev->pdev,
-			       ppgtt->pdp.pagedir[pdpe].page, 0,
+			       ppgtt->pdp.pagedir[pdpe]->page, 0,
 			       PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
 
 	ret = pci_dma_mapping_error(ppgtt->base.dev->pdev, pd_addr);
 	if (ret)
 		return ret;
 
-	ppgtt->pdp.pagedir[pdpe].daddr = pd_addr;
+	ppgtt->pdp.pagedir[pdpe]->daddr = pd_addr;
 
 	return 0;
 }

@@ -517,8 +599,8 @@ static int gen8_ppgtt_setup_page_tables(struct i915_hw_ppgtt *ppgtt,
 					const int pde)
 {
 	dma_addr_t pt_addr;
-	struct i915_pagedir *pd = &ppgtt->pdp.pagedir[pdpe];
-	struct i915_pagetab *pt = &pd->page_tables[pde];
+	struct i915_pagedir *pd = ppgtt->pdp.pagedir[pdpe];
+	struct i915_pagetab *pt = pd->page_tables[pde];
 	struct page *p = pt->page;
 	int ret;
 

@@ -581,10 +663,12 @@ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size)
 	 * will never need to touch the PDEs again.
 	 */
 	for (i = 0; i < max_pdp; i++) {
+		struct i915_pagedir *pd = ppgtt->pdp.pagedir[i];
 		gen8_ppgtt_pde_t *pd_vaddr;
-		pd_vaddr = kmap_atomic(ppgtt->pdp.pagedir[i].page);
+		pd_vaddr = kmap_atomic(ppgtt->pdp.pagedir[i]->page);
 		for (j = 0; j < I915_PDES_PER_PD; j++) {
-			dma_addr_t addr = ppgtt->pdp.pagedir[i].page_tables[j].daddr;
+			struct i915_pagetab *pt = pd->page_tables[j];
+			dma_addr_t addr = pt->daddr;
 			pd_vaddr[j] = gen8_pde_encode(ppgtt->base.dev, addr,
 						      I915_CACHE_LLC);
 		}

@@ -634,7 +718,7 @@ static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m)
 	for (pde = 0; pde < ppgtt->num_pd_entries; pde++) {
 		u32 expected;
 		gen6_gtt_pte_t *pt_vaddr;
-		dma_addr_t pt_addr = ppgtt->pd.page_tables[pde].daddr;
+		dma_addr_t pt_addr = ppgtt->pd.page_tables[pde]->daddr;
 		pd_entry = readl(pd_addr + pde);
 		expected = (GEN6_PDE_ADDR_ENCODE(pt_addr) | GEN6_PDE_VALID);
 

@@ -645,7 +729,7 @@ static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m)
 				   expected);
 		seq_printf(m, "\tPDE: %x\n", pd_entry);
 
-		pt_vaddr = kmap_atomic(ppgtt->pd.page_tables[pde].page);
+		pt_vaddr = kmap_atomic(ppgtt->pd.page_tables[pde]->page);
 		for (pte = 0; pte < GEN6_PTES_PER_PT; pte+=4) {
 			unsigned long va =
 				(pde * PAGE_SIZE * GEN6_PTES_PER_PT) +

@@ -695,7 +779,7 @@ static void gen6_map_page_tables(struct i915_hw_ppgtt *ppgtt)
 
 	WARN_ON(ppgtt->pd.pd_offset & 0x3f);
 	for (i = 0; i < ppgtt->num_pd_entries; i++)
-		gen6_map_single(ppgtt, i, ppgtt->pd.page_tables[i].daddr);
+		gen6_map_single(ppgtt, i, ppgtt->pd.page_tables[i]->daddr);
 
 	readl(dev_priv->gtt.gsm);
 }

@@ -939,7 +1023,7 @@ static void gen6_ppgtt_clear_range(struct i915_address_space *vm,
 		if (last_pte > GEN6_PTES_PER_PT)
 			last_pte = GEN6_PTES_PER_PT;
 
-		pt_vaddr = kmap_atomic(ppgtt->pd.page_tables[pde].page);
+		pt_vaddr = kmap_atomic(ppgtt->pd.page_tables[pde]->page);
 
 		for (i = pte; i < last_pte; i++)
 			pt_vaddr[i] = scratch_pte;

@@ -967,7 +1051,7 @@ static void gen6_ppgtt_insert_entries(struct i915_address_space *vm,
 	pt_vaddr = NULL;
 	for_each_sg_page(pages->sgl, &sg_iter, pages->nents, 0) {
 		if (pt_vaddr == NULL)
-			pt_vaddr = kmap_atomic(ppgtt->pd.page_tables[pde].page);
+			pt_vaddr = kmap_atomic(ppgtt->pd.page_tables[pde]->page);
 
 		pt_vaddr[pte] =
 			vm->pte_encode(sg_page_iter_dma_address(&sg_iter),

@@ -989,7 +1073,7 @@ static void gen6_ppgtt_dma_unmap_pages(struct i915_hw_ppgtt *ppgtt)
 
 	for (i = 0; i < ppgtt->num_pd_entries; i++)
 		pci_unmap_page(ppgtt->base.dev->pdev,
-			       ppgtt->pd.page_tables[i].daddr,
+			       ppgtt->pd.page_tables[i]->daddr,
 			       4096, PCI_DMA_BIDIRECTIONAL);
 }
 

@@ -998,8 +1082,9 @@ static void gen6_ppgtt_free(struct i915_hw_ppgtt *ppgtt)
 	int i;
 
 	for (i = 0; i < ppgtt->num_pd_entries; i++)
-		__free_page(ppgtt->pd.page_tables[i].page);
-	kfree(ppgtt->pd.page_tables);
+		free_pt_single(ppgtt->pd.page_tables[i]);
+
+	free_pd_single(&ppgtt->pd);
 }
 
 static void gen6_ppgtt_cleanup(struct i915_address_space *vm)

@@ -1054,27 +1139,6 @@ alloc:
 	return 0;
 }
 
-static int gen6_ppgtt_allocate_page_tables(struct i915_hw_ppgtt *ppgtt)
-{
-	struct i915_pagetab *pt;
-	int i;
-
-	pt = kcalloc(ppgtt->num_pd_entries, sizeof(*pt), GFP_KERNEL);
-	if (!pt)
-		return -ENOMEM;
-
-	for (i = 0; i < ppgtt->num_pd_entries; i++) {
-		pt[i].page = alloc_page(GFP_KERNEL | __GFP_ZERO);
-		if (!pt->page) {
-			gen6_ppgtt_free(ppgtt);
-			return -ENOMEM;
-		}
-	}
-
-	ppgtt->pd.page_tables = pt;
-	return 0;
-}
-
 static int gen6_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt)
 {
 	int ret;

@@ -1083,7 +1147,7 @@ static int gen6_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt)
 	if (ret)
 		return ret;
 
-	ret = gen6_ppgtt_allocate_page_tables(ppgtt);
+	ret = alloc_pt_range(&ppgtt->pd, 0, ppgtt->num_pd_entries);
 	if (ret) {
 		drm_mm_remove_node(&ppgtt->node);
 		return ret;

@@ -1101,7 +1165,7 @@ static int gen6_ppgtt_setup_page_tables(struct i915_hw_ppgtt *ppgtt)
 		struct page *page;
 		dma_addr_t pt_addr;
 
-		page = ppgtt->pd.page_tables[i].page;
+		page = ppgtt->pd.page_tables[i]->page;
 		pt_addr = pci_map_page(dev->pdev, page, 0, 4096,
 				       PCI_DMA_BIDIRECTIONAL);
 

@@ -1110,7 +1174,7 @@ static int gen6_ppgtt_setup_page_tables(struct i915_hw_ppgtt *ppgtt)
 			return -EIO;
 		}
 
-		ppgtt->pd.page_tables[i].daddr = pt_addr;
+		ppgtt->pd.page_tables[i]->daddr = pt_addr;
 	}
 
 	return 0;

index 07a5cd4..9b714b5 100644
@@ -264,12 +264,12 @@ struct i915_pagedir {
 		dma_addr_t daddr;
 	};
 
-	struct i915_pagetab *page_tables;
+	struct i915_pagetab *page_tables[I915_PDES_PER_PD]; /* PDEs */
 };
 
 struct i915_pagedirpo {
 	/* struct page *page; */
-	struct i915_pagedir pagedir[GEN8_LEGACY_PDPES];
+	struct i915_pagedir *pagedir[GEN8_LEGACY_PDPES];
 };
 
 struct i915_hw_ppgtt {

index 35370eb..e396b89 100644
@@ -700,18 +700,13 @@ bail:
 
 static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m)
 {
-	struct drm_i915_private *dev_priv = ppgtt->base.dev->dev_private;
 	struct i915_address_space *vm = &ppgtt->base;
-	gen6_gtt_pte_t __iomem *pd_addr;
 	gen6_gtt_pte_t scratch_pte;
 	uint32_t pd_entry;
 	int pte, pde;
 
 	scratch_pte = vm->pte_encode(vm->scratch.addr, I915_CACHE_LLC, true);
 
-	pd_addr = (gen6_gtt_pte_t __iomem *)dev_priv->gtt.gsm +
-		ppgtt->pd.pd_offset / sizeof(gen6_gtt_pte_t);
-
 	seq_printf(m, "  VM %p (pd_offset %x-%x):\n", vm,
 		   ppgtt->pd.pd_offset,
 		   ppgtt->pd.pd_offset + ppgtt->num_pd_entries);

@@ -719,7 +714,7 @@ static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m)
 		u32 expected;
 		gen6_gtt_pte_t *pt_vaddr;
 		dma_addr_t pt_addr = ppgtt->pd.page_tables[pde]->daddr;
-		pd_entry = readl(pd_addr + pde);
+		pd_entry = readl(ppgtt->pd_addr + pde);
 		expected = (GEN6_PDE_ADDR_ENCODE(pt_addr) | GEN6_PDE_VALID);
 
 		if (pd_entry != expected)

@@ -755,39 +750,43 @@ static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m)
 	}
 }
 
-static void gen6_map_single(struct i915_hw_ppgtt *ppgtt,
-			    const unsigned pde_index,
-			    dma_addr_t daddr)
+/* Map pde (index) from the page directory @pd to the page table @pt */
+static void gen6_map_single(struct i915_pagedir *pd,
+			    const int pde, struct i915_pagetab *pt)
 {
-	struct drm_i915_private *dev_priv = ppgtt->base.dev->dev_private;
-	uint32_t pd_entry;
-	gen6_gtt_pte_t __iomem *pd_addr = (gen6_gtt_pte_t __iomem*)dev_priv->gtt.gsm;
-	pd_addr	+= ppgtt->pd.pd_offset / sizeof(gen6_gtt_pte_t);
+	struct i915_hw_ppgtt *ppgtt =
+		container_of(pd, struct i915_hw_ppgtt, pd);
+	u32 pd_entry;
 
-	pd_entry = GEN6_PDE_ADDR_ENCODE(daddr);
+	pd_entry = GEN6_PDE_ADDR_ENCODE(pt->daddr);
 	pd_entry |= GEN6_PDE_VALID;
 
-	writel(pd_entry, pd_addr + pde_index);
+	writel(pd_entry, ppgtt->pd_addr + pde);
+
+	/* XXX: Caller needs to make sure the write completes if necessary */
 }
 
 /* Map all the page tables found in the ppgtt structure to incrementing page
  * directories. */
-static void gen6_map_page_tables(struct i915_hw_ppgtt *ppgtt)
+static void gen6_map_page_range(struct drm_i915_private *dev_priv,
+				struct i915_pagedir *pd, unsigned pde, size_t n)
 {
-	struct drm_i915_private *dev_priv = ppgtt->base.dev->dev_private;
-	int i;
+	if (WARN_ON(pde + n > I915_PDES_PER_PD))
+		n = I915_PDES_PER_PD - pde;
 
-	WARN_ON(ppgtt->pd.pd_offset & 0x3f);
-	for (i = 0; i < ppgtt->num_pd_entries; i++)
-		gen6_map_single(ppgtt, i, ppgtt->pd.page_tables[i]->daddr);
+	n += pde;
+
+	for (; pde < n; pde++)
+		gen6_map_single(pd, pde, pd->page_tables[pde]);
 
+	/* Make sure write is complete before other code can use this page
+	 * table. Also require for WC mapped PTEs */
 	readl(dev_priv->gtt.gsm);
 }
 
 static uint32_t get_pd_offset(struct i915_hw_ppgtt *ppgtt)
 {
 	BUG_ON(ppgtt->pd.pd_offset & 0x3f);
-
 	return (ppgtt->pd.pd_offset / 64) << 16;
 }
 

@@ -1219,7 +1218,10 @@ static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
 	ppgtt->pd.pd_offset =
 		ppgtt->node.start / PAGE_SIZE * sizeof(gen6_gtt_pte_t);
 
-	gen6_map_page_tables(ppgtt);
+	ppgtt->pd_addr = (gen6_gtt_pte_t __iomem*)dev_priv->gtt.gsm +
+		ppgtt->pd.pd_offset / sizeof(gen6_gtt_pte_t);
+
+	gen6_map_page_range(dev_priv, &ppgtt->pd, 0, ppgtt->num_pd_entries);
 
 	DRM_DEBUG_DRIVER("Allocated pde space (%ldM) at GTT entry: %lx\n",
 			 ppgtt->node.size >> 20,

@@ -1405,13 +1407,14 @@ void i915_gem_restore_gtt_mappings(struct drm_device *dev)
 
 	list_for_each_entry(vm, &dev_priv->vm_list, global_link) {
 		/* TODO: Perhaps it shouldn't be gen6 specific */
-		if (i915_is_ggtt(vm)) {
-			if (dev_priv->mm.aliasing_ppgtt)
-				gen6_map_page_tables(dev_priv->mm.aliasing_ppgtt);
-			continue;
-		}
 
-		gen6_map_page_tables(container_of(vm, struct i915_hw_ppgtt, base));
+		struct i915_hw_ppgtt *ppgtt =
+			container_of(vm, struct i915_hw_ppgtt, base);
+
+		if (i915_is_ggtt(vm))
+			ppgtt = dev_priv->mm.aliasing_ppgtt;
+
+		gen6_map_page_range(dev_priv, &ppgtt->pd, 0, ppgtt->num_pd_entries);
 	}
 
 	i915_gem_chipset_flush(dev);

index 9b714b5..fea846d 100644
@@ -285,6 +285,8 @@ struct i915_hw_ppgtt {
 
 	struct i915_hw_context *ctx;
 
+	gen6_gtt_pte_t __iomem *pd_addr;
+
 	int (*enable)(struct i915_hw_ppgtt *ppgtt);
 	int (*switch_mm)(struct i915_hw_ppgtt *ppgtt,
 			 struct intel_ring_buffer *ring,

index e396b89..e8d4dfa 100644
@@ -211,6 +211,76 @@ static gen6_gtt_pte_t iris_pte_encode(dma_addr_t addr,
 	return pte;
 }
 
+#define dma_unmap_pt_single(pt, dev) do { \
+	pci_unmap_page((dev)->pdev, (pt)->daddr, 4096, PCI_DMA_BIDIRECTIONAL); \
+} while (0);
+
+
+static void dma_unmap_pt_range(struct i915_pagedir *pd,
+			       unsigned pde, size_t n,
+			       struct drm_device *dev)
+{
+	if (WARN_ON(pde + n > I915_PDES_PER_PD))
+		n = I915_PDES_PER_PD - pde;
+
+	n += pde;
+
+	for (; pde < n; pde++)
+		dma_unmap_pt_single(pd->page_tables[pde], dev);
+}
+
+/**
+ * dma_map_pt_single() - Create a dma mapping for a page table
+ * @pt:		Page table to get a DMA map for
+ * @dev:	drm device
+ *
+ * Page table allocations are unified across all gens. They always require a
+ * single 4k allocation, as well as a DMA mapping.
+ *
+ * Return: 0 if success.
+ */
+static int dma_map_pt_single(struct i915_pagetab *pt, struct drm_device *dev)
+{
+	struct page *page;
+	dma_addr_t pt_addr;
+	int ret;
+
+	page = pt->page;
+	pt_addr = pci_map_page(dev->pdev, page, 0, 4096,
+			       PCI_DMA_BIDIRECTIONAL);
+
+	ret = pci_dma_mapping_error(dev->pdev, pt_addr);
+	if (ret)
+		return ret;
+
+	pt->daddr = pt_addr;
+
+	return 0;
+}
+
+static int dma_map_pt_range(struct i915_pagedir *pd,
+			    unsigned pde, size_t n,
+			    struct drm_device *dev)
+{
+	const int first = pde;
+
+	if (WARN_ON(pde + n > I915_PDES_PER_PD))
+		n = I915_PDES_PER_PD - pde;
+
+	n += pde;
+
+	for (; pde < n; pde++) {
+		int ret;
+		ret = dma_map_pt_single(pd->page_tables[pde], dev);
+		if (ret) {
+			dma_unmap_pt_range(pd, first, pde, dev);
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
 static void free_pt_single(struct i915_pagetab *pt)
 {
 	if (WARN_ON(!pt->page))

@@ -219,7 +289,7 @@ static void free_pt_single(struct i915_pagetab *pt)
 	kfree(pt);
 }
 
-static struct i915_pagetab *alloc_pt_single(void)
+static struct i915_pagetab *alloc_pt_single(struct drm_device *dev)
 {
 	struct i915_pagetab *pt;
 

@@ -242,6 +312,7 @@ static struct i915_pagetab *alloc_pt_single(void)
  *		available to point to the allocated page tables.
  * @pde:	First page directory entry for which we are allocating.
  * @count:	Number of pages to allocate.
+ * @dev		DRM device used for DMA mapping.
  *
  * Allocates multiple page table pages and sets the appropriate entries in the
  * page table structure within the page directory. Function cleans up after

@@ -249,7 +320,8 @@ static struct i915_pagetab *alloc_pt_single(void)
  *
  * Return: 0 if allocation succeeded.
  */
-static int alloc_pt_range(struct i915_pagedir *pd, uint16_t pde, size_t count)
+static int alloc_pt_range(struct i915_pagedir *pd, uint16_t pde, size_t count,
+			  struct drm_device *dev)
 {
 	int i, ret;
 

@@ -259,7 +331,7 @@ static int alloc_pt_range(struct i915_pagedir *pd, uint16_t pde, size_t count)
 	BUG_ON(pde + count > I915_PDES_PER_PD);
 
 	for (i = pde; i < pde + count; i++) {
-		struct i915_pagetab *pt = alloc_pt_single();
+		struct i915_pagetab *pt = alloc_pt_single(dev);
 		if (IS_ERR(pt)) {
 			ret = PTR_ERR(pt);
 			goto err_out;

@@ -515,7 +587,7 @@ static int gen8_ppgtt_allocate_page_tables(struct i915_hw_ppgtt *ppgtt)
 
 	for (i = 0; i < ppgtt->num_pd_pages; i++) {
 		ret = alloc_pt_range(ppgtt->pdp.pagedir[i],
-				     0, I915_PDES_PER_PD);
+				     0, I915_PDES_PER_PD, ppgtt->base.dev);
 		if (ret)
 			goto unwind_out;
 	}

@@ -594,27 +666,6 @@ static int gen8_ppgtt_setup_page_directories(struct i915_hw_ppgtt *ppgtt,
 	return 0;
 }
 
-static int gen8_ppgtt_setup_page_tables(struct i915_hw_ppgtt *ppgtt,
-					const int pdpe,
-					const int pde)
-{
-	dma_addr_t pt_addr;
-	struct i915_pagedir *pd = ppgtt->pdp.pagedir[pdpe];
-	struct i915_pagetab *pt = pd->page_tables[pde];
-	struct page *p = pt->page;
-	int ret;
-
-	pt_addr = pci_map_page(ppgtt->base.dev->pdev,
-			       p, 0, PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
-	ret = pci_dma_mapping_error(ppgtt->base.dev->pdev, pt_addr);
-	if (ret)
-		return ret;
-
-	pt->daddr = pt_addr;
-
-	return 0;
-}
-
 /**
  * GEN8 legacy ppgtt programming is accomplished through a max 4 PDP registers
  * with a net effect resembling a 2-level page table in normal x86 terms. Each

@@ -643,12 +694,15 @@ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size)
 	 * 2. Create DMA mappings for the page directories and page tables.
 	 */
 	for (i = 0; i < max_pdp; i++) {
+		struct i915_pagedir *pd;
 		ret = gen8_ppgtt_setup_page_directories(ppgtt, i);
 		if (ret)
 			goto bail;
 
+		pd = ppgtt->pdp.pagedir[i];
+
 		for (j = 0; j < I915_PDES_PER_PD; j++) {
-			ret = gen8_ppgtt_setup_page_tables(ppgtt, i, j);
+			ret = dma_map_pt_single(pd->page_tables[j], ppgtt->base.dev);
 			if (ret)
 				goto bail;
 		}

@@ -1066,16 +1120,6 @@ static void gen6_ppgtt_insert_entries(struct i915_address_space *vm,
 		kunmap_atomic(pt_vaddr);
 }
 
-static void gen6_ppgtt_dma_unmap_pages(struct i915_hw_ppgtt *ppgtt)
-{
-	int i;
-
-	for (i = 0; i < ppgtt->num_pd_entries; i++)
-		pci_unmap_page(ppgtt->base.dev->pdev,
-			       ppgtt->pd.page_tables[i]->daddr,
-			       4096, PCI_DMA_BIDIRECTIONAL);
-}
-
 static void gen6_ppgtt_free(struct i915_hw_ppgtt *ppgtt)
 {
 	int i;

@@ -1095,7 +1139,7 @@ static void gen6_ppgtt_cleanup(struct i915_address_space *vm)
 	drm_mm_takedown(&ppgtt->base.mm);
 	drm_mm_remove_node(&ppgtt->node);
 
-	gen6_ppgtt_dma_unmap_pages(ppgtt);
+	dma_unmap_pt_range(&ppgtt->pd, 0, ppgtt->num_pd_entries, vm->dev);
 	gen6_ppgtt_free(ppgtt);
 }
 

@@ -1146,7 +1190,8 @@ static int gen6_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt)
 	if (ret)
 		return ret;
 
-	ret = alloc_pt_range(&ppgtt->pd, 0, ppgtt->num_pd_entries);
+	ret = alloc_pt_range(&ppgtt->pd, 0, ppgtt->num_pd_entries,
+			     ppgtt->base.dev);
 	if (ret) {
 		drm_mm_remove_node(&ppgtt->node);
 		return ret;

@@ -1155,29 +1200,6 @@ static int gen6_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt)
 	return 0;
 }
 
-static int gen6_ppgtt_setup_page_tables(struct i915_hw_ppgtt *ppgtt)
-{
-	struct drm_device *dev = ppgtt->base.dev;
-	int i;
-
-	for (i = 0; i < ppgtt->num_pd_entries; i++) {
-		struct page *page;
-		dma_addr_t pt_addr;
-
-		page = ppgtt->pd.page_tables[i]->page;
-		pt_addr = pci_map_page(dev->pdev, page, 0, 4096,
-				       PCI_DMA_BIDIRECTIONAL);
-
-		if (pci_dma_mapping_error(dev->pdev, pt_addr)) {
-			gen6_ppgtt_dma_unmap_pages(ppgtt);
-			return -EIO;
-		}
-
-		ppgtt->pd.page_tables[i]->daddr = pt_addr;
-	}
-
-	return 0;
-}
 
 static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
 {

@@ -1202,7 +1224,8 @@ static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
 	if (ret)
 		return ret;
 
-	ret = gen6_ppgtt_setup_page_tables(ppgtt);
+	ret = dma_map_pt_range(&ppgtt->pd, 0, ppgtt->num_pd_entries,
+			       ppgtt->base.dev);
 	if (ret) {
 		gen6_ppgtt_free(ppgtt);
 		return ret;

index e8d4dfa..92ffee7 100644
@@ -215,20 +215,6 @@ static gen6_gtt_pte_t iris_pte_encode(dma_addr_t addr,
 	pci_unmap_page((dev)->pdev, (pt)->daddr, 4096, PCI_DMA_BIDIRECTIONAL); \
 } while (0);
 
-
-static void dma_unmap_pt_range(struct i915_pagedir *pd,
-			       unsigned pde, size_t n,
-			       struct drm_device *dev)
-{
-	if (WARN_ON(pde + n > I915_PDES_PER_PD))
-		n = I915_PDES_PER_PD - pde;
-
-	n += pde;
-
-	for (; pde < n; pde++)
-		dma_unmap_pt_single(pd->page_tables[pde], dev);
-}
-
 /**
  * dma_map_pt_single() - Create a dma mapping for a page table
  * @pt:		Page table to get a DMA map for

@@ -258,33 +244,12 @@ static int dma_map_pt_single(struct i915_pagetab *pt, struct drm_device *dev)
 	return 0;
 }
 
-static int dma_map_pt_range(struct i915_pagedir *pd,
-			    unsigned pde, size_t n,
-			    struct drm_device *dev)
-{
-	const int first = pde;
-
-	if (WARN_ON(pde + n > I915_PDES_PER_PD))
-		n = I915_PDES_PER_PD - pde;
-
-	n += pde;
-
-	for (; pde < n; pde++) {
-		int ret;
-		ret = dma_map_pt_single(pd->page_tables[pde], dev);
-		if (ret) {
-			dma_unmap_pt_range(pd, first, pde, dev);
-			return ret;
-		}
-	}
-
-	return 0;
-}
-
-static void free_pt_single(struct i915_pagetab *pt)
+static void free_pt_single(struct i915_pagetab *pt, struct drm_device *dev)
 {
 	if (WARN_ON(!pt->page))
 		return;
+
+	dma_unmap_pt_single(pt, dev);
 	__free_page(pt->page);
 	kfree(pt);
 }

@@ -292,6 +257,7 @@ static void free_pt_single(struct i915_pagetab *pt)
 static struct i915_pagetab *alloc_pt_single(struct drm_device *dev)
 {
 	struct i915_pagetab *pt;
+	int ret;
 
 	pt = kzalloc(sizeof(*pt), GFP_KERNEL);
 	if (!pt)

@@ -303,6 +269,13 @@ static struct i915_pagetab *alloc_pt_single(struct drm_device *dev)
 		return ERR_PTR(-ENOMEM);
 	}
 
+	ret = dma_map_pt_single(pt, dev);
+	if (ret) {
+		__free_page(pt->page);
+		kfree(pt);
+		return ERR_PTR(ret);
+	}
+
 	return pt;
 }
 

@@ -346,7 +319,7 @@ static int alloc_pt_range(struct i915_pagedir *pd, uint16_t pde, size_t count,
 
 err_out:
 	while (i--)
-		free_pt_single(pd->page_tables[i]);
+		free_pt_single(pd->page_tables[i], dev);
 	return ret;
 }
 

@@ -521,7 +494,7 @@ static void gen8_ppgtt_insert_entries(struct i915_address_space *vm,
 	}
 }
 
-static void gen8_free_page_tables(struct i915_pagedir *pd)
+static void gen8_free_page_tables(struct i915_pagedir *pd, struct drm_device *dev)
 {
 	int i;
 

@@ -529,7 +502,7 @@ static void gen8_free_page_tables(struct i915_pagedir *pd)
 		return;
 
 	for (i = 0; i < I915_PDES_PER_PD; i++) {
-		free_pt_single(pd->page_tables[i]);
+		free_pt_single(pd->page_tables[i], dev);
 		pd->page_tables[i] = NULL;
 	}
 }

@@ -539,7 +512,7 @@ static void gen8_ppgtt_free(struct i915_hw_ppgtt *ppgtt)
 	int i;
 
 	for (i = 0; i < ppgtt->num_pd_pages; i++) {
-		gen8_free_page_tables(ppgtt->pdp.pagedir[i]);
+		gen8_free_page_tables(ppgtt->pdp.pagedir[i], ppgtt->base.dev);
 		free_pd_single(ppgtt->pdp.pagedir[i]);
 	}
 }

@@ -596,7 +569,7 @@ static int gen8_ppgtt_allocate_page_tables(struct i915_hw_ppgtt *ppgtt)
 
 unwind_out:
 	while (i--)
-		gen8_free_page_tables(ppgtt->pdp.pagedir[i]);
+		gen8_free_page_tables(ppgtt->pdp.pagedir[i], ppgtt->base.dev);
 
 	return -ENOMEM;
 }

@@ -694,18 +667,9 @@ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size)
 	 * 2. Create DMA mappings for the page directories and page tables.
 	 */
 	for (i = 0; i < max_pdp; i++) {
-		struct i915_pagedir *pd;
 		ret = gen8_ppgtt_setup_page_directories(ppgtt, i);
 		if (ret)
 			goto bail;
-
-		pd = ppgtt->pdp.pagedir[i];
-
-		for (j = 0; j < I915_PDES_PER_PD; j++) {
-			ret = dma_map_pt_single(pd->page_tables[j], ppgtt->base.dev);
-			if (ret)
-				goto bail;
-		}
 	}
 
 	/*

@@ -1125,7 +1089,7 @@ static void gen6_ppgtt_free(struct i915_hw_ppgtt *ppgtt)
 	int i;
 
 	for (i = 0; i < ppgtt->num_pd_entries; i++)
-		free_pt_single(ppgtt->pd.page_tables[i]);
+		free_pt_single(ppgtt->pd.page_tables[i], ppgtt->base.dev);
 
 	free_pd_single(&ppgtt->pd);
 }

@@ -1139,7 +1103,6 @@ static void gen6_ppgtt_cleanup(struct i915_address_space *vm)
 	drm_mm_takedown(&ppgtt->base.mm);
 	drm_mm_remove_node(&ppgtt->node);
 
-	dma_unmap_pt_range(&ppgtt->pd, 0, ppgtt->num_pd_entries, vm->dev);
 	gen6_ppgtt_free(ppgtt);
 }
 

@@ -1224,13 +1187,6 @@ static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
 	if (ret)
 		return ret;
 
-	ret = dma_map_pt_range(&ppgtt->pd, 0, ppgtt->num_pd_entries,
-			       ppgtt->base.dev);
-	if (ret) {
-		gen6_ppgtt_free(ppgtt);
-		return ret;
-	}
-
 	ppgtt->base.clear_range = gen6_ppgtt_clear_range;
 	ppgtt->base.insert_entries = gen6_ppgtt_insert_entries;
 	ppgtt->base.cleanup = gen6_ppgtt_cleanup;

index 92ffee7..bb909e9 100644
@@ -211,45 +211,33 @@ static gen6_gtt_pte_t iris_pte_encode(dma_addr_t addr,
 	return pte;
 }
 
-#define dma_unmap_pt_single(pt, dev) do { \
-	pci_unmap_page((dev)->pdev, (pt)->daddr, 4096, PCI_DMA_BIDIRECTIONAL); \
+#define i915_dma_unmap_single(px, dev) do { \
+	pci_unmap_page((dev)->pdev, (px)->daddr, 4096, PCI_DMA_BIDIRECTIONAL); \
 } while (0);
 
 /**
- * dma_map_pt_single() - Create a dma mapping for a page table
- * @pt:		Page table to get a DMA map for
+ * i915_dma_map_px_single() - Create a dma mapping for a page table/dir/etc.
+ * @px:		Page table/dir/etc to get a DMA map for
  * @dev:	drm device
  *
  * Page table allocations are unified across all gens. They always require a
- * single 4k allocation, as well as a DMA mapping.
+ * single 4k allocation, as well as a DMA mapping. If we keep the structs
+ * symmetric here, the simple macro covers us for every page table type.
  *
  * Return: 0 if success.
  */
-static int dma_map_pt_single(struct i915_pagetab *pt, struct drm_device *dev)
-{
-	struct page *page;
-	dma_addr_t pt_addr;
-	int ret;
-
-	page = pt->page;
-	pt_addr = pci_map_page(dev->pdev, page, 0, 4096,
-			       PCI_DMA_BIDIRECTIONAL);
-
-	ret = pci_dma_mapping_error(dev->pdev, pt_addr);
-	if (ret)
-		return ret;
-
-	pt->daddr = pt_addr;
-
-	return 0;
-}
+#define i915_dma_map_px_single(px, dev) \
+	pci_dma_mapping_error((dev)->pdev, \
+			      (px)->daddr = pci_map_page((dev)->pdev, \
+							 (px)->page, 0, 4096, \
+							 PCI_DMA_BIDIRECTIONAL))
 
 static void free_pt_single(struct i915_pagetab *pt, struct drm_device *dev)
 {
 	if (WARN_ON(!pt->page))
 		return;
 
-	dma_unmap_pt_single(pt, dev);
+	i915_dma_unmap_single(pt, dev);
 	__free_page(pt->page);
 	kfree(pt);
 }

@@ -269,7 +257,7 @@ static struct i915_pagetab *alloc_pt_single(struct drm_device *dev)
 		return ERR_PTR(-ENOMEM);
 	}
 
-	ret = dma_map_pt_single(pt, dev);
+	ret = i915_dma_map_px_single(pt, dev);
 	if (ret) {
 		__free_page(pt->page);
 		kfree(pt);

@@ -519,7 +507,7 @@ static void gen8_ppgtt_free(struct i915_hw_ppgtt *ppgtt)
 
 static void gen8_ppgtt_dma_unmap_pages(struct i915_hw_ppgtt *ppgtt)
 {
-	struct pci_dev *hwdev = ppgtt->base.dev->pdev;
+	struct drm_device *dev = ppgtt->base.dev;
 	int i, j;
 
 	for (i = 0; i < ppgtt->num_pd_pages; i++) {

@@ -528,16 +516,14 @@ static void gen8_ppgtt_dma_unmap_pages(struct i915_hw_ppgtt *ppgtt)
 		if (!ppgtt->pdp.pagedir[i]->daddr)
 			continue;
 
-		pci_unmap_page(hwdev, ppgtt->pdp.pagedir[i]->daddr, PAGE_SIZE,
-			       PCI_DMA_BIDIRECTIONAL);
+		i915_dma_unmap_single(ppgtt->pdp.pagedir[i], dev);
 
 		for (j = 0; j < I915_PDES_PER_PD; j++) {
 			struct i915_pagedir *pd = ppgtt->pdp.pagedir[i];
 			struct i915_pagetab *pt =  pd->page_tables[j];
 			dma_addr_t addr = pt->daddr;
 			if (addr)
-				pci_unmap_page(hwdev, addr, PAGE_SIZE,
-					       PCI_DMA_BIDIRECTIONAL);
+				i915_dma_unmap_single(pt, dev);
 		}
 	}
 }

@@ -623,19 +609,13 @@ err_out:
 static int gen8_ppgtt_setup_page_directories(struct i915_hw_ppgtt *ppgtt,
 					     const int pdpe)
 {
-	dma_addr_t pd_addr;
 	int ret;
 
-	pd_addr = pci_map_page(ppgtt->base.dev->pdev,
-			       ppgtt->pdp.pagedir[pdpe]->page, 0,
-			       PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
-
-	ret = pci_dma_mapping_error(ppgtt->base.dev->pdev, pd_addr);
+	ret = i915_dma_map_px_single(ppgtt->pdp.pagedir[pdpe],
+				     ppgtt->base.dev);
 	if (ret)
 		return ret;
 
-	ppgtt->pdp.pagedir[pdpe]->daddr = pd_addr;
-
 	return 0;
 }
 

index bb909e9..51fc036 100644
@@ -311,21 +311,23 @@ err_out:
 	return ret;
 }
 
-static void __free_pd_single(struct i915_pagedir *pd)
+static void __free_pd_single(struct i915_pagedir *pd, struct drm_device *dev)
 {
+	i915_dma_unmap_single(pd, dev);
 	__free_page(pd->page);
 	kfree(pd);
 }
 
-#define free_pd_single(pd) do { \
+#define free_pd_single(pd, dev) do { \
 	if ((pd)->page) { \
-		__free_pd_single(pd); \
+		__free_pd_single(pd, dev); \
 	} \
 } while (0)
 
-static struct i915_pagedir *alloc_pd_single(void)
+static struct i915_pagedir *alloc_pd_single(struct drm_device *dev)
 {
 	struct i915_pagedir *pd;
+	int ret;
 
 	pd = kzalloc(sizeof(*pd), GFP_KERNEL);
 	if (!pd)

@@ -337,6 +339,13 @@ static struct i915_pagedir *alloc_pd_single(void)
 		return ERR_PTR(-ENOMEM);
 	}
 
+	ret = i915_dma_map_px_single(pd, dev);
+	if (ret) {
+		__free_page(pd->page);
+		kfree(pd);
+		return ERR_PTR(ret);
+	}
+
 	return pd;
 }
 

@@ -501,30 +510,7 @@ static void gen8_ppgtt_free(struct i915_hw_ppgtt *ppgtt)
 
 	for (i = 0; i < ppgtt->num_pd_pages; i++) {
 		gen8_free_page_tables(ppgtt->pdp.pagedir[i], ppgtt->base.dev);
-		free_pd_single(ppgtt->pdp.pagedir[i]);
-	}
-}
-
-static void gen8_ppgtt_dma_unmap_pages(struct i915_hw_ppgtt *ppgtt)
-{
-	struct drm_device *dev = ppgtt->base.dev;
-	int i, j;
-
-	for (i = 0; i < ppgtt->num_pd_pages; i++) {
-		/* TODO: In the future we'll support sparse mappings, so this
-		 * will have to change. */
-		if (!ppgtt->pdp.pagedir[i]->daddr)
-			continue;
-
-		i915_dma_unmap_single(ppgtt->pdp.pagedir[i], dev);
-
-		for (j = 0; j < I915_PDES_PER_PD; j++) {
-			struct i915_pagedir *pd = ppgtt->pdp.pagedir[i];
-			struct i915_pagetab *pt =  pd->page_tables[j];
-			dma_addr_t addr = pt->daddr;
-			if (addr)
-				i915_dma_unmap_single(pt, dev);
-		}
+		free_pd_single(ppgtt->pdp.pagedir[i], ppgtt->base.dev);
 	}
 }
 

@@ -536,7 +522,6 @@ static void gen8_ppgtt_cleanup(struct i915_address_space *vm)
 	list_del(&vm->global_link);
 	drm_mm_takedown(&vm->mm);
 
-	gen8_ppgtt_dma_unmap_pages(ppgtt);
 	gen8_ppgtt_free(ppgtt);
 }
 

@@ -566,7 +551,7 @@ static int gen8_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt,
 	int i;
 
 	for (i = 0; i < max_pdp; i++) {
-		ppgtt->pdp.pagedir[i] = alloc_pd_single();
+		ppgtt->pdp.pagedir[i] = alloc_pd_single(ppgtt->base.dev);
 		if (IS_ERR(ppgtt->pdp.pagedir[i]))
 			goto unwind_out;
 	}

@@ -578,7 +563,8 @@ static int gen8_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt,
 
 unwind_out:
 	while (i--)
-		free_pd_single(ppgtt->pdp.pagedir[i]);
+		free_pd_single(ppgtt->pdp.pagedir[i],
+			       ppgtt->base.dev);
 
 	return -ENOMEM;
 }

@@ -606,19 +592,6 @@ err_out:
 	return ret;
 }
 
-static int gen8_ppgtt_setup_page_directories(struct i915_hw_ppgtt *ppgtt,
-					     const int pdpe)
-{
-	int ret;
-
-	ret = i915_dma_map_px_single(ppgtt->pdp.pagedir[pdpe],
-				     ppgtt->base.dev);
-	if (ret)
-		return ret;
-
-	return 0;
-}
-
 /**
  * GEN8 legacy ppgtt programming is accomplished through a max 4 PDP registers
  * with a net effect resembling a 2-level page table in normal x86 terms. Each

@@ -644,16 +617,7 @@ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size)
 		return ret;
 
 	/*
-	 * 2. Create DMA mappings for the page directories and page tables.
-	 */
-	for (i = 0; i < max_pdp; i++) {
-		ret = gen8_ppgtt_setup_page_directories(ppgtt, i);
-		if (ret)
-			goto bail;
-	}
-
-	/*
-	 * 3. Map all the page directory entires to point to the page tables
+	 * 2. Map all the page directory entires to point to the page tables
 	 * we've allocated.
 	 *
 	 * For now, the PPGTT helper functions all require that the PDEs are

@@ -689,11 +653,6 @@ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size)
 			 ppgtt->num_pd_entries,
 			 (ppgtt->num_pd_entries - min_pt_pages) + size % (1<<30));
 	return 0;
-
-bail:
-	gen8_ppgtt_dma_unmap_pages(ppgtt);
-	gen8_ppgtt_free(ppgtt);
-	return ret;
 }
 
 static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m)

@@ -1071,7 +1030,7 @@ static void gen6_ppgtt_free(struct i915_hw_ppgtt *ppgtt)
 	for (i = 0; i < ppgtt->num_pd_entries; i++)
 		free_pt_single(ppgtt->pd.page_tables[i], ppgtt->base.dev);
 
-	free_pd_single(&ppgtt->pd);
+	free_pd_single(&ppgtt->pd, ppgtt->base.dev);
 }
 
 static void gen6_ppgtt_cleanup(struct i915_address_space *vm)

index 51fc036..b7a0232 100644
@@ -66,10 +66,9 @@ static int sanitize_enable_ppgtt(struct drm_device *dev, int enable_ppgtt)
 	return HAS_ALIASING_PPGTT(dev) ? 1 : 0;
 }
 
-
-static void ppgtt_bind_vma(struct i915_vma *vma,
-			   enum i915_cache_level cache_level,
-			   u32 flags);
+static int ppgtt_bind_vma(struct i915_vma *vma,
+			  enum i915_cache_level cache_level,
+			  u32 flags);
 static void ppgtt_unbind_vma(struct i915_vma *vma);
 static int gen8_ppgtt_enable(struct i915_hw_ppgtt *ppgtt);
 

@@ -232,37 +231,78 @@ static gen6_gtt_pte_t iris_pte_encode(dma_addr_t addr,
 							 (px)->page, 0, 4096, \
 							 PCI_DMA_BIDIRECTIONAL))
 
-static void free_pt_single(struct i915_pagetab *pt, struct drm_device *dev)
+static void __free_pt_single(struct i915_pagetab *pt, struct drm_device *dev,
+			     int scratch)
 {
+	if (WARN(scratch ^ pt->scratch,
+		 "Tried to free scratch = %d. Is scratch = %d\n",
+		 scratch, pt->scratch))
+		return;
+
 	if (WARN_ON(!pt->page))
 		return;
 
+	if (!scratch) {
+		const size_t count = INTEL_INFO(dev)->gen >= 8 ?
+			GEN8_PTES_PER_PT : GEN6_PTES_PER_PT;
+		WARN(!bitmap_empty(pt->used_ptes, count),
+		     "Free page table with %d used pages\n",
+		     bitmap_weight(pt->used_ptes, count));
+	}
+
 	i915_dma_unmap_single(pt, dev);
 	__free_page(pt->page);
+	kfree(pt->used_ptes);
 	kfree(pt);
 }
 
+#define free_pt_single(pt, dev) \
+	__free_pt_single(pt, dev, false)
+#define free_pt_scratch(pt, dev) \
+	__free_pt_single(pt, dev, true)
+
 static struct i915_pagetab *alloc_pt_single(struct drm_device *dev)
 {
 	struct i915_pagetab *pt;
-	int ret;
+	const size_t count = INTEL_INFO(dev)->gen >= 8 ?
+		GEN8_PTES_PER_PT : GEN6_PTES_PER_PT;
+	int ret = -ENOMEM;
 
 	pt = kzalloc(sizeof(*pt), GFP_KERNEL);
 	if (!pt)
 		return ERR_PTR(-ENOMEM);
 
+	pt->used_ptes = kcalloc(BITS_TO_LONGS(count), sizeof(*pt->used_ptes),
+				GFP_KERNEL);
+
+	if (!pt->used_ptes)
+		goto fail_bitmap;
+
 	pt->page = alloc_page(GFP_KERNEL | __GFP_ZERO);
-	if (!pt->page) {
-		kfree(pt);
-		return ERR_PTR(-ENOMEM);
-	}
+	if (!pt->page)
+		goto fail_page;
 
 	ret = i915_dma_map_px_single(pt, dev);
-	if (ret) {
-		__free_page(pt->page);
-		kfree(pt);
-		return ERR_PTR(ret);
-	}
+	if (ret)
+		goto fail_dma;
+
+	return pt;
+
+fail_dma:
+	__free_page(pt->page);
+fail_page:
+	kfree(pt->used_ptes);
+fail_bitmap:
+	kfree(pt);
+
+	return ERR_PTR(ret);
+}
+
+static inline struct i915_pagetab *alloc_pt_scratch(struct drm_device *dev)
+{
+	struct i915_pagetab *pt = alloc_pt_single(dev);
+	if (!IS_ERR(pt))
+		pt->scratch = 1;
 
 	return pt;
 }

@@ -389,7 +429,7 @@ static int gen8_mm_switch(struct i915_hw_ppgtt *ppgtt,
 	int used_pd = ppgtt->num_pd_entries / I915_PDES_PER_PD;
 
 	for (i = used_pd - 1; i >= 0; i--) {
-		dma_addr_t addr = ppgtt->pdp.pagedir[i]->daddr;
+		dma_addr_t addr = ppgtt->pdp.pagedirs[i]->daddr;
 		ret = gen8_write_pdp(ring, i, addr, synchronous);
 		if (ret)
 			return ret;

@@ -416,7 +456,7 @@ static void gen8_ppgtt_clear_range(struct i915_address_space *vm,
 				      I915_CACHE_LLC, use_scratch);
 
 	while (num_entries) {
-		struct i915_pagedir *pd = ppgtt->pdp.pagedir[pdpe];
+		struct i915_pagedir *pd = ppgtt->pdp.pagedirs[pdpe];
 		struct i915_pagetab *pt = pd->page_tables[pde];
 		struct page *page_table = pt->page;
 

@@ -463,7 +503,7 @@ static void gen8_ppgtt_insert_entries(struct i915_address_space *vm,
 			break;
 
 		if (pt_vaddr == NULL) {
-			struct i915_pagedir *pd = ppgtt->pdp.pagedir[pdpe];
+			struct i915_pagedir *pd = ppgtt->pdp.pagedirs[pdpe];
 			struct i915_pagetab *pt = pd->page_tables[pde];
 			struct page *page_table = pt->page;
 			pt_vaddr = kmap_atomic(page_table);

@@ -509,8 +549,8 @@ static void gen8_ppgtt_free(struct i915_hw_ppgtt *ppgtt)
 	int i;
 
 	for (i = 0; i < ppgtt->num_pd_pages; i++) {
-		gen8_free_page_tables(ppgtt->pdp.pagedir[i], ppgtt->base.dev);
-		free_pd_single(ppgtt->pdp.pagedir[i], ppgtt->base.dev);
+		gen8_free_page_tables(ppgtt->pdp.pagedirs[i], ppgtt->base.dev);
+		free_pd_single(ppgtt->pdp.pagedirs[i], ppgtt->base.dev);
 	}
 }
 

@@ -530,7 +570,7 @@ static int gen8_ppgtt_allocate_page_tables(struct i915_hw_ppgtt *ppgtt)
 	int i, ret;
 
 	for (i = 0; i < ppgtt->num_pd_pages; i++) {
-		ret = alloc_pt_range(ppgtt->pdp.pagedir[i],
+		ret = alloc_pt_range(ppgtt->pdp.pagedirs[i],
 				     0, I915_PDES_PER_PD, ppgtt->base.dev);
 		if (ret)
 			goto unwind_out;

@@ -540,7 +580,7 @@ static int gen8_ppgtt_allocate_page_tables(struct i915_hw_ppgtt *ppgtt)
 
 unwind_out:
 	while (i--)
-		gen8_free_page_tables(ppgtt->pdp.pagedir[i], ppgtt->base.dev);
+		gen8_free_page_tables(ppgtt->pdp.pagedirs[i], ppgtt->base.dev);
 
 	return -ENOMEM;
 }

@@ -551,8 +591,8 @@ static int gen8_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt,
 	int i;
 
 	for (i = 0; i < max_pdp; i++) {
-		ppgtt->pdp.pagedir[i] = alloc_pd_single(ppgtt->base.dev);
-		if (IS_ERR(ppgtt->pdp.pagedir[i]))
+		ppgtt->pdp.pagedirs[i] = alloc_pd_single(ppgtt->base.dev);
+		if (IS_ERR(ppgtt->pdp.pagedirs[i]))
 			goto unwind_out;
 	}
 

@@ -563,7 +603,7 @@ static int gen8_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt,
 
 unwind_out:
 	while (i--)
-		free_pd_single(ppgtt->pdp.pagedir[i],
+		free_pd_single(ppgtt->pdp.pagedirs[i],
 			       ppgtt->base.dev);
 
 	return -ENOMEM;

@@ -625,9 +665,9 @@ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size)
 	 * will never need to touch the PDEs again.
 	 */
 	for (i = 0; i < max_pdp; i++) {
-		struct i915_pagedir *pd = ppgtt->pdp.pagedir[i];
+		struct i915_pagedir *pd = ppgtt->pdp.pagedirs[i];
 		gen8_ppgtt_pde_t *pd_vaddr;
-		pd_vaddr = kmap_atomic(ppgtt->pdp.pagedir[i]->page);
+		pd_vaddr = kmap_atomic(ppgtt->pdp.pagedirs[i]->page);
 		for (j = 0; j < I915_PDES_PER_PD; j++) {
 			struct i915_pagetab *pt = pd->page_tables[j];
 			dma_addr_t addr = pt->daddr;

@@ -726,15 +766,13 @@ static void gen6_map_single(struct i915_pagedir *pd,
 /* Map all the page tables found in the ppgtt structure to incrementing page
  * directories. */
 static void gen6_map_page_range(struct drm_i915_private *dev_priv,
-				struct i915_pagedir *pd, unsigned pde, size_t n)
+				struct i915_pagedir *pd, uint32_t start, uint32_t length)
 {
-	if (WARN_ON(pde + n > I915_PDES_PER_PD))
-		n = I915_PDES_PER_PD - pde;
-
-	n += pde;
+	struct i915_pagetab *pt;
+	uint32_t pde, temp;
 
-	for (; pde < n; pde++)
-		gen6_map_single(pd, pde, pd->page_tables[pde]);
+	gen6_for_each_pde(pt, pd, start, length, temp, pde)
+		gen6_map_single(pd, pde, pt);
 
 	/* Make sure write is complete before other code can use this page
 	 * table. Also require for WC mapped PTEs */

@@ -1023,6 +1061,51 @@ static void gen6_ppgtt_insert_entries(struct i915_address_space *vm,
 		kunmap_atomic(pt_vaddr);
 }
 
+static int gen6_alloc_va_range(struct i915_address_space *vm,
+			       uint64_t start, uint64_t length)
+{
+	struct i915_hw_ppgtt *ppgtt =
+		        container_of(vm, struct i915_hw_ppgtt, base);
+	struct i915_pagetab *pt;
+	uint32_t pde, temp;
+
+	gen6_for_each_pde(pt, &ppgtt->pd, start, length, temp, pde) {
+		int j;
+
+		DECLARE_BITMAP(tmp_bitmap, GEN6_PTES_PER_PT);
+		bitmap_zero(tmp_bitmap, GEN6_PTES_PER_PT);
+		bitmap_set(tmp_bitmap, gen6_pte_index(start),
+			   gen6_pte_count(start, length));
+
+		/* TODO: To be done in the next patch. Map the page/insert
+		 * entries here */
+		for_each_set_bit(j, tmp_bitmap, GEN6_PTES_PER_PT) {
+			if (test_bit(j, pt->used_ptes)) {
+				/* Check that we're changing cache levels */
+			}
+		}
+
+		bitmap_or(pt->used_ptes, pt->used_ptes, tmp_bitmap,
+			  GEN6_PTES_PER_PT);
+	}
+
+	return 0;
+}
+
+static void gen6_teardown_va_range(struct i915_address_space *vm,
+				   uint64_t start, uint64_t length)
+{
+	struct i915_hw_ppgtt *ppgtt =
+		        container_of(vm, struct i915_hw_ppgtt, base);
+	struct i915_pagetab *pt;
+	uint32_t pde, temp;
+
+	gen6_for_each_pde(pt, &ppgtt->pd, start, length, temp, pde) {
+		bitmap_clear(pt->used_ptes, gen6_pte_index(start),
+			     gen6_pte_count(start, length));
+	}
+}
+
 static void gen6_ppgtt_free(struct i915_hw_ppgtt *ppgtt)
 {
 	int i;

@@ -1030,6 +1113,7 @@ static void gen6_ppgtt_free(struct i915_hw_ppgtt *ppgtt)
 	for (i = 0; i < ppgtt->num_pd_entries; i++)
 		free_pt_single(ppgtt->pd.page_tables[i], ppgtt->base.dev);
 
+	free_pt_scratch(ppgtt->scratch_pt, ppgtt->base.dev);
 	free_pd_single(&ppgtt->pd, ppgtt->base.dev);
 }
 

@@ -1057,6 +1141,9 @@ static int gen6_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt)
 	 * size. We allocate at the top of the GTT to avoid fragmentation.
 	 */
 	BUG_ON(!drm_mm_initialized(&dev_priv->gtt.base.mm));
+	ppgtt->scratch_pt = alloc_pt_scratch(ppgtt->base.dev);
+	if (IS_ERR(ppgtt->scratch_pt))
+		return PTR_ERR(ppgtt->scratch_pt);
 alloc:
 	ret = drm_mm_insert_node_in_range_generic(&dev_priv->gtt.base.mm,
 						  &ppgtt->node, GEN6_PD_SIZE,

@@ -1068,20 +1155,25 @@ alloc:
 					       GEN6_PD_SIZE, GEN6_PD_ALIGN,
 					       I915_CACHE_NONE, 0);
 		if (ret)
-			return ret;
+			goto err_out;
 
 		retried = true;
 		goto alloc;
 	}
 
 	if (ret)
-		return ret;
+		goto err_out;
+
 
 	if (ppgtt->node.start < dev_priv->gtt.mappable_end)
 		DRM_DEBUG("Forced to use aperture for PDEs\n");
 
 	ppgtt->num_pd_entries = I915_PDES_PER_PD;
 	return 0;
+
+err_out:
+	free_pt_scratch(ppgtt->scratch_pt, ppgtt->base.dev);
+	return ret;
 }
 
 static int gen6_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt)

@@ -1126,6 +1218,8 @@ static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
 	if (ret)
 		return ret;
 
+	ppgtt->base.allocate_va_range = gen6_alloc_va_range;
+	ppgtt->base.teardown_va_range = gen6_teardown_va_range;
 	ppgtt->base.clear_range = gen6_ppgtt_clear_range;
 	ppgtt->base.insert_entries = gen6_ppgtt_insert_entries;
 	ppgtt->base.cleanup = gen6_ppgtt_cleanup;

@@ -1139,7 +1233,7 @@ static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
 	ppgtt->pd_addr = (gen6_gtt_pte_t __iomem*)dev_priv->gtt.gsm +
 		ppgtt->pd.pd_offset / sizeof(gen6_gtt_pte_t);
 
-	gen6_map_page_range(dev_priv, &ppgtt->pd, 0, ppgtt->num_pd_entries);
+	gen6_map_page_range(dev_priv, &ppgtt->pd, 0, ppgtt->base.total);
 
 	DRM_DEBUG_DRIVER("Allocated pde space (%ldM) at GTT entry: %lx\n",
 			 ppgtt->node.size >> 20,

@@ -1174,13 +1268,25 @@ int i915_gem_init_ppgtt(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt)
 	return 0;
 }
 
-static void
+static int
 ppgtt_bind_vma(struct i915_vma *vma,
 	       enum i915_cache_level cache_level,
 	       u32 flags)
 {
+	int ret;
+
+	WARN_ON(flags);
+	if (vma->vm->allocate_va_range) {
+		ret = vma->vm->allocate_va_range(vma->vm,
+						 vma->node.start,
+						 vma->node.size);
+		if (ret)
+			return ret;
+	}
+
 	vma->vm->insert_entries(vma->vm, vma->obj->pages, vma->node.start,
 				cache_level);
+	return 0;
 }
 
 static void ppgtt_unbind_vma(struct i915_vma *vma)

@@ -1189,6 +1295,9 @@ static void ppgtt_unbind_vma(struct i915_vma *vma)
 			     vma->node.start,
 			     vma->obj->base.size,
 			     true);
+	if (vma->vm->teardown_va_range)
+		vma->vm->teardown_va_range(vma->vm,
+					   vma->node.start, vma->node.size);
 }
 
 extern int intel_iommu_gfx_mapped;

@@ -1496,9 +1605,9 @@ static void gen6_ggtt_clear_range(struct i915_address_space *vm,
 }
 
 
-static void i915_ggtt_bind_vma(struct i915_vma *vma,
-			       enum i915_cache_level cache_level,
-			       u32 unused)
+static int i915_ggtt_bind_vma(struct i915_vma *vma,
+			      enum i915_cache_level cache_level,
+			      u32 unused)
 {
 	const unsigned long entry = vma->node.start >> PAGE_SHIFT;
 	unsigned int flags = (cache_level == I915_CACHE_NONE) ?

@@ -1507,6 +1616,8 @@ static void i915_ggtt_bind_vma(struct i915_vma *vma,
 	BUG_ON(!i915_is_ggtt(vma->vm));
 	intel_gtt_insert_sg_entries(vma->obj->pages, entry, flags);
 	vma->obj->has_global_gtt_mapping = 1;
+
+	return 0;
 }
 
 static void i915_ggtt_clear_range(struct i915_address_space *vm,

@@ -1529,9 +1640,9 @@ static void i915_ggtt_unbind_vma(struct i915_vma *vma)
 	intel_gtt_clear_range(first, size);
 }
 
-static void ggtt_bind_vma(struct i915_vma *vma,
-			  enum i915_cache_level cache_level,
-			  u32 flags)
+static int ggtt_bind_vma(struct i915_vma *vma,
+			 enum i915_cache_level cache_level,
+			 u32 flags)
 {
 	struct drm_device *dev = vma->vm->dev;
 	struct drm_i915_private *dev_priv = dev->dev_private;

@@ -1559,7 +1670,7 @@ static void ggtt_bind_vma(struct i915_vma *vma,
 	}
 
 	if (!(flags & ALIASING_BIND))
-		return;
+		return 0;
 
 	if (dev_priv->mm.aliasing_ppgtt &&
 	    (!obj->has_aliasing_ppgtt_mapping ||