drm/i915: initial support for huge gtt pages

index 0f3fa34377c6..c4a39b7d7dd0 100644
@@ -103,7 +103,7 @@ fake_dma_object(struct drm_i915_private *i915, u64 size)
 
 	obj = i915_gem_object_alloc(i915);
 	if (!obj)
-		return ERR_PTR(-ENOMEM);
+		goto err;
 
 	drm_gem_private_object_init(&i915->drm, &obj->base, size);
 	i915_gem_object_init(obj, &fake_ops);

@@ -114,10 +114,15 @@ fake_dma_object(struct drm_i915_private *i915, u64 size)
 
 	/* Preallocate the "backing storage" */
 	if (i915_gem_object_pin_pages(obj))
-		return ERR_PTR(-ENOMEM);
+		goto err_obj;
 
 	i915_gem_object_unpin_pages(obj);
 	return obj;
+
+err_obj:
+	i915_gem_object_put(obj);
+err:
+	return ERR_PTR(-ENOMEM);
 }
 
 static int igt_ppgtt_alloc(void *arg)

index a0de5734f7d0..2da3a94fc9f3 100644
@@ -299,12 +299,12 @@ int i915_gem_evict_for_node(struct i915_address_space *vm,
 		 * those as well to make room for our guard pages.
 		 */
 		if (check_color) {
-			if (vma->node.start + vma->node.size == node->start) {
-				if (vma->node.color == node->color)
+			if (node->start + node->size == target->start) {
+				if (node->color == target->color)
 					continue;
 			}
-			if (vma->node.start == node->start + node->size) {
-				if (vma->node.color == node->color)
+			if (node->start == target->start + target->size) {
+				if (node->color == target->color)
 					continue;
 			}
 		}

index 97af353db218..e2838fb428f3 100644
@@ -202,6 +202,94 @@ static int igt_evict_for_vma(void *arg)
 	return err;
 }
 
+static void mock_color_adjust(const struct drm_mm_node *node,
+			      unsigned long color,
+			      u64 *start,
+			      u64 *end)
+{
+}
+
+static int igt_evict_for_cache_color(void *arg)
+{
+	struct drm_i915_private *i915 = arg;
+	struct i915_ggtt *ggtt = &i915->ggtt;
+	const unsigned long flags = PIN_OFFSET_FIXED;
+	struct drm_mm_node target = {
+		.start = I915_GTT_PAGE_SIZE * 2,
+		.size = I915_GTT_PAGE_SIZE,
+		.color = I915_CACHE_LLC,
+	};
+	struct drm_i915_gem_object *obj;
+	struct i915_vma *vma;
+	int err;
+
+	/* Currently the use of color_adjust is limited to cache domains within
+	 * the ggtt, and so the presence of mm.color_adjust is assumed to be
+	 * i915_gtt_color_adjust throughout our driver, so using a mock color
+	 * adjust will work just fine for our purposes.
+	 */
+	ggtt->base.mm.color_adjust = mock_color_adjust;
+
+	obj = i915_gem_object_create_internal(i915, I915_GTT_PAGE_SIZE);
+	if (IS_ERR(obj)) {
+		err = PTR_ERR(obj);
+		goto cleanup;
+	}
+	i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
+
+	vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0,
+				       I915_GTT_PAGE_SIZE | flags);
+	if (IS_ERR(vma)) {
+		pr_err("[0]i915_gem_object_ggtt_pin failed\n");
+		err = PTR_ERR(vma);
+		goto cleanup;
+	}
+
+	obj = i915_gem_object_create_internal(i915, I915_GTT_PAGE_SIZE);
+	if (IS_ERR(obj)) {
+		err = PTR_ERR(obj);
+		goto cleanup;
+	}
+	i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
+
+	/* Neighbouring; same colour - should fit */
+	vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0,
+				       (I915_GTT_PAGE_SIZE * 2) | flags);
+	if (IS_ERR(vma)) {
+		pr_err("[1]i915_gem_object_ggtt_pin failed\n");
+		err = PTR_ERR(vma);
+		goto cleanup;
+	}
+
+	i915_vma_unpin(vma);
+
+	/* Remove just the second vma */
+	err = i915_gem_evict_for_node(&ggtt->base, &target, 0);
+	if (err) {
+		pr_err("[0]i915_gem_evict_for_node returned err=%d\n", err);
+		goto cleanup;
+	}
+
+	/* Attempt to remove the first *pinned* vma, by removing the (empty)
+	 * neighbour */
+	target.color = I915_CACHE_L3_LLC;
+
+	err = i915_gem_evict_for_node(&ggtt->base, &target, 0);
+	if (!err) {
+		pr_err("[1]i915_gem_evict_for_node returned err=%d\n", err);
+		err = -EINVAL;
+		goto cleanup;
+	}
+
+	err = 0;
+
+cleanup:
+	unpin_ggtt(i915);
+	cleanup_objects(i915);
+	ggtt->base.mm.color_adjust = NULL;
+	return err;
+}
+
 static int igt_evict_vm(void *arg)
 {
 	struct drm_i915_private *i915 = arg;

@@ -241,6 +329,7 @@ int i915_gem_evict_mock_selftests(void)
 	static const struct i915_subtest tests[] = {
 		SUBTEST(igt_evict_something),
 		SUBTEST(igt_evict_for_vma),
+		SUBTEST(igt_evict_for_cache_color),
 		SUBTEST(igt_evict_vm),
 		SUBTEST(igt_overcommit),
 	};

index 1fd4128a10b1..e45b8d74cebf 100644
@@ -913,6 +913,7 @@ struct intel_device_info {
 	enum intel_platform platform;
 	u8 ring_mask; /* Rings supported by the HW */
 	u8 num_rings;
+	unsigned long page_size_mask; /* page sizes supported by the HW */
 #define DEFINE_FLAG(name) u8 name:1
 	DEV_INFO_FOR_EACH_FLAG(DEFINE_FLAG);
 #undef DEFINE_FLAG

index fb15684c1d83..6c90a2ffd0e1 100644
@@ -43,8 +43,18 @@
 #include "i915_selftest.h"
 
 #define I915_GTT_PAGE_SIZE 4096UL
+#define I915_GTT_PAGE_SIZE_64K 65536UL
+#define I915_GTT_PAGE_SIZE_2M 2097152UL
+#define I915_GTT_PAGE_SIZE_1G 1073741824UL
+
+#define I915_GTT_PAGE_SIZE_MASK (I915_GTT_PAGE_SIZE | \
+				 I915_GTT_PAGE_SIZE_64K | \
+				 I915_GTT_PAGE_SIZE_2M | \
+				 I915_GTT_PAGE_SIZE_1G)
+
 #define I915_GTT_MIN_ALIGNMENT I915_GTT_PAGE_SIZE
 
+
 #define I915_FENCE_REG_NONE -1
 #define I915_MAX_NUM_FENCES 32
 /* 32 fences + sign bit for FENCE_REG_NONE */

@@ -143,6 +153,10 @@ typedef u64 gen8_ppgtt_pml4e_t;
 #define GEN8_PPAT_ELLC_OVERRIDE		(0<<2)
 #define GEN8_PPAT(i, x)			((u64)(x) << ((i) * 8))
 
+#define GEN8_PDPE_PS_1G  BIT(7)
+#define GEN8_PDE_PS_2M   BIT(7)
+#define GEN8_PDE_IPS_64K BIT(11)
+
 struct sg_table;
 
 struct intel_rotation_info {

index 732101ed57fb..5abb7d84b65a 100644
@@ -56,6 +56,10 @@
 	.color = { .degamma_lut_size = 65, .gamma_lut_size = 257 }
 
 /* Keep in gen based order, and chronological order within a gen */
+
+#define GEN_DEFAULT_PAGE_SZ \
+	.page_size_mask = I915_GTT_PAGE_SIZE
+
 #define GEN2_FEATURES \
 	.gen = 2, .num_pipes = 1, \
 	.has_overlay = 1, .overlay_needs_physical = 1, \

@@ -63,6 +67,7 @@
 	.hws_needs_physical = 1, \
 	.ring_mask = RENDER_RING, \
 	GEN_DEFAULT_PIPEOFFSETS, \
+	GEN_DEFAULT_PAGE_SZ, \
 	CURSOR_OFFSETS
 
 static const struct intel_device_info intel_i830_info = {

@@ -95,6 +100,7 @@ static const struct intel_device_info intel_i865g_info = {
 	.has_gmch_display = 1, \
 	.ring_mask = RENDER_RING, \
 	GEN_DEFAULT_PIPEOFFSETS, \
+	GEN_DEFAULT_PAGE_SZ, \
 	CURSOR_OFFSETS
 
 static const struct intel_device_info intel_i915g_info = {

@@ -153,6 +159,7 @@ static const struct intel_device_info intel_pineview_info = {
 	.has_gmch_display = 1, \
 	.ring_mask = RENDER_RING, \
 	GEN_DEFAULT_PIPEOFFSETS, \
+	GEN_DEFAULT_PAGE_SZ, \
 	CURSOR_OFFSETS
 
 static const struct intel_device_info intel_i965g_info = {

@@ -193,6 +200,7 @@ static const struct intel_device_info intel_gm45_info = {
 	.has_gmbus_irq = 1, \
 	.ring_mask = RENDER_RING | BSD_RING, \
 	GEN_DEFAULT_PIPEOFFSETS, \
+	GEN_DEFAULT_PAGE_SZ, \
 	CURSOR_OFFSETS
 
 static const struct intel_device_info intel_ironlake_d_info = {

@@ -218,6 +226,7 @@ static const struct intel_device_info intel_ironlake_m_info = {
 	.has_hw_contexts = 1, \
 	.has_aliasing_ppgtt = 1, \
 	GEN_DEFAULT_PIPEOFFSETS, \
+	GEN_DEFAULT_PAGE_SZ, \
 	CURSOR_OFFSETS
 
 static const struct intel_device_info intel_sandybridge_d_info = {

@@ -244,6 +253,7 @@ static const struct intel_device_info intel_sandybridge_m_info = {
 	.has_aliasing_ppgtt = 1, \
 	.has_full_ppgtt = 1, \
 	GEN_DEFAULT_PIPEOFFSETS, \
+	GEN_DEFAULT_PAGE_SZ, \
 	IVB_CURSOR_OFFSETS
 
 static const struct intel_device_info intel_ivybridge_d_info = {

@@ -282,6 +292,7 @@ static const struct intel_device_info intel_valleyview_info = {
 	.has_full_ppgtt = 1,
 	.ring_mask = RENDER_RING | BSD_RING | BLT_RING,
 	.display_mmio_offset = VLV_DISPLAY_BASE,
+	GEN_DEFAULT_PAGE_SZ,
 	GEN_DEFAULT_PIPEOFFSETS,
 	CURSOR_OFFSETS
 };

@@ -308,7 +319,8 @@ static const struct intel_device_info intel_haswell_info = {
 	BDW_COLORS, \
 	.has_logical_ring_contexts = 1, \
 	.has_full_48bit_ppgtt = 1, \
-	.has_64bit_reloc = 1
+	.has_64bit_reloc = 1, \
+	.page_size_mask = I915_GTT_PAGE_SIZE | I915_GTT_PAGE_SIZE_2M | I915_GTT_PAGE_SIZE_1G
 
 static const struct intel_device_info intel_broadwell_info = {
 	BDW_FEATURES,

@@ -341,13 +353,18 @@ static const struct intel_device_info intel_cherryview_info = {
 	.has_aliasing_ppgtt = 1,
 	.has_full_ppgtt = 1,
 	.display_mmio_offset = VLV_DISPLAY_BASE,
+	.page_size_mask = I915_GTT_PAGE_SIZE | I915_GTT_PAGE_SIZE_64K | I915_GTT_PAGE_SIZE_2M | I915_GTT_PAGE_SIZE_1G,
 	GEN_CHV_PIPEOFFSETS,
 	CURSOR_OFFSETS,
 	CHV_COLORS,
 };
 
+#define GEN9_DEFAULT_PAGE_SZ \
+	.page_size_mask = I915_GTT_PAGE_SIZE | I915_GTT_PAGE_SIZE_64K | I915_GTT_PAGE_SIZE_2M | I915_GTT_PAGE_SIZE_1G
+
 static const struct intel_device_info intel_skylake_info = {
 	BDW_FEATURES,
+	GEN9_DEFAULT_PAGE_SZ,
 	.platform = INTEL_SKYLAKE,
 	.gen = 9,
 	.has_csr = 1,

@@ -357,6 +374,7 @@ static const struct intel_device_info intel_skylake_info = {
 
 static const struct intel_device_info intel_skylake_gt3_info = {
 	BDW_FEATURES,
+	GEN9_DEFAULT_PAGE_SZ,
 	.platform = INTEL_SKYLAKE,
 	.gen = 9,
 	.has_csr = 1,

@@ -389,6 +407,7 @@ static const struct intel_device_info intel_skylake_gt3_info = {
 	.has_aliasing_ppgtt = 1, \
 	.has_full_ppgtt = 1, \
 	.has_full_48bit_ppgtt = 1, \
+	GEN9_DEFAULT_PAGE_SZ, \
 	GEN_DEFAULT_PIPEOFFSETS, \
 	IVB_CURSOR_OFFSETS, \
 	BDW_COLORS

@@ -409,6 +428,7 @@ static const struct intel_device_info intel_geminilake_info = {
 
 static const struct intel_device_info intel_kabylake_info = {
 	BDW_FEATURES,
+	GEN9_DEFAULT_PAGE_SZ,
 	.platform = INTEL_KABYLAKE,
 	.gen = 9,
 	.has_csr = 1,

@@ -418,6 +438,7 @@ static const struct intel_device_info intel_kabylake_info = {
 
 static const struct intel_device_info intel_kabylake_gt3_info = {
 	BDW_FEATURES,
+	GEN9_DEFAULT_PAGE_SZ,
 	.platform = INTEL_KABYLAKE,
 	.gen = 9,
 	.has_csr = 1,

index 7c20601fe1de..0a6ed2c54629 100644
@@ -3975,6 +3975,8 @@ void i915_gem_object_init(struct drm_i915_gem_object *obj,
 
 	obj->ops = ops;
 
+	obj->page_size = I915_GTT_PAGE_SIZE;
+
 	reservation_object_init(&obj->__builtin_resv);
 	obj->resv = &obj->__builtin_resv;
 

index 33b0dc4782a9..31bcf87d3745 100644
@@ -90,6 +90,8 @@ struct drm_i915_gem_object {
 
 	unsigned long flags;
 
+	unsigned long page_size; /* 4K(default), 64K, 2M, 1G */
+
 	/**
 	 * Have we taken a reference for the object for incomplete GPU
 	 * activity?

index cee9c4fec52a..9274974956b3 100644
@@ -205,7 +205,7 @@ static int ppgtt_bind_vma(struct i915_vma *vma,
 		pte_flags |= PTE_READ_ONLY;
 
 	vma->vm->insert_entries(vma->vm, vma->pages, vma->node.start,
-				cache_level, pte_flags);
+				vma->obj->page_size, cache_level, pte_flags);
 
 	return 0;
 }

@@ -906,6 +906,7 @@ gen8_ppgtt_insert_pte_entries(struct i915_hw_ppgtt *ppgtt,
 static void gen8_ppgtt_insert_3lvl(struct i915_address_space *vm,
 				   struct sg_table *pages,
 				   u64 start,
+				   unsigned long page_size,
 				   enum i915_cache_level cache_level,
 				   u32 unused)
 {

@@ -924,6 +925,7 @@ static void gen8_ppgtt_insert_3lvl(struct i915_address_space *vm,
 static void gen8_ppgtt_insert_4lvl(struct i915_address_space *vm,
 				   struct sg_table *pages,
 				   u64 start,
+				   unsigned long page_size,
 				   enum i915_cache_level cache_level,
 				   u32 unused)
 {

@@ -935,9 +937,23 @@ static void gen8_ppgtt_insert_4lvl(struct i915_address_space *vm,
 	};
 	struct i915_page_directory_pointer **pdps = ppgtt->pml4.pdps;
 	struct gen8_insert_pte idx = gen8_insert_pte(start);
+	bool (*insert_entries)(struct i915_hw_ppgtt *ppgtt,
+			       struct i915_page_directory_pointer *pdp,
+			       struct sgt_dma *iter,
+			       struct gen8_insert_pte *idx,
+			       enum i915_cache_level cache_level);
+
+	switch (page_size) {
+	case I915_GTT_PAGE_SIZE:
+		insert_entries = gen8_ppgtt_insert_pte_entries;
+		break;
+	default:
+		MISSING_CASE(page_size);
+		return;
+	} 
 
-	while (gen8_ppgtt_insert_pte_entries(ppgtt, pdps[idx.pml4e++], &iter,
-					     &idx, cache_level))
+	while (insert_entries(ppgtt, pdps[idx.pml4e++], &iter, &idx,
+			      cache_level))
 		GEM_BUG_ON(idx.pml4e >= GEN8_PML4ES_PER_PML4);
 }
 

@@ -1620,6 +1636,7 @@ static void gen6_ppgtt_clear_range(struct i915_address_space *vm,
 static void gen6_ppgtt_insert_entries(struct i915_address_space *vm,
 				      struct sg_table *pages,
 				      u64 start,
+				      unsigned long page_size,
 				      enum i915_cache_level cache_level,
 				      u32 flags)
 {

@@ -2093,6 +2110,7 @@ static void gen8_ggtt_insert_page(struct i915_address_space *vm,
 static void gen8_ggtt_insert_entries(struct i915_address_space *vm,
 				     struct sg_table *st,
 				     u64 start,
+				     unsigned long page_size,
 				     enum i915_cache_level level,
 				     u32 unused)
 {

@@ -2140,6 +2158,7 @@ static void gen6_ggtt_insert_page(struct i915_address_space *vm,
 static void gen6_ggtt_insert_entries(struct i915_address_space *vm,
 				     struct sg_table *st,
 				     u64 start,
+				     unsigned long page_size,
 				     enum i915_cache_level level,
 				     u32 flags)
 {

@@ -2224,6 +2243,7 @@ static void i915_ggtt_insert_page(struct i915_address_space *vm,
 static void i915_ggtt_insert_entries(struct i915_address_space *vm,
 				     struct sg_table *pages,
 				     u64 start,
+				     unsigned long page_size,
 				     enum i915_cache_level cache_level,
 				     u32 unused)
 {

@@ -2260,7 +2280,7 @@ static int ggtt_bind_vma(struct i915_vma *vma,
 
 	intel_runtime_pm_get(i915);
 	vma->vm->insert_entries(vma->vm, vma->pages, vma->node.start,
-				cache_level, pte_flags);
+				I915_GTT_PAGE_SIZE, cache_level, pte_flags);
 	intel_runtime_pm_put(i915);
 
 	/*

@@ -2314,14 +2334,14 @@ static int aliasing_gtt_bind_vma(struct i915_vma *vma,
 
 		appgtt->base.insert_entries(&appgtt->base,
 					    vma->pages, vma->node.start,
-					    cache_level, pte_flags);
+					    I915_GTT_PAGE_SIZE, cache_level, pte_flags);
 	}
 
 	if (flags & I915_VMA_GLOBAL_BIND) {
 		intel_runtime_pm_get(i915);
 		vma->vm->insert_entries(vma->vm,
 					vma->pages, vma->node.start,
-					cache_level, pte_flags);
+					I915_GTT_PAGE_SIZE, cache_level, pte_flags);
 		intel_runtime_pm_put(i915);
 	}
 

index 6c90a2ffd0e1..1f51402cf816 100644
@@ -328,6 +328,7 @@ struct i915_address_space {
 	void (*insert_entries)(struct i915_address_space *vm,
 			       struct sg_table *st,
 			       u64 start,
+			       unsigned long page_size,
 			       enum i915_cache_level cache_level,
 			       u32 flags);
 	void (*cleanup)(struct i915_address_space *vm);

index c4a39b7d7dd0..c7963efe46ba 100644
@@ -256,7 +256,8 @@ static int lowlevel_hole(struct drm_i915_private *i915,
 				break;
 
 			vm->insert_entries(vm, obj->mm.pages, addr,
-					   I915_CACHE_NONE, 0);
+					   I915_GTT_PAGE_SIZE, I915_CACHE_NONE,
+					   0);
 		}
 		count = n;
 

index a61309c7cb3e..a2137100d2f5 100644
@@ -35,6 +35,7 @@ static void mock_insert_page(struct i915_address_space *vm,
 static void mock_insert_entries(struct i915_address_space *vm,
 				struct sg_table *st,
 				u64 start,
+				unsigned long page_size,
 				enum i915_cache_level level, u32 flags)
 {
 }

index 9274974956b3..6b70332cafac 100644
@@ -2394,10 +2394,10 @@ void i915_gem_gtt_finish_pages(struct drm_i915_gem_object *obj,
 	dma_unmap_sg(kdev, pages->sgl, pages->nents, PCI_DMA_BIDIRECTIONAL);
 }
 
-static void i915_gtt_color_adjust(const struct drm_mm_node *node,
-				  unsigned long color,
-				  u64 *start,
-				  u64 *end)
+static void i915_cache_color_adjust(const struct drm_mm_node *node,
+				    unsigned long color,
+				    u64 *start,
+				    u64 *end)
 {
 	if (node->allocated && node->color != color)
 		*start += I915_GTT_PAGE_SIZE;

@@ -2969,7 +2969,7 @@ int i915_ggtt_init_hw(struct drm_i915_private *dev_priv)
 	mutex_lock(&dev_priv->drm.struct_mutex);
 	i915_address_space_init(&ggtt->base, dev_priv, "[global]");
 	if (!HAS_LLC(dev_priv) && !USES_PPGTT(dev_priv))
-		ggtt->base.mm.color_adjust = i915_gtt_color_adjust;
+		ggtt->base.mm.color_adjust = i915_cache_color_adjust;
 	mutex_unlock(&dev_priv->drm.struct_mutex);
 
 	if (!io_mapping_init_wc(&dev_priv->ggtt.mappable,

index e45b8d74cebf..aac764b5aad4 100644
@@ -3526,7 +3526,7 @@ int i915_perf_open_ioctl(struct drm_device *dev, void *data,
 /* i915_gem_evict.c */
 int __must_check i915_gem_evict_something(struct i915_address_space *vm,
 					  u64 min_size, u64 alignment,
-					  unsigned cache_level,
+					  unsigned long color,
 					  u64 start, u64 end,
 					  unsigned flags);
 int __must_check i915_gem_evict_for_node(struct i915_address_space *vm,

index 0a6ed2c54629..9acf279e5f93 100644
@@ -3392,7 +3392,8 @@ int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
 		obj->cache_dirty = true;
 
 	list_for_each_entry(vma, &obj->vma_list, obj_link)
-		vma->node.color = cache_level;
+		if (i915_uses_cache_coloring(vma->vm))
+			vma->node.color = cache_level;
 	obj->cache_level = cache_level;
 
 	return 0;

index 2da3a94fc9f3..f9364f917b67 100644
@@ -74,7 +74,7 @@ mark_free(struct drm_mm_scan *scan,
  * @vm: address space to evict from
  * @min_size: size of the desired free space
  * @alignment: alignment constraint of the desired free space
- * @cache_level: cache_level for the desired space
+ * @color: color for the desired space
  * @start: start (inclusive) of the range from which to evict objects
  * @end: end (exclusive) of the range from which to evict objects
  * @flags: additional flags to control the eviction algorithm

@@ -95,7 +95,7 @@ mark_free(struct drm_mm_scan *scan,
 int
 i915_gem_evict_something(struct i915_address_space *vm,
 			 u64 min_size, u64 alignment,
-			 unsigned cache_level,
+			 unsigned long color,
 			 u64 start, u64 end,
 			 unsigned flags)
 {

@@ -134,7 +134,7 @@ i915_gem_evict_something(struct i915_address_space *vm,
 	if (flags & PIN_MAPPABLE)
 		mode = DRM_MM_INSERT_LOW;
 	drm_mm_scan_init_with_range(&scan, &vm->mm,
-				    min_size, alignment, cache_level,
+				    min_size, alignment, color,
 				    start, end, mode);
 
 	/* Retire before we search the active list. Although we have

@@ -254,7 +254,6 @@ int i915_gem_evict_for_node(struct i915_address_space *vm,
 	u64 start = target->start;
 	u64 end = start + target->size;
 	struct i915_vma *vma, *next;
-	bool check_color;
 	int ret = 0;
 
 	lockdep_assert_held(&vm->i915->drm.struct_mutex);

@@ -271,8 +270,7 @@ int i915_gem_evict_for_node(struct i915_address_space *vm,
 	if (!(flags & PIN_NONBLOCK))
 		i915_gem_retire_requests(vm->i915);
 
-	check_color = vm->mm.color_adjust;
-	if (check_color) {
+	if (i915_uses_cache_coloring(vm)) {
 		/* Expand search to cover neighbouring guard pages (or lack!) */
 		if (start)
 			start -= I915_GTT_PAGE_SIZE;

@@ -298,7 +296,7 @@ int i915_gem_evict_for_node(struct i915_address_space *vm,
 		 * abutt and conflict. If they are in conflict, then we evict
 		 * those as well to make room for our guard pages.
 		 */
-		if (check_color) {
+		if (i915_uses_cache_coloring(vm)) {
 			if (node->start + node->size == target->start) {
 				if (node->color == target->color)
 					continue;

index 1f51402cf816..8d7436105718 100644
@@ -346,6 +346,12 @@ struct i915_address_space {
 #define i915_is_ggtt(V) (!(V)->file)
 
 static inline bool
+i915_uses_cache_coloring(const struct i915_address_space *vm)
+{
+	return vm->mm.color_adjust && i915_is_ggtt(vm);
+}
+
+static inline bool
 i915_vm_is_48bit(const struct i915_address_space *vm)
 {
 	return (vm->total - 1) >> 32;

index 1aba47024656..31e2327492ba 100644
@@ -390,7 +390,7 @@ bool i915_gem_valid_gtt_space(struct i915_vma *vma, unsigned long cache_level)
 	 * these constraints apply and set the drm_mm.color_adjust
 	 * appropriately.
 	 */
-	if (vma->vm->mm.color_adjust == NULL)
+	if (!i915_uses_cache_coloring(vma->vm))
 		return true;
 
 	/* Only valid to be called on an already inserted vma */

@@ -429,6 +429,7 @@ i915_vma_insert(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
 	struct drm_i915_gem_object *obj = vma->obj;
 	u64 start, end;
 	int ret;
+	unsigned long color = 0;
 
 	GEM_BUG_ON(vma->flags & (I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND));
 	GEM_BUG_ON(drm_mm_node_allocated(&vma->node));

@@ -471,6 +472,9 @@ i915_vma_insert(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
 	if (ret)
 		return ret;
 
+	if (i915_uses_cache_coloring(vma->vm))
+		color = obj->cache_level;
+
 	if (flags & PIN_OFFSET_FIXED) {
 		u64 offset = flags & PIN_OFFSET_MASK;
 		if (!IS_ALIGNED(offset, alignment) ||

@@ -480,13 +484,13 @@ i915_vma_insert(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
 		}
 
 		ret = i915_gem_gtt_reserve(vma->vm, &vma->node,
-					   size, offset, obj->cache_level,
+					   size, offset, color,
 					   flags);
 		if (ret)
 			goto err_unpin;
 	} else {
 		ret = i915_gem_gtt_insert(vma->vm, &vma->node,
-					  size, alignment, obj->cache_level,
+					  size, alignment, color,
 					  start, end, flags);
 		if (ret)
 			goto err_unpin;

index 31e2327492ba..0d289cbc0123 100644
@@ -373,11 +373,6 @@ void __i915_vma_set_map_and_fenceable(struct i915_vma *vma)
 		vma->flags &= ~I915_VMA_CAN_FENCE;
 }
 
-static bool color_differs(struct drm_mm_node *node, unsigned long color)
-{
-	return node->allocated && node->color != color;
-}
-
 bool i915_gem_valid_gtt_space(struct i915_vma *vma, unsigned long cache_level)
 {
 	struct drm_mm_node *node = &vma->node;

@@ -398,11 +393,11 @@ bool i915_gem_valid_gtt_space(struct i915_vma *vma, unsigned long cache_level)
 	GEM_BUG_ON(list_empty(&node->node_list));
 
 	other = list_prev_entry(node, node_list);
-	if (color_differs(other, cache_level) && !drm_mm_hole_follows(other))
+	if (i915_color_differs(other, cache_level) && !drm_mm_hole_follows(other))
 		return false;
 
 	other = list_next_entry(node, node_list);
-	if (color_differs(other, cache_level) && !drm_mm_hole_follows(node))
+	if (i915_color_differs(other, cache_level) && !drm_mm_hole_follows(node))
 		return false;
 
 	return true;

index 2e03f81dddbe..5492b03d6a77 100644
@@ -284,6 +284,12 @@ static inline void i915_vma_unpin(struct i915_vma *vma)
 	__i915_vma_unpin(vma);
 }
 
+static inline bool i915_color_differs(const struct drm_mm_node *node,
+				      unsigned long color)
+{
+	return node->allocated && node->color != color;
+}
+
 /**
  * i915_vma_pin_iomap - calls ioremap_wc to map the GGTT VMA via the aperture
  * @vma: VMA to iomap

index 6b70332cafac..0fb67941ba6b 100644
@@ -1331,6 +1331,28 @@ static int gen8_preallocate_top_level_pdp(struct i915_hw_ppgtt *ppgtt)
 	return -ENOMEM;
 }
 
+static void i915_page_color_adjust(const struct drm_mm_node *node,
+				   unsigned long color,
+				   u64 *start,
+				   u64 *end)
+{
+	GEM_BUG_ON(color & ~I915_GTT_PAGE_SIZE_MASK);
+
+	if (!(color & (I915_GTT_PAGE_SIZE_64K | I915_GTT_PAGE_SIZE)))
+		return;
+
+	GEM_BUG_ON(node->allocated && node->color & ~I915_GTT_PAGE_SIZE_MASK);
+
+	if (i915_color_differs(node, color))
+		*start = roundup(*start, 1 << GEN8_PDE_SHIFT);
+
+	node = list_next_entry(node, node_list);
+	if (i915_color_differs(node, color))
+		*end = rounddown(*end, 1 << GEN8_PDE_SHIFT);
+
+	GEM_BUG_ON(node->allocated && node->color & ~I915_GTT_PAGE_SIZE_MASK);
+}
+
 /*
  * GEN8 legacy ppgtt programming is accomplished through a max 4 PDP registers
  * with a net effect resembling a 2-level page table in normal x86 terms. Each

@@ -1371,6 +1393,9 @@ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
 		ppgtt->base.allocate_va_range = gen8_ppgtt_alloc_4lvl;
 		ppgtt->base.insert_entries = gen8_ppgtt_insert_4lvl;
 		ppgtt->base.clear_range = gen8_ppgtt_clear_4lvl;
+
+		if (INTEL_INFO(dev_priv)->page_size_mask & I915_GTT_PAGE_SIZE_64K)
+			ppgtt->base.mm.color_adjust = i915_page_color_adjust;
 	} else {
 		ret = __pdp_init(&ppgtt->base, &ppgtt->pdp);
 		if (ret)

index 8d7436105718..e1b8d1c4734b 100644
@@ -352,6 +352,12 @@ i915_uses_cache_coloring(const struct i915_address_space *vm)
 }
 
 static inline bool
+i915_uses_page_coloring(const struct i915_address_space *vm)
+{
+	return vm->mm.color_adjust && !i915_is_ggtt(vm);
+}
+
+static inline bool
 i915_vm_is_48bit(const struct i915_address_space *vm)
 {
 	return (vm->total - 1) >> 32;

index 0d289cbc0123..ee5395e744e1 100644
@@ -469,6 +469,8 @@ i915_vma_insert(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
 
 	if (i915_uses_cache_coloring(vma->vm))
 		color = obj->cache_level;
+	else if (i915_uses_page_coloring(vma->vm))
+		color = obj->page_size;
 
 	if (flags & PIN_OFFSET_FIXED) {
 		u64 offset = flags & PIN_OFFSET_MASK;

index 0fb67941ba6b..4d863de6473d 100644
@@ -848,6 +848,73 @@ static __always_inline struct gen8_insert_pte gen8_insert_pte(u64 start)
 }
 
 static __always_inline bool
+gen8_ppgtt_insert_64K_pte_entries(struct i915_hw_ppgtt *ppgtt,
+				  struct i915_page_directory_pointer *pdp,
+				  struct sgt_dma *iter,
+				  struct gen8_insert_pte *idx,
+				  enum i915_cache_level cache_level)
+{
+	struct i915_page_directory *pd;
+	const gen8_pte_t pte_encode = gen8_pte_encode(0, cache_level);
+	gen8_pte_t *vaddr;
+	bool ret;
+
+	GEM_BUG_ON(idx->pte % 16);
+	GEM_BUG_ON(idx->pdpe >= i915_pdpes_per_pdp(&ppgtt->base));
+	/* TODO: probably move this to the allocation phase.. */
+	pd = pdp->page_directory[idx->pdpe];
+	vaddr = kmap_atomic_px(pd);
+	vaddr[idx->pde] |= GEN8_PDE_IPS_64K;
+	kunmap_atomic(vaddr);
+
+	vaddr = kmap_atomic_px(pd->page_table[idx->pde]);
+	do {
+		vaddr[idx->pte] = pte_encode | iter->dma;
+		iter->dma += I915_GTT_PAGE_SIZE_64K;
+		if (iter->dma >= iter->max) {
+			iter->sg = __sg_next(iter->sg);
+			if (!iter->sg) {
+				ret = false;
+				break;
+			}
+
+			iter->dma = sg_dma_address(iter->sg);
+			iter->max = iter->dma + iter->sg->length;
+		}
+
+		idx->pte += 16;
+
+		if (idx->pte == GEN8_PTES) {
+			idx->pte = 0;
+
+			if (++idx->pde == I915_PDES) {
+				idx->pde = 0;
+
+				/* Limited by sg length for 3lvl */
+				if (++idx->pdpe == GEN8_PML4ES_PER_PML4) {
+					idx->pdpe = 0;
+					ret = true;
+					break;
+				}
+
+				GEM_BUG_ON(idx->pdpe >= i915_pdpes_per_pdp(&ppgtt->base));
+				pd = pdp->page_directory[idx->pdpe];
+			}
+
+			kunmap_atomic(vaddr);
+			vaddr = kmap_atomic_px(pd);
+			vaddr[idx->pde] |= GEN8_PDE_IPS_64K;
+			kunmap_atomic(vaddr);
+
+			vaddr = kmap_atomic_px(pd->page_table[idx->pde]);
+		}
+	} while (1);
+	kunmap_atomic(vaddr);
+
+	return ret;
+}
+
+static __always_inline bool
 gen8_ppgtt_insert_pte_entries(struct i915_hw_ppgtt *ppgtt,
 			      struct i915_page_directory_pointer *pdp,
 			      struct sgt_dma *iter,

@@ -947,6 +1014,9 @@ static void gen8_ppgtt_insert_4lvl(struct i915_address_space *vm,
 	case I915_GTT_PAGE_SIZE:
 		insert_entries = gen8_ppgtt_insert_pte_entries;
 		break;
+	case I915_GTT_PAGE_SIZE_64K:
+		insert_entries = gen8_ppgtt_insert_64K_pte_entries;
+		break;
 	default:
 		MISSING_CASE(page_size);
 		return;

index 4d863de6473d..f71a6320ae1b 100644
@@ -848,6 +848,56 @@ static __always_inline struct gen8_insert_pte gen8_insert_pte(u64 start)
 }
 
 static __always_inline bool
+gen8_ppgtt_insert_2M_pde_entries(struct i915_hw_ppgtt *ppgtt,
+				 struct i915_page_directory_pointer *pdp,
+				 struct sgt_dma *iter,
+				 struct gen8_insert_pte *idx,
+				 enum i915_cache_level cache_level)
+{
+	const gen8_pte_t pde_encode = gen8_pte_encode(GEN8_PDE_PS_2M,
+						      cache_level);
+	gen8_pte_t *vaddr;
+	bool ret;
+
+	GEM_BUG_ON(idx->pte);
+	GEM_BUG_ON(idx->pdpe >= i915_pdpes_per_pdp(&ppgtt->base));
+	vaddr = kmap_atomic_px(pdp->page_directory[idx->pdpe]);
+	do {
+		vaddr[idx->pde] = pde_encode | iter->dma;
+		iter->dma += I915_GTT_PAGE_SIZE_2M;
+		if (iter->dma >= iter->max) {
+			iter->sg = __sg_next(iter->sg);
+			if (!iter->sg) {
+				ret = false;
+				break;
+			}
+
+			iter->dma = sg_dma_address(iter->sg);
+			iter->max = iter->dma + iter->sg->length;
+		}
+
+		if (++idx->pde == I915_PDES) {
+			idx->pde = 0;
+
+			if (++idx->pdpe == GEN8_PML4ES_PER_PML4) {
+				idx->pdpe = 0;
+				ret = true;
+				break;
+			}
+
+			kunmap_atomic(vaddr);
+			vaddr = kmap_atomic_px(pdp->page_directory[idx->pdpe]);
+		}
+
+	} while (1);
+	kunmap_atomic(vaddr);
+
+	mark_tlbs_dirty(ppgtt);
+
+	return ret;
+}
+
+static __always_inline bool
 gen8_ppgtt_insert_64K_pte_entries(struct i915_hw_ppgtt *ppgtt,
 				  struct i915_page_directory_pointer *pdp,
 				  struct sgt_dma *iter,

@@ -1017,6 +1067,9 @@ static void gen8_ppgtt_insert_4lvl(struct i915_address_space *vm,
 	case I915_GTT_PAGE_SIZE_64K:
 		insert_entries = gen8_ppgtt_insert_64K_pte_entries;
 		break;
+	case I915_GTT_PAGE_SIZE_2M:
+		insert_entries = gen8_ppgtt_insert_2M_pde_entries;
+		break;
 	default:
 		MISSING_CASE(page_size);
 		return;

index f71a6320ae1b..ecc63bc1fcc1 100644
@@ -848,6 +848,48 @@ static __always_inline struct gen8_insert_pte gen8_insert_pte(u64 start)
 }
 
 static __always_inline bool
+gen8_ppgtt_insert_1G_pdpe_entries(struct i915_hw_ppgtt *ppgtt,
+				  struct i915_page_directory_pointer *pdp,
+				  struct sgt_dma *iter,
+				  struct gen8_insert_pte *idx,
+				  enum i915_cache_level cache_level)
+{
+	const gen8_pte_t pdpe_encode = gen8_pte_encode(GEN8_PDPE_PS_1G,
+						       cache_level);
+	gen8_pte_t *vaddr;
+	bool ret;
+
+	GEM_BUG_ON(idx->pte);
+	GEM_BUG_ON(idx->pde);
+	GEM_BUG_ON(idx->pdpe >= i915_pdpes_per_pdp(&ppgtt->base));
+	vaddr = kmap_atomic_px(pdp);
+	do {
+		vaddr[idx->pdpe] = pdpe_encode | iter->dma;
+		iter->dma += I915_GTT_PAGE_SIZE_1G;
+		if (iter->dma >= iter->max) {
+			iter->sg = __sg_next(iter->sg);
+			if (!iter->sg) {
+				ret = false;
+				break;
+			}
+
+			iter->dma = sg_dma_address(iter->sg);
+			iter->max = iter->dma + iter->sg->length;
+		}
+
+		if (++idx->pdpe == GEN8_PML4ES_PER_PML4) {
+			idx->pdpe = 0;
+			ret = true;
+			break;
+		}
+
+	} while (1);
+	kunmap_atomic(vaddr);
+
+	return ret;
+}
+
+static __always_inline bool
 gen8_ppgtt_insert_2M_pde_entries(struct i915_hw_ppgtt *ppgtt,
 				 struct i915_page_directory_pointer *pdp,
 				 struct sgt_dma *iter,

@@ -1070,6 +1112,9 @@ static void gen8_ppgtt_insert_4lvl(struct i915_address_space *vm,
 	case I915_GTT_PAGE_SIZE_2M:
 		insert_entries = gen8_ppgtt_insert_2M_pde_entries;
 		break;
+	case I915_GTT_PAGE_SIZE_1G:
+		insert_entries = gen8_ppgtt_insert_1G_pdpe_entries;
+		break;
 	default:
 		MISSING_CASE(page_size);
 		return;

index c7963efe46ba..9b2a7228a78f 100644
@@ -31,6 +31,7 @@
 #include "mock_context.h"
 #include "mock_drm.h"
 #include "mock_gem_device.h"
+#include "mock_gtt.h"
 
 static void fake_free_pages(struct drm_i915_gem_object *obj,
 			    struct sg_table *pages)

@@ -1306,6 +1307,72 @@ static int igt_gtt_reserve(void *arg)
 	return err;
 }
 
+static int igt_ppgtt_page_color(void *arg)
+{
+	struct drm_mm mm;
+	struct drm_mm_node *node, *prev, *next;
+	unsigned long page_colors[] = {
+		I915_GTT_PAGE_SIZE,
+		I915_GTT_PAGE_SIZE_64K,
+	};
+	int idx = 0;
+	u64 count = 0;
+	u64 size;
+
+	drm_mm_init(&mm, 0, U64_MAX);
+	mm.color_adjust = i915_page_color_adjust;
+
+	/* Running out of memory is okay. */
+
+	for_each_prime_number_from(size, 0, U64_MAX) {
+		node = kzalloc(sizeof(*node), GFP_KERNEL);
+		if (!node) {
+			pr_info("finished test early, unable to allocate node, count=%llu\n", count);
+			break;
+		}
+
+		size = roundup(size, page_colors[idx]);
+
+		if (drm_mm_insert_node_in_range(&mm, node, size,
+						page_colors[idx],
+						page_colors[idx],
+						0, U64_MAX,
+						DRM_MM_INSERT_BEST)) {
+			pr_info("test finished, unable to insert node: color=%lu, size=%llx, count=%llu\n",
+				page_colors[idx], size, count);
+			kfree(node);
+			break;
+		}
+
+		GEM_BUG_ON(!IS_ALIGNED(node->start, node->color));
+		GEM_BUG_ON(!IS_ALIGNED(node->size, node->color));
+
+		/* We can't mix 4K and 64K pte's in the same pt. */
+
+		prev = list_prev_entry(node, node_list);
+		if (i915_color_differs(prev, node->color))
+			GEM_BUG_ON(prev->start >> GEN8_PDE_SHIFT ==
+				   node->start >> GEN8_PDE_SHIFT);
+
+		next = list_next_entry(node, node_list);
+		if (i915_color_differs(next, node->color))
+			GEM_BUG_ON(((next->start + next->size) >> GEN8_PDE_SHIFT) ==
+				   ((node->start + node->size) >> GEN8_PDE_SHIFT));
+
+		idx ^= 1;
+		++count;
+	}
+
+	drm_mm_for_each_node_safe(node, next, &mm) {
+		drm_mm_remove_node(node);
+		kfree(node);
+	}
+
+	drm_mm_takedown(&mm);
+
+	return 0;
+}
+
 static int igt_gtt_insert(void *arg)
 {
 	struct drm_i915_private *i915 = arg;

@@ -1522,6 +1589,7 @@ int i915_gem_gtt_mock_selftests(void)
 		SUBTEST(igt_mock_fill),
 		SUBTEST(igt_gtt_reserve),
 		SUBTEST(igt_gtt_insert),
+		SUBTEST(igt_ppgtt_page_color),
 	};
 	struct drm_i915_private *i915;
 	int err;

index 9b2a7228a78f..4625f93a3890 100644
@@ -92,12 +92,14 @@ static const struct drm_i915_gem_object_ops fake_ops = {
 };
 
 static struct drm_i915_gem_object *
-fake_dma_object(struct drm_i915_private *i915, u64 size)
+fake_dma_object(struct drm_i915_private *i915, u64 size, unsigned long page_size)
 {
 	struct drm_i915_gem_object *obj;
 
 	GEM_BUG_ON(!size);
-	GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE));
+	GEM_BUG_ON(page_size & ~I915_GTT_PAGE_SIZE_MASK);
+
+	size = roundup(size, page_size);
 
 	if (overflows_type(size, obj->base.size))
 		return ERR_PTR(-E2BIG);

@@ -107,8 +109,13 @@ fake_dma_object(struct drm_i915_private *i915, u64 size)
 		goto err;
 
 	drm_gem_private_object_init(&i915->drm, &obj->base, size);
+
 	i915_gem_object_init(obj, &fake_ops);
 
+	obj->page_size = page_size;
+
+	GEM_BUG_ON(!IS_ALIGNED(obj->base.size, obj->page_size));
+
 	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
 	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
 	obj->cache_level = I915_CACHE_NONE;

@@ -194,13 +201,14 @@ static int igt_ppgtt_alloc(void *arg)
 static int lowlevel_hole(struct drm_i915_private *i915,
 			 struct i915_address_space *vm,
 			 u64 hole_start, u64 hole_end,
+			 unsigned long page_size,
 			 unsigned long end_time)
 {
 	I915_RND_STATE(seed_prng);
 	unsigned int size;
 
 	/* Keep creating larger objects until one cannot fit into the hole */
-	for (size = 12; (hole_end - hole_start) >> size; size++) {
+	for (size = ilog2(page_size); (hole_end - hole_start) >> size; size++) {
 		I915_RND_SUBSTATE(prng, seed_prng);
 		struct drm_i915_gem_object *obj;
 		unsigned int *order, count, n;

@@ -226,7 +234,7 @@ static int lowlevel_hole(struct drm_i915_private *i915,
 		 * memory. We expect to hit -ENOMEM.
 		 */
 
-		obj = fake_dma_object(i915, BIT_ULL(size));
+		obj = fake_dma_object(i915, BIT_ULL(size), page_size);
 		if (IS_ERR(obj)) {
 			kfree(order);
 			break;

@@ -303,18 +311,25 @@ static void close_object_list(struct list_head *objects,
 static int fill_hole(struct drm_i915_private *i915,
 		     struct i915_address_space *vm,
 		     u64 hole_start, u64 hole_end,
+		     unsigned long page_size,
 		     unsigned long end_time)
 {
 	const u64 hole_size = hole_end - hole_start;
 	struct drm_i915_gem_object *obj;
-	const unsigned long max_pages =
-		min_t(u64, ULONG_MAX - 1, hole_size/2 >> PAGE_SHIFT);
-	const unsigned long max_step = max(int_sqrt(max_pages), 2UL);
-	unsigned long npages, prime, flags;
+	const unsigned page_shift = ilog2(page_size);
+	unsigned long max_pages, max_step, npages, prime, flags;
 	struct i915_vma *vma;
 	LIST_HEAD(objects);
 	int err;
 
+	hole_start = roundup(hole_start, page_size);
+	hole_end = rounddown(hole_end, page_size);
+
+	GEM_BUG_ON(hole_start >= hole_end);
+
+	max_pages = min_t(u64, ULONG_MAX - 1, hole_size/2 >> page_shift);
+	max_step = max(int_sqrt(max_pages), 2UL);
+
 	/* Try binding many VMA working inwards from either edge */
 
 	flags = PIN_OFFSET_FIXED | PIN_USER;

@@ -323,7 +338,7 @@ static int fill_hole(struct drm_i915_private *i915,
 
 	for_each_prime_number_from(prime, 2, max_step) {
 		for (npages = 1; npages <= max_pages; npages *= prime) {
-			const u64 full_size = npages << PAGE_SHIFT;
+			const u64 full_size = npages << page_shift;
 			const struct {
 				const char *name;
 				u64 offset;

@@ -334,7 +349,7 @@ static int fill_hole(struct drm_i915_private *i915,
 				{ }
 			}, *p;
 
-			obj = fake_dma_object(i915, full_size);
+			obj = fake_dma_object(i915, full_size, page_size);
 			if (IS_ERR(obj))
 				break;
 

@@ -359,7 +374,7 @@ static int fill_hole(struct drm_i915_private *i915,
 						offset -= obj->base.size;
 					}
 
-					err = i915_vma_pin(vma, 0, 0, offset | flags);
+					err = i915_vma_pin(vma, 0, page_size, offset | flags);
 					if (err) {
 						pr_err("%s(%s) pin (forward) failed with err=%d on size=%lu pages (prime=%lu), offset=%llx\n",
 						       __func__, p->name, err, npages, prime, offset);

@@ -367,7 +382,7 @@ static int fill_hole(struct drm_i915_private *i915,
 					}
 
 					if (!drm_mm_node_allocated(&vma->node) ||
-					    i915_vma_misplaced(vma, 0, 0, offset | flags)) {
+					    i915_vma_misplaced(vma, 0, page_size, offset | flags)) {
 						pr_err("%s(%s) (forward) insert failed: vma.node=%llx + %llx [allocated? %d], expected offset %llx\n",
 						       __func__, p->name, vma->node.start, vma->node.size, drm_mm_node_allocated(&vma->node),
 						       offset);

@@ -397,7 +412,7 @@ static int fill_hole(struct drm_i915_private *i915,
 					}
 
 					if (!drm_mm_node_allocated(&vma->node) ||
-					    i915_vma_misplaced(vma, 0, 0, offset | flags)) {
+					    i915_vma_misplaced(vma, 0, page_size, offset | flags)) {
 						pr_err("%s(%s) (forward) moved vma.node=%llx + %llx, expected offset %llx\n",
 						       __func__, p->name, vma->node.start, vma->node.size,
 						       offset);

@@ -432,7 +447,7 @@ static int fill_hole(struct drm_i915_private *i915,
 						offset -= obj->base.size;
 					}
 
-					err = i915_vma_pin(vma, 0, 0, offset | flags);
+					err = i915_vma_pin(vma, 0, page_size, offset | flags);
 					if (err) {
 						pr_err("%s(%s) pin (backward) failed with err=%d on size=%lu pages (prime=%lu), offset=%llx\n",
 						       __func__, p->name, err, npages, prime, offset);

@@ -440,7 +455,7 @@ static int fill_hole(struct drm_i915_private *i915,
 					}
 
 					if (!drm_mm_node_allocated(&vma->node) ||
-					    i915_vma_misplaced(vma, 0, 0, offset | flags)) {
+					    i915_vma_misplaced(vma, 0, page_size, offset | flags)) {
 						pr_err("%s(%s) (backward) insert failed: vma.node=%llx + %llx [allocated? %d], expected offset %llx\n",
 						       __func__, p->name, vma->node.start, vma->node.size, drm_mm_node_allocated(&vma->node),
 						       offset);

@@ -470,7 +485,7 @@ static int fill_hole(struct drm_i915_private *i915,
 					}
 
 					if (!drm_mm_node_allocated(&vma->node) ||
-					    i915_vma_misplaced(vma, 0, 0, offset | flags)) {
+					    i915_vma_misplaced(vma, 0, page_size, offset | flags)) {
 						pr_err("%s(%s) (backward) moved vma.node=%llx + %llx [allocated? %d], expected offset %llx\n",
 						       __func__, p->name, vma->node.start, vma->node.size, drm_mm_node_allocated(&vma->node),
 						       offset);

@@ -514,11 +529,13 @@ static int fill_hole(struct drm_i915_private *i915,
 static int walk_hole(struct drm_i915_private *i915,
 		     struct i915_address_space *vm,
 		     u64 hole_start, u64 hole_end,
+		     unsigned long page_size,
 		     unsigned long end_time)
 {
 	const u64 hole_size = hole_end - hole_start;
+	const unsigned page_shift = ilog2(page_size);
 	const unsigned long max_pages =
-		min_t(u64, ULONG_MAX - 1, hole_size >> PAGE_SHIFT);
+		min_t(u64, ULONG_MAX - 1, hole_size >> page_shift);
 	unsigned long flags;
 	u64 size;
 

@@ -534,7 +551,7 @@ static int walk_hole(struct drm_i915_private *i915,
 		u64 addr;
 		int err = 0;
 
-		obj = fake_dma_object(i915, size << PAGE_SHIFT);
+		obj = fake_dma_object(i915, size << page_shift, page_size);
 		if (IS_ERR(obj))
 			break;
 

@@ -547,7 +564,7 @@ static int walk_hole(struct drm_i915_private *i915,
 		for (addr = hole_start;
 		     addr + obj->base.size < hole_end;
 		     addr += obj->base.size) {
-			err = i915_vma_pin(vma, 0, 0, addr | flags);
+			err = i915_vma_pin(vma, 0, page_size, addr | flags);
 			if (err) {
 				pr_err("%s bind failed at %llx + %llx [hole %llx- %llx] with err=%d\n",
 				       __func__, addr, vma->size,

@@ -557,7 +574,7 @@ static int walk_hole(struct drm_i915_private *i915,
 			i915_vma_unpin(vma);
 
 			if (!drm_mm_node_allocated(&vma->node) ||
-			    i915_vma_misplaced(vma, 0, 0, addr | flags)) {
+			    i915_vma_misplaced(vma, 0, page_size, addr | flags)) {
 				pr_err("%s incorrect at %llx + %llx\n",
 				       __func__, addr, vma->size);
 				err = -EINVAL;

@@ -595,6 +612,7 @@ static int walk_hole(struct drm_i915_private *i915,
 static int pot_hole(struct drm_i915_private *i915,
 		    struct i915_address_space *vm,
 		    u64 hole_start, u64 hole_end,
+		    unsigned long page_size,
 		    unsigned long end_time)
 {
 	struct drm_i915_gem_object *obj;

@@ -607,7 +625,7 @@ static int pot_hole(struct drm_i915_private *i915,
 	if (i915_is_ggtt(vm))
 		flags |= PIN_GLOBAL;
 
-	obj = i915_gem_object_create_internal(i915, 2 * I915_GTT_PAGE_SIZE);
+	obj = fake_dma_object(i915, 2 * page_size, page_size);
 	if (IS_ERR(obj))
 		return PTR_ERR(obj);
 

@@ -619,15 +637,15 @@ static int pot_hole(struct drm_i915_private *i915,
 
 	/* Insert a pair of pages across every pot boundary within the hole */
 	for (pot = fls64(hole_end - 1) - 1;
-	     pot > ilog2(2 * I915_GTT_PAGE_SIZE);
+	     pot > ilog2(2 * page_size);
 	     pot--) {
 		u64 step = BIT_ULL(pot);
 		u64 addr;
 
-		for (addr = round_up(hole_start + I915_GTT_PAGE_SIZE, step) - I915_GTT_PAGE_SIZE;
-		     addr <= round_down(hole_end - 2*I915_GTT_PAGE_SIZE, step) - I915_GTT_PAGE_SIZE;
+		for (addr = round_up(hole_start + page_size, step) - page_size;
+		     addr <= round_down(hole_end - 2*page_size, step) - page_size;
 		     addr += step) {
-			err = i915_vma_pin(vma, 0, 0, addr | flags);
+			err = i915_vma_pin(vma, 0, page_size, addr | flags);
 			if (err) {
 				pr_err("%s failed to pin object at %llx in hole [%llx - %llx], with err=%d\n",
 				       __func__,

@@ -671,6 +689,7 @@ static int pot_hole(struct drm_i915_private *i915,
 static int drunk_hole(struct drm_i915_private *i915,
 		      struct i915_address_space *vm,
 		      u64 hole_start, u64 hole_end,
+		      unsigned long page_size,
 		      unsigned long end_time)
 {
 	I915_RND_STATE(prng);

@@ -682,7 +701,7 @@ static int drunk_hole(struct drm_i915_private *i915,
 		flags |= PIN_GLOBAL;
 
 	/* Keep creating larger objects until one cannot fit into the hole */
-	for (size = 12; (hole_end - hole_start) >> size; size++) {
+	for (size = ilog2(page_size); (hole_end - hole_start) >> size; size++) {
 		struct drm_i915_gem_object *obj;
 		unsigned int *order, count, n;
 		struct i915_vma *vma;

@@ -706,7 +725,7 @@ static int drunk_hole(struct drm_i915_private *i915,
 		 * memory. We expect to hit -ENOMEM.
 		 */
 
-		obj = fake_dma_object(i915, BIT_ULL(size));
+		obj = fake_dma_object(i915, BIT_ULL(size), page_size);
 		if (IS_ERR(obj)) {
 			kfree(order);
 			break;

@@ -723,6 +742,8 @@ static int drunk_hole(struct drm_i915_private *i915,
 		for (n = 0; n < count; n++) {
 			u64 addr = hole_start + order[n] * BIT_ULL(size);
 
+			GEM_BUG_ON(!IS_ALIGNED(addr, page_size));
+
 			err = i915_vma_pin(vma, 0, 0, addr | flags);
 			if (err) {
 				pr_err("%s failed to pin object at %llx + %llx in hole [%llx - %llx], with err=%d\n",

@@ -734,7 +755,7 @@ static int drunk_hole(struct drm_i915_private *i915,
 			}
 
 			if (!drm_mm_node_allocated(&vma->node) ||
-			    i915_vma_misplaced(vma, 0, 0, addr | flags)) {
+			    i915_vma_misplaced(vma, 0, page_size, addr | flags)) {
 				pr_err("%s incorrect at %llx + %llx\n",
 				       __func__, addr, BIT_ULL(size));
 				i915_vma_unpin(vma);

@@ -771,11 +792,12 @@ static int drunk_hole(struct drm_i915_private *i915,
 static int __shrink_hole(struct drm_i915_private *i915,
 			 struct i915_address_space *vm,
 			 u64 hole_start, u64 hole_end,
+			 unsigned long page_size,
 			 unsigned long end_time)
 {
 	struct drm_i915_gem_object *obj;
 	unsigned long flags = PIN_OFFSET_FIXED | PIN_USER;
-	unsigned int order = 12;
+	unsigned int order = ilog2(page_size);
 	LIST_HEAD(objects);
 	int err = 0;
 	u64 addr;

@@ -786,7 +808,7 @@ static int __shrink_hole(struct drm_i915_private *i915,
 		u64 size = BIT_ULL(order++);
 
 		size = min(size, hole_end - addr);
-		obj = fake_dma_object(i915, size);
+		obj = fake_dma_object(i915, size, page_size);
 		if (IS_ERR(obj)) {
 			err = PTR_ERR(obj);
 			break;

@@ -802,7 +824,7 @@ static int __shrink_hole(struct drm_i915_private *i915,
 
 		GEM_BUG_ON(vma->size != size);
 
-		err = i915_vma_pin(vma, 0, 0, addr | flags);
+		err = i915_vma_pin(vma, 0, page_size, addr | flags);
 		if (err) {
 			pr_err("%s failed to pin object at %llx + %llx in hole [%llx - %llx], with err=%d\n",
 			       __func__, addr, size, hole_start, hole_end, err);

@@ -810,7 +832,7 @@ static int __shrink_hole(struct drm_i915_private *i915,
 		}
 
 		if (!drm_mm_node_allocated(&vma->node) ||
-		    i915_vma_misplaced(vma, 0, 0, addr | flags)) {
+		    i915_vma_misplaced(vma, 0, page_size, addr | flags)) {
 			pr_err("%s incorrect at %llx + %llx\n",
 			       __func__, addr, size);
 			i915_vma_unpin(vma);

@@ -837,6 +859,7 @@ static int __shrink_hole(struct drm_i915_private *i915,
 static int shrink_hole(struct drm_i915_private *i915,
 		       struct i915_address_space *vm,
 		       u64 hole_start, u64 hole_end,
+		       unsigned long page_size,
 		       unsigned long end_time)
 {
 	unsigned long prime;

@@ -847,7 +870,8 @@ static int shrink_hole(struct drm_i915_private *i915,
 
 	for_each_prime_number_from(prime, 0, ULONG_MAX - 1) {
 		vm->fault_attr.interval = prime;
-		err = __shrink_hole(i915, vm, hole_start, hole_end, end_time);
+		err = __shrink_hole(i915, vm, hole_start, hole_end, page_size,
+				    end_time);
 		if (err)
 			break;
 	}

@@ -861,12 +885,20 @@ static int exercise_ppgtt(struct drm_i915_private *dev_priv,
 			  int (*func)(struct drm_i915_private *i915,
 				      struct i915_address_space *vm,
 				      u64 hole_start, u64 hole_end,
+				      unsigned long page_size,
 				      unsigned long end_time))
 {
 	struct drm_file *file;
 	struct i915_hw_ppgtt *ppgtt;
 	IGT_TIMEOUT(end_time);
-	int err;
+	unsigned long page_sizes[] = {
+		I915_GTT_PAGE_SIZE,
+		I915_GTT_PAGE_SIZE_64K,
+		I915_GTT_PAGE_SIZE_2M,
+		I915_GTT_PAGE_SIZE_1G,
+	};
+	int err = 0;
+	int i;
 
 	if (!USES_FULL_PPGTT(dev_priv))
 		return 0;

@@ -884,7 +916,11 @@ static int exercise_ppgtt(struct drm_i915_private *dev_priv,
 	GEM_BUG_ON(offset_in_page(ppgtt->base.total));
 	GEM_BUG_ON(ppgtt->base.closed);
 
-	err = func(dev_priv, &ppgtt->base, 0, ppgtt->base.total, end_time);
+	for (i = 0; i < ARRAY_SIZE(page_sizes); ++i) {
+		if (INTEL_INFO(dev_priv)->page_size_mask & page_sizes[i])
+			err = func(dev_priv, &ppgtt->base, 0, ppgtt->base.total,
+				   page_sizes[i], end_time);
+	}
 
 	i915_ppgtt_close(&ppgtt->base);
 	i915_ppgtt_put(ppgtt);

@@ -940,6 +976,7 @@ static int exercise_ggtt(struct drm_i915_private *i915,
 			 int (*func)(struct drm_i915_private *i915,
 				     struct i915_address_space *vm,
 				     u64 hole_start, u64 hole_end,
+				     unsigned long page_size,
 				     unsigned long end_time))
 {
 	struct i915_ggtt *ggtt = &i915->ggtt;

@@ -961,7 +998,8 @@ static int exercise_ggtt(struct drm_i915_private *i915,
 		if (hole_start >= hole_end)
 			continue;
 
-		err = func(i915, &ggtt->base, hole_start, hole_end, end_time);
+		err = func(i915, &ggtt->base, hole_start, hole_end,
+			   I915_GTT_PAGE_SIZE, end_time);
 		if (err)
 			break;
 

@@ -1104,12 +1142,20 @@ static int exercise_mock(struct drm_i915_private *i915,
 			 int (*func)(struct drm_i915_private *i915,
 				     struct i915_address_space *vm,
 				     u64 hole_start, u64 hole_end,
+				     unsigned long page_size,
 				     unsigned long end_time))
 {
 	struct i915_gem_context *ctx;
 	struct i915_hw_ppgtt *ppgtt;
 	IGT_TIMEOUT(end_time);
+	unsigned long page_sizes[] = {
+		I915_GTT_PAGE_SIZE,
+		I915_GTT_PAGE_SIZE_64K,
+		I915_GTT_PAGE_SIZE_2M,
+		I915_GTT_PAGE_SIZE_1G,
+	};
 	int err;
+	int i;
 
 	ctx = mock_context(i915, "mock");
 	if (!ctx)

@@ -1118,7 +1164,10 @@ static int exercise_mock(struct drm_i915_private *i915,
 	ppgtt = ctx->ppgtt;
 	GEM_BUG_ON(!ppgtt);
 
-	err = func(i915, &ppgtt->base, 0, ppgtt->base.total, end_time);
+	for (i = 0; i < ARRAY_SIZE(page_sizes); ++i) {
+		err = func(i915, &ppgtt->base, 0, ppgtt->base.total,
+			   page_sizes[i], end_time);
+	}
 
 	mock_context_close(ctx);
 	return err;

index a2137100d2f5..9428ea09d05d 100644
@@ -88,6 +88,9 @@ mock_ppgtt(struct drm_i915_private *i915,
 	ppgtt->base.unbind_vma = mock_unbind_ppgtt;
 	ppgtt->base.cleanup = mock_cleanup;
 
+	/* For mock testing huge-page support */
+	ppgtt->base.mm.color_adjust = i915_page_color_adjust;
+
 	return ppgtt;
 }