Skip to contentSkip to conversation

Explicit fencing on multiple timelines, again

PATCH In review Chris Wilson 5 participants 107 messages Oct 7, 2016 – Oct 17, 2016 Page 1 of 2

Subsystems (17)
lore
Share & tools
lore
#0 lore raw

After a hiatus to fix up some fallout from the reset work, here we are again. Biggest features needing review are the changes to memory management to allow deferred freeing to prevent recursion whilst unbinding the active object, and the interaction with atomic modesets.
-Chris

_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx

#1 lore raw

PATCH 1/42 drm/i915: Allow disabling error capture

We currently capture the GPU state after we detect a hang. This is vital for us to both triage and debug hangs in the wild (post-mortem debugging). However, it comes at the cost of running some potentially dangerous code (since it has to make very few assumption about the state of the driver) that is quite resource intensive.

---
 drivers/gpu/drm/i915/Kconfig          | 10 ++++++++++
 drivers/gpu/drm/i915/i915_debugfs.c   |  6 ++++++
 drivers/gpu/drm/i915/i915_drv.h       | 16 ++++++++++++++++
 drivers/gpu/drm/i915/i915_gpu_error.c |  7 +++++++
 drivers/gpu/drm/i915/i915_params.c    |  9 +++++++++
 drivers/gpu/drm/i915/i915_params.h    |  1 +
 drivers/gpu/drm/i915/i915_sysfs.c     |  8 ++++++++
 drivers/gpu/drm/i915/intel_display.c  |  4 ++++
 drivers/gpu/drm/i915/intel_overlay.c  |  4 ++++
 9 files changed, 65 insertions(+)
drivers/gpu/drm/i915/Kconfig +10 -0
index 7769e469118f..10a6ac11b6a9 100644
@@ -46,6 +46,16 @@ config DRM_I915_PRELIMINARY_HW_SUPPORT
 
 	  If in doubt, say "N".
 
+config DRM_I915_CAPTURE_ERROR
+	bool "Enable capturing GPU state following a hang"
+	depends on DRM_I915
+	default y
+	help
+	  This option enables capturing the GPU state when a hang is detected.
+	  This information is vital for triaging hangs and assists in debugging.
+
+	  If in doubt, say "Y".
+
 config DRM_I915_USERPTR
 	bool "Always enable userptr support"
 	depends on DRM_I915
drivers/gpu/drm/i915/i915_debugfs.c +6 -0
index 20689f1cd719..e4b5ba771bea 100644
@@ -960,6 +960,8 @@ static int i915_hws_info(struct seq_file *m, void *data)
 	return 0;
 }
 
+#ifdef CONFIG_DRM_I915_CAPTURE_ERROR
+
 static ssize_t
 i915_error_state_write(struct file *filp,
 		       const char __user *ubuf,
@@ -1042,6 +1044,8 @@ static const struct file_operations i915_error_state_fops = {
 	.release = i915_error_state_release,
 };
 
+#endif
+
 static int
 i915_next_seqno_get(void *data, u64 *val)
 {
@@ -5398,7 +5402,9 @@ static const struct i915_debugfs_files {
 	{"i915_ring_missed_irq", &i915_ring_missed_irq_fops},
 	{"i915_ring_test_irq", &i915_ring_test_irq_fops},
 	{"i915_gem_drop_caches", &i915_drop_caches_fops},
+#ifdef CONFIG_DRM_I915_CAPTURE_ERROR
 	{"i915_error_state", &i915_error_state_fops},
+#endif
 	{"i915_next_seqno", &i915_next_seqno_fops},
 	{"i915_display_crc_ctl", &i915_display_crc_ctl_fops},
 	{"i915_pri_wm_latency", &i915_pri_wm_latency_fops},
drivers/gpu/drm/i915/i915_drv.h +16 -0
index a219a3534750..e24f9ccf4467 100644
@@ -3544,6 +3544,8 @@ static inline void intel_display_crc_init(struct drm_i915_private *dev_priv) {}
 #endif
 
 /* i915_gpu_error.c */
+#ifdef CONFIG_DRM_I915_CAPTURE_ERROR
+
 __printf(2, 3)
 void i915_error_printf(struct drm_i915_error_state_buf *e, const char *f, ...);
 int i915_error_state_to_str(struct drm_i915_error_state_buf *estr,
@@ -3564,6 +3566,20 @@ void i915_error_state_get(struct drm_device *dev,
 void i915_error_state_put(struct i915_error_state_file_priv *error_priv);
 void i915_destroy_error_state(struct drm_device *dev);
 
+#else
+
+static inline void i915_capture_error_state(struct drm_i915_private *dev_priv,
+					    u32 engine_mask,
+					    const char *error_msg)
+{
+}
+
+static inline void i915_destroy_error_state(struct drm_device *dev)
+{
+}
+
+#endif
+
 void i915_get_engine_instdone(struct drm_i915_private *dev_priv,
 			      enum intel_engine_id engine_id,
 			      struct intel_instdone *instdone);
drivers/gpu/drm/i915/i915_gpu_error.c +7 -0
index b5b58692ac5a..9b395ffa3b6a 100644
@@ -30,6 +30,8 @@
 #include <generated/utsrelease.h>
 #include "i915_drv.h"
 
+#ifdef CONFIG_DRM_I915_CAPTURE_ERROR
+
 static const char *engine_str(int engine)
 {
 	switch (engine) {
@@ -1464,6 +1466,9 @@ void i915_capture_error_state(struct drm_i915_private *dev_priv,
 	struct drm_i915_error_state *error;
 	unsigned long flags;
 
+	if (!i915.error_capture)
+		return;
+
 	if (READ_ONCE(dev_priv->gpu_error.first_error))
 		return;
 
@@ -1549,6 +1554,8 @@ void i915_destroy_error_state(struct drm_device *dev)
 		kref_put(&error->ref, i915_error_state_free);
 }
 
+#endif
+
 const char *i915_cache_level_str(struct drm_i915_private *i915, int type)
 {
 	switch (type) {
drivers/gpu/drm/i915/i915_params.c +9 -0
index 768ad89d9cd4..e72a41223535 100644
@@ -47,6 +47,7 @@ struct i915_params i915 __read_mostly = {
 	.load_detect_test = 0,
 	.force_reset_modeset_test = 0,
 	.reset = true,
+	.error_capture = true,
 	.invert_brightness = 0,
 	.disable_display = 0,
 	.enable_cmd_parser = 1,
@@ -115,6 +116,14 @@ MODULE_PARM_DESC(vbt_sdvo_panel_type,
 module_param_named_unsafe(reset, i915.reset, bool, 0600);
 MODULE_PARM_DESC(reset, "Attempt GPU resets (default: true)");
 
+#ifdef CONFIG_DRM_I915_CAPTURE_ERROR
+module_param_named(error_capture, i915.error_capture, bool, 0600);
+MODULE_PARM_DESC(error_capture,
+	"Record the GPU state following a hang. "
+	"This information in /sys/class/drm/card<N>/error is vital for "
+	"triaging and debugging hangs.");
+#endif
+
 module_param_named_unsafe(enable_hangcheck, i915.enable_hangcheck, bool, 0644);
 MODULE_PARM_DESC(enable_hangcheck,
 	"Periodically check GPU activity for detecting hangs. "
drivers/gpu/drm/i915/i915_params.h +1 -0
index 3a0dd78ddb38..94efc899c1ef 100644
@@ -59,6 +59,7 @@ struct i915_params {
 	bool load_detect_test;
 	bool force_reset_modeset_test;
 	bool reset;
+	bool error_capture;
 	bool disable_display;
 	bool verbose_state_checks;
 	bool nuclear_pageflip;
drivers/gpu/drm/i915/i915_sysfs.c +8 -0
index 1012eeea1324..c9b71f676a57 100644
@@ -514,6 +514,8 @@ static const struct attribute *vlv_attrs[] = {
 	NULL,
 };
 
+#ifdef CONFIG_DRM_I915_CAPTURE_ERROR
+
 static ssize_t error_state_read(struct file *filp, struct kobject *kobj,
 				struct bin_attribute *attr, char *buf,
 				loff_t off, size_t count)
@@ -571,6 +573,8 @@ static struct bin_attribute error_state_attr = {
 	.write = error_state_write,
 };
 
+#endif
+
 void i915_setup_sysfs(struct drm_i915_private *dev_priv)
 {
 	struct device *kdev = dev_priv->drm.primary->kdev;
@@ -617,17 +621,21 @@ void i915_setup_sysfs(struct drm_i915_private *dev_priv)
 	if (ret)
 		DRM_ERROR("RPS sysfs setup failed\n");
 
+#ifdef CONFIG_DRM_I915_CAPTURE_ERROR
 	ret = sysfs_create_bin_file(&kdev->kobj,
 				    &error_state_attr);
 	if (ret)
 		DRM_ERROR("error_state sysfs setup failed\n");
+#endif
 }
 
 void i915_teardown_sysfs(struct drm_i915_private *dev_priv)
 {
 	struct device *kdev = dev_priv->drm.primary->kdev;
 
+#ifdef CONFIG_DRM_I915_CAPTURE_ERROR
 	sysfs_remove_bin_file(&kdev->kobj, &error_state_attr);
+#endif
 	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
 		sysfs_remove_files(&kdev->kobj, vlv_attrs);
 	else
drivers/gpu/drm/i915/intel_display.c +4 -0
index a366656bcec5..4b3d315c5254 100644
@@ -17096,6 +17096,8 @@ int intel_modeset_vga_set_state(struct drm_device *dev, bool state)
 	return 0;
 }
 
+#ifdef CONFIG_DRM_I915_CAPTURE_ERROR
+
 struct intel_display_error_state {
 
 	u32 power_well_driver;
@@ -17278,3 +17280,5 @@ intel_display_print_error_state(struct drm_i915_error_state_buf *m,
 		err_printf(m, "  VSYNC: %08x\n", error->transcoder[i].vsync);
 	}
 }
+
+#endif
drivers/gpu/drm/i915/intel_overlay.c +4 -0
index a24bc8c7889f..7c392547711f 100644
@@ -1470,6 +1470,8 @@ void intel_cleanup_overlay(struct drm_i915_private *dev_priv)
 	kfree(dev_priv->overlay);
 }
 
+#ifdef CONFIG_DRM_I915_CAPTURE_ERROR
+
 struct intel_overlay_error_state {
 	struct overlay_registers regs;
 	unsigned long base;
@@ -1587,3 +1589,5 @@ intel_overlay_print_error_state(struct drm_i915_error_state_buf *m,
 	P(UVSCALEV);
 #undef P
 }
+
+#endif
Signature · 6 lines
2.9.3

_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
#2 lore raw

PATCH 2/42 drm/i915: Stop the machine whilst capturing the GPU crash dump

In reply to Chris Wilson

The error state is purposefully racy as we expect it to be called at any time and so have avoided any locking whilst capturing the crash dump. However, with multi-engine GPUs and multiple CPUs, those races can manifest into OOPSes as we attempt to chase dangling pointers freed on other CPUs. Under discussion are lots of ways to slow down normal operation in order to protect the post-mortem error capture, but what it we take the opposite approach and freeze the machine whilst the error capture runs (note the GPU may still running, but as long as we don't process any of the results the driver's bookkeeping will be static).

Note that by of itself, this is not a complete fix. It also depends on the compiler barriers in list_add/list_del to prevent traversing the lists into the void. We also depend that we only require state from carefully controlled sources - i.e. all the state we require for post-mortem debugging should be reachable from the request itself so that we only have to worry about retrieving the request carefully. Once we have the request, we know that all pointers from it are intact.

v2: Avoid drm_clflush_pages() inside stop_machine() as it may use stop_machine() itself for its wbinvd fallback.

---
 drivers/gpu/drm/i915/Kconfig          |  1 +
 drivers/gpu/drm/i915/i915_drv.h       |  2 ++
 drivers/gpu/drm/i915/i915_gpu_error.c | 46 +++++++++++++++++++++--------------
 3 files changed, 31 insertions(+), 18 deletions(-)
drivers/gpu/drm/i915/Kconfig +1 -0
index 10a6ac11b6a9..0f46a9c04c0e 100644
@@ -4,6 +4,7 @@ config DRM_I915
 	depends on X86 && PCI
 	select INTEL_GTT
 	select INTERVAL_TREE
+	select STOP_MACHINE
 	# we need shmfs for the swappable backing store, and in particular
 	# the shmem_readpage() which depends upon tmpfs
 	select SHMEM
drivers/gpu/drm/i915/i915_drv.h +2 -0
index e24f9ccf4467..303ffb7c882a 100644
@@ -746,6 +746,8 @@ struct drm_i915_error_state {
 	struct kref ref;
 	struct timeval time;
 
+	struct drm_i915_private *i915;
+
 	char error_msg[128];
 	bool simulated;
 	int iommu;
drivers/gpu/drm/i915/i915_gpu_error.c +28 -18

87 lines not on this page · raw message

Signature · 6 lines
2.9.3

_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
maintainer Reviewed-by on v1 2/42
#3 lore raw

Re: [PATCH 02/42] drm/i915: Stop the machine whilst capturing the GPU crash dump

Quoted 18 lines · Chris Wilson

The error state is purposefully racy as we expect it to be called at any time and so have avoided any locking whilst capturing the crash dump. However, with multi-engine GPUs and multiple CPUs, those races can manifest into OOPSes as we attempt to chase dangling pointers freed on other CPUs. Under discussion are lots of ways to slow down normal operation in order to protect the post-mortem error capture, but what it we take the opposite approach and freeze the machine whilst the error capture runs (note the GPU may still running, but as long as we don't process any of the results the driver's bookkeeping will be static).

Note that by of itself, this is not a complete fix. It also depends on the compiler barriers in list_add/list_del to prevent traversing the lists into the void. We also depend that we only require state from carefully controlled sources - i.e. all the state we require for post-mortem debugging should be reachable from the request itself so that we only have to worry about retrieving the request carefully. Once we have the request, we know that all pointers from it are intact.

Quoted 4 lines

v2: Avoid drm_clflush_pages() inside stop_machine() as it may use stop_machine() itself for its wbinvd fallback.

CC'in Daniel to add A-b.

Regards, Joonas

Signature · 7 lines
Joonas Lahtinen
Open Source Technology Center
Intel Corporation
_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
#4 lore raw

PATCH 3/42 drm/i915: Always use the GTT for error capture

In reply to Chris Wilson

Since the GTT provides universal access to any GPU page, we can use it to reduce our plethora of read methods to just one. It also has the important characteristic of being exactly what the GPU sees - if there are incoherency problems, seeing the batch as executed (rather than as trapped inside the cpu cache) is important.

---
 drivers/gpu/drm/i915/i915_gem_gtt.c   |  43 ++++++++----
 drivers/gpu/drm/i915/i915_gem_gtt.h   |   2 +
 drivers/gpu/drm/i915/i915_gpu_error.c | 120 ++++++++++++----------------------
 3 files changed, 74 insertions(+), 91 deletions(-)
drivers/gpu/drm/i915/i915_gem_gtt.c +30 -13
index 0bb4232f66bc..2d846aa39ca5 100644
@@ -2717,6 +2717,7 @@ int i915_gem_init_ggtt(struct drm_i915_private *dev_priv)
 	 */
 	struct i915_ggtt *ggtt = &dev_priv->ggtt;
 	unsigned long hole_start, hole_end;
+	struct i915_hw_ppgtt *ppgtt;
 	struct drm_mm_node *entry;
 	int ret;
 
@@ -2724,6 +2725,15 @@ int i915_gem_init_ggtt(struct drm_i915_private *dev_priv)
 	if (ret)
 		return ret;
 
+	/* Reserve a mappable slot for our lockless error capture */
+	ret = drm_mm_insert_node_in_range_generic(&ggtt->base.mm,
+						  &ggtt->error_capture,
+						  4096, 0, -1,
+						  0, ggtt->mappable_end,
+						  0, 0);
+	if (ret)
+		return ret;
+
 	/* Clear any non-preallocated blocks */
 	drm_mm_for_each_hole(entry, &ggtt->base.mm, hole_start, hole_end) {
 		DRM_DEBUG_KMS("clearing unused GTT space: [%lx, %lx]\n",
@@ -2738,25 +2748,21 @@ int i915_gem_init_ggtt(struct drm_i915_private *dev_priv)
 			       true);
 
 	if (USES_PPGTT(dev_priv) && !USES_FULL_PPGTT(dev_priv)) {
-		struct i915_hw_ppgtt *ppgtt;
-
 		ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);
-		if (!ppgtt)
-			return -ENOMEM;
+		if (!ppgtt) {
+			ret = -ENOMEM;
+			goto err;
+		}
 
 		ret = __hw_ppgtt_init(ppgtt, dev_priv);
-		if (ret) {
-			kfree(ppgtt);
-			return ret;
-		}
+		if (ret)
+			goto err_ppgtt;
 
-		if (ppgtt->base.allocate_va_range)
+		if (ppgtt->base.allocate_va_range) {
 			ret = ppgtt->base.allocate_va_range(&ppgtt->base, 0,
 							    ppgtt->base.total);
-		if (ret) {
-			ppgtt->base.cleanup(&ppgtt->base);
-			kfree(ppgtt);
-			return ret;
+			if (ret)
+				goto err_ppgtt_cleanup;
 		}
 
 		ppgtt->base.clear_range(&ppgtt->base,
@@ -2770,6 +2776,14 @@ int i915_gem_init_ggtt(struct drm_i915_private *dev_priv)
 	}
 
 	return 0;
+
+err_ppgtt_cleanup:
+	ppgtt->base.cleanup(&ppgtt->base);
+err_ppgtt:
+	kfree(ppgtt);
+err:
+	drm_mm_remove_node(&ggtt->error_capture);
+	return ret;
 }
 
 /**
@@ -2788,6 +2802,9 @@ void i915_ggtt_cleanup_hw(struct drm_i915_private *dev_priv)
 
 	i915_gem_cleanup_stolen(&dev_priv->drm);
 
+	if (drm_mm_node_allocated(&ggtt->error_capture))
+		drm_mm_remove_node(&ggtt->error_capture);
+
 	if (drm_mm_initialized(&ggtt->base.mm)) {
 		intel_vgt_deballoon(dev_priv);
 
drivers/gpu/drm/i915/i915_gem_gtt.h +2 -0
index ec78be2f8c77..bd93fb8f99d2 100644
@@ -450,6 +450,8 @@ struct i915_ggtt {
 	bool do_idle_maps;
 
 	int mtrr;
+
+	struct drm_mm_node error_capture;
 };
 
 struct i915_hw_ppgtt {
drivers/gpu/drm/i915/i915_gpu_error.c +42 -78

172 lines not on this page · raw message

Signature · 6 lines
2.9.3

_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
#5 lore raw

PATCH 4/42 drm/i915: Consolidate error object printing

In reply to Chris Wilson

Leave all the pretty printing to userspace and simplify the error capture to only have a single common object printer. It makes the kernel code more compact, and the refactoring allows us to apply more complex transformations like compressing the output.

---
 drivers/gpu/drm/i915/i915_gpu_error.c | 100 +++++++++-------------------------
 1 file changed, 25 insertions(+), 75 deletions(-)
drivers/gpu/drm/i915/i915_gpu_error.c +25 -75
index be82eadac9bb..bf2498297341 100644
@@ -343,10 +343,22 @@ void i915_error_printf(struct drm_i915_error_state_buf *e, const char *f, ...)
 }
 
 static void print_error_obj(struct drm_i915_error_state_buf *m,
+			    struct intel_engine_cs *engine,
+			    const char *name,
 			    struct drm_i915_error_object *obj)
 {
 	int page, offset, elt;
 
+	if (!obj)
+		return;
+
+	if (name) {
+		err_printf(m, "%s --- %s = 0x%08x %08x\n",
+			   engine ? engine->name : "global", name,
+			   upper_32_bits(obj->gtt_offset),
+			   lower_32_bits(obj->gtt_offset));
+	}
+
 	for (page = offset = 0; page < obj->page_count; page++) {
 		for (elt = 0; elt < PAGE_SIZE/4; elt++) {
 			err_printf(m, "%08x :  %08x\n", offset,
@@ -372,8 +384,8 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m,
 	struct pci_dev *pdev = dev_priv->drm.pdev;
 	struct drm_i915_error_state *error = error_priv->error;
 	struct drm_i915_error_object *obj;
-	int i, j, offset, elt;
 	int max_hangcheck_score;
+	int i, j;
 
 	if (!error) {
 		err_printf(m, "no error state collected\n");
@@ -493,15 +505,7 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m,
 			err_printf(m, " --- gtt_offset = 0x%08x %08x\n",
 				   upper_32_bits(obj->gtt_offset),
 				   lower_32_bits(obj->gtt_offset));
-			print_error_obj(m, obj);
-		}
-
-		obj = ee->wa_batchbuffer;
-		if (obj) {
-			err_printf(m, "%s (w/a) --- gtt_offset = 0x%08x\n",
-				   dev_priv->engine[i].name,
-				   lower_32_bits(obj->gtt_offset));
-			print_error_obj(m, obj);
+			print_error_obj(m, &dev_priv->engine[i], NULL, obj);
 		}
 
 		if (ee->num_requests) {
@@ -533,77 +537,23 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m,
 			}
 		}
 
-		if ((obj = ee->ringbuffer)) {
-			err_printf(m, "%s --- ringbuffer = 0x%08x\n",
-				   dev_priv->engine[i].name,
-				   lower_32_bits(obj->gtt_offset));
-			print_error_obj(m, obj);
-		}
+		print_error_obj(m, &dev_priv->engine[i],
+				"ringbuffer", ee->ringbuffer);
 
-		if ((obj = ee->hws_page)) {
-			u64 hws_offset = obj->gtt_offset;
-			u32 *hws_page = &obj->pages[0][0];
+		print_error_obj(m, &dev_priv->engine[i],
+				"HW Status", ee->hws_page);
 
-			if (i915.enable_execlists) {
-				hws_offset += LRC_PPHWSP_PN * PAGE_SIZE;
-				hws_page = &obj->pages[LRC_PPHWSP_PN][0];
-			}
-			err_printf(m, "%s --- HW Status = 0x%08llx\n",
-				   dev_priv->engine[i].name, hws_offset);
-			offset = 0;
-			for (elt = 0; elt < PAGE_SIZE/16; elt += 4) {
-				err_printf(m, "[%04x] %08x %08x %08x %08x\n",
-					   offset,
-					   hws_page[elt],
-					   hws_page[elt+1],
-					   hws_page[elt+2],
-					   hws_page[elt+3]);
-				offset += 16;
-			}
-		}
+		print_error_obj(m, &dev_priv->engine[i],
+				"HW context", ee->ctx);
 
-		obj = ee->wa_ctx;
-		if (obj) {
-			u64 wa_ctx_offset = obj->gtt_offset;
-			u32 *wa_ctx_page = &obj->pages[0][0];
-			struct intel_engine_cs *engine = &dev_priv->engine[RCS];
-			u32 wa_ctx_size = (engine->wa_ctx.indirect_ctx.size +
-					   engine->wa_ctx.per_ctx.size);
-
-			err_printf(m, "%s --- WA ctx batch buffer = 0x%08llx\n",
-				   dev_priv->engine[i].name, wa_ctx_offset);
-			offset = 0;
-			for (elt = 0; elt < wa_ctx_size; elt += 4) {
-				err_printf(m, "[%04x] %08x %08x %08x %08x\n",
-					   offset,
-					   wa_ctx_page[elt + 0],
-					   wa_ctx_page[elt + 1],
-					   wa_ctx_page[elt + 2],
-					   wa_ctx_page[elt + 3]);
-				offset += 16;
-			}
-		}
+		print_error_obj(m, &dev_priv->engine[i],
+				"WA context", ee->wa_ctx);
 
-		if ((obj = ee->ctx)) {
-			err_printf(m, "%s --- HW Context = 0x%08x\n",
-				   dev_priv->engine[i].name,
-				   lower_32_bits(obj->gtt_offset));
-			print_error_obj(m, obj);
-		}
+		print_error_obj(m, &dev_priv->engine[i],
+				"WA batchbuffer", ee->wa_batchbuffer);
 	}
 
-	if ((obj = error->semaphore)) {
-		err_printf(m, "Semaphore page = 0x%08x\n",
-			   lower_32_bits(obj->gtt_offset));
-		for (elt = 0; elt < PAGE_SIZE/16; elt += 4) {
-			err_printf(m, "[%04x] %08x %08x %08x %08x\n",
-				   elt * 4,
-				   obj->pages[0][elt],
-				   obj->pages[0][elt+1],
-				   obj->pages[0][elt+2],
-				   obj->pages[0][elt+3]);
-		}
-	}
+	print_error_obj(m, NULL, "Semaphores", error->semaphore);
 
 	if (error->overlay)
 		intel_overlay_print_error_state(m, error->overlay);
Signature · 6 lines
2.9.3

_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
#6 lore raw

PATCH 5/42 drm/i915: Compress GPU objects in error state

In reply to Chris Wilson

Our error states are quickly growing, pinning kernel memory with them. The majority of the space is taken up by the error objects. These compress well using zlib and without decode are mostly meaningless, so encoding them does not hinder quickly parsing the error state for familiarity.

v2: Make the zlib dependency optional

---
 drivers/gpu/drm/i915/Kconfig          |  12 +++
 drivers/gpu/drm/i915/i915_drv.h       |   3 +-
 drivers/gpu/drm/i915/i915_gpu_error.c | 176 ++++++++++++++++++++++++++++++----
 3 files changed, 169 insertions(+), 22 deletions(-)
drivers/gpu/drm/i915/Kconfig +12 -0
index 0f46a9c04c0e..92ecced1bc8f 100644
@@ -57,6 +57,18 @@ config DRM_I915_CAPTURE_ERROR
 
 	  If in doubt, say "Y".
 
+config DRM_I915_COMPRESS_ERROR
+	bool "Compress GPU error state"
+	depends on DRM_I915_CAPTURE_ERROR
+	select ZLIB_DEFLATE
+	default y
+	help
+	  This option selects ZLIB_DEFLATE if it isn't already
+	  selected and causes any error state captured upon a GPU hang
+	  to be compressed using zlib.
+
+	  If in doubt, say "Y".
+
 config DRM_I915_USERPTR
 	bool "Always enable userptr support"
 	depends on DRM_I915
drivers/gpu/drm/i915/i915_drv.h +2 -1
index 303ffb7c882a..54b1b931573c 100644
@@ -820,9 +820,10 @@ struct drm_i915_error_state {
 		struct intel_instdone instdone;
 
 		struct drm_i915_error_object {
-			int page_count;
 			u64 gtt_offset;
 			u64 gtt_size;
+			int page_count;
+			int unused;
 			u32 *pages[0];
 		} *ringbuffer, *batchbuffer, *wa_batchbuffer, *ctx, *hws_page;
 
drivers/gpu/drm/i915/i915_gpu_error.c +155 -21

243 lines not on this page · raw message

Signature · 6 lines
2.9.3

_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
#7 lore raw

PATCH 6/42 drm/i915: Support asynchronous waits on struct fence from i915_gem_request

In reply to Chris Wilson

We will need to wait on DMA completion (as signaled via struct fence) before executing our i915_gem_request. Therefore we want to expose a method for adding the await on the fence itself to the request.

---
 drivers/gpu/drm/i915/i915_gem_request.c | 40 +++++++++++++++++++++++++++++++++
 drivers/gpu/drm/i915/i915_gem_request.h |  2 ++
 2 files changed, 42 insertions(+)
drivers/gpu/drm/i915/i915_gem_request.c +40 -0
index 8832f8ec1583..e1f7a32d4080 100644
@@ -23,6 +23,7 @@
  */
 
 #include <linux/prefetch.h>
+#include <linux/fence-array.h>
 
 #include "i915_drv.h"
 
@@ -495,6 +496,45 @@ i915_gem_request_await_request(struct drm_i915_gem_request *to,
 	return 0;
 }
 
+int
+i915_gem_request_await_fence(struct drm_i915_gem_request *req,
+			     struct fence *fence)
+{
+	struct fence_array *array;
+	int ret;
+	int i;
+
+	if (test_bit(FENCE_FLAG_SIGNALED_BIT, &fence->flags))
+		return 0;
+
+	if (fence_is_i915(fence))
+		return i915_gem_request_await_request(req, to_request(fence));
+
+	if (!fence_is_array(fence)) {
+		ret = i915_sw_fence_await_dma_fence(&req->submit,
+						    fence, 10*HZ,
+						    GFP_KERNEL);
+		return ret < 0 ? ret : 0;
+	}
+
+	array = to_fence_array(fence);
+	for (i = 0; i < array->num_fences; i++) {
+		struct fence *child = array->fences[i];
+
+		if (fence_is_i915(child))
+			ret = i915_gem_request_await_request(req,
+							     to_request(child));
+		else
+			ret = i915_sw_fence_await_dma_fence(&req->submit,
+							    child, 10*HZ,
+							    GFP_KERNEL);
+		if (ret < 0)
+			return ret;
+	}
+
+	return 0;
+}
+
 /**
  * i915_gem_request_await_object - set this request to (async) wait upon a bo
  *
drivers/gpu/drm/i915/i915_gem_request.h +2 -0
index 974bd7bcc801..c85a3d82febf 100644
@@ -214,6 +214,8 @@ int
 i915_gem_request_await_object(struct drm_i915_gem_request *to,
 			      struct drm_i915_gem_object *obj,
 			      bool write);
+int i915_gem_request_await_fence(struct drm_i915_gem_request *req,
+				 struct fence *fence);
 
 void __i915_add_request(struct drm_i915_gem_request *req, bool flush_caches);
 #define i915_add_request(req) \
Signature · 6 lines
2.9.3

_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
maintainer Reviewed-by on v1 6/42
#8 lore raw

Re: [PATCH 06/42] drm/i915: Support asynchronous waits on struct fence from i915_gem_request

drivers/gpu/drm/i915/i915_gem_request.c i915_gem_request_await_request(struct drm_i915_gem_request *to, · line 519

Quoted 7 lines · Chris Wilson
drivers/gpu/drm/i915/i915_gem_request.c +7 -0
+	if (!fence_is_array(fence)) {
+		ret = i915_sw_fence_await_dma_fence(&req->submit,
+						    fence, 10*HZ,
+						    GFP_KERNEL);
+		return ret < 0 ? ret : 0;
+	}
+

With the magic removed in two spots,

Regards, Joonas

Signature · 7 lines
Joonas Lahtinen
Open Source Technology Center
Intel Corporation
_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
#9 lore raw

Re: [PATCH 06/42] drm/i915: Support asynchronous waits on struct fence from i915_gem_request

In reply to Chris Wilson
Quoted 40 lines · Chris Wilson

We will need to wait on DMA completion (as signaled via struct fence) before executing our i915_gem_request. Therefore we want to expose a method for adding the await on the fence itself to the request.

---
  drivers/gpu/drm/i915/i915_gem_request.c | 40 +++++++++++++++++++++++++++++++++
  drivers/gpu/drm/i915/i915_gem_request.h |  2 ++
  2 files changed, 42 insertions(+)
drivers/gpu/drm/i915/i915_gem_request.c +15 -0
index 8832f8ec1583..e1f7a32d4080 100644
@@ -23,6 +23,7 @@
   */
 
  #include <linux/prefetch.h>
+#include <linux/fence-array.h>
 
  #include "i915_drv.h"
 
@@ -495,6 +496,45 @@ i915_gem_request_await_request(struct drm_i915_gem_request *to,
  	return 0;
  }
 
+int
+i915_gem_request_await_fence(struct drm_i915_gem_request *req,
+			     struct fence *fence)
+{
+	struct fence_array *array;
+	int ret;
+	int i;
+
+	if (test_bit(FENCE_FLAG_SIGNALED_BIT, &fence->flags))
+		return 0;
+
+	if (fence_is_i915(fence))
+		return i915_gem_request_await_request(req, to_request(fence));
+

drivers/gpu/drm/i915/i915_gem_request.c i915_gem_request_await_request(struct drm_i915_gem_request *to, · line 522

Quoted 10 lines
drivers/gpu/drm/i915/i915_gem_request.c +10 -0
+	if (!fence_is_array(fence)) {
+		ret = i915_sw_fence_await_dma_fence(&req->submit,
+						    fence, 10*HZ,
+						    GFP_KERNEL);
+		return ret < 0 ? ret : 0;
+	}
+
+	array = to_fence_array(fence);
+	for (i = 0; i < array->num_fences; i++) {
+		struct fence *child = array->fences[i];

1. What is the advantage in manually waiting on array elements rather than just the array? Is the existance of signal_on_any a problem?

2. Can child be another array?

Regards,

Tvrtko

Quoted 23 lines
drivers/gpu/drm/i915/i915_gem_request.c +15 -0
+
+		if (fence_is_i915(child))
+			ret = i915_gem_request_await_request(req,
+							     to_request(child));
+		else
+			ret = i915_sw_fence_await_dma_fence(&req->submit,
+							    child, 10*HZ,
+							    GFP_KERNEL);
+		if (ret < 0)
+			return ret;
+	}
+
+	return 0;
+}
+
  /**
   * i915_gem_request_await_object - set this request to (async) wait upon a bo
   *
drivers/gpu/drm/i915/i915_gem_request.h +0 -0
index 974bd7bcc801..c85a3d82febf 100644

drivers/gpu/drm/i915/i915_gem_request.h int · line 221

Quoted 8 lines
drivers/gpu/drm/i915/i915_gem_request.h +2 -0
@@ -214,6 +214,8 @@ int
  i915_gem_request_await_object(struct drm_i915_gem_request *to,
  			      struct drm_i915_gem_object *obj,
  			      bool write);
+int i915_gem_request_await_fence(struct drm_i915_gem_request *req,
+				 struct fence *fence);
 
  void __i915_add_request(struct drm_i915_gem_request *req, bool flush_caches);
  #define i915_add_request(req) \

_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx

#10 lore raw

Re: [PATCH 06/42] drm/i915: Support asynchronous waits on struct fence from i915_gem_request

Quoted 44 lines · Tvrtko Ursulin
Quoted 42 lines · Chris Wilson

We will need to wait on DMA completion (as signaled via struct fence) before executing our i915_gem_request. Therefore we want to expose a method for adding the await on the fence itself to the request.

---
 drivers/gpu/drm/i915/i915_gem_request.c | 40 +++++++++++++++++++++++++++++++++
 drivers/gpu/drm/i915/i915_gem_request.h |  2 ++
 2 files changed, 42 insertions(+)
drivers/gpu/drm/i915/i915_gem_request.c +21 -0
index 8832f8ec1583..e1f7a32d4080 100644
@@ -23,6 +23,7 @@
  */
 #include <linux/prefetch.h>
+#include <linux/fence-array.h>
 #include "i915_drv.h"
@@ -495,6 +496,45 @@ i915_gem_request_await_request(struct drm_i915_gem_request *to,
 	return 0;
 }
+int
+i915_gem_request_await_fence(struct drm_i915_gem_request *req,
+			     struct fence *fence)
+{
+	struct fence_array *array;
+	int ret;
+	int i;
+
+	if (test_bit(FENCE_FLAG_SIGNALED_BIT, &fence->flags))
+		return 0;
+
+	if (fence_is_i915(fence))
+		return i915_gem_request_await_request(req, to_request(fence));
+
+	if (!fence_is_array(fence)) {
+		ret = i915_sw_fence_await_dma_fence(&req->submit,
+						    fence, 10*HZ,
+						    GFP_KERNEL);
+		return ret < 0 ? ret : 0;
+	}
Quoted 8 lines

drivers/gpu/drm/i915/i915_gem_request.c i915_gem_request_await_request(struct drm_i915_gem_request *to, · line 522

Quoted 4 lines
drivers/gpu/drm/i915/i915_gem_request.c +4 -0
+
+	array = to_fence_array(fence);
+	for (i = 0; i < array->num_fences; i++) {
+		struct fence *child = array->fences[i];

1. What is the advantage in manually waiting on array elements rather than just the array? Is the existance of signal_on_any a problem?

We can wait on native fences much more efficiently. In the typical case, we don't even need the wait at all since the ordering is imposed by the timeline. Or we can hook into the submission fence to avoid the interrupt. Failing that we have to respond to the interrupt. Also note that foriegn fences handle autosignaling differently to native fences, as we cannot rely on third parties having a working hangcheck.

Quoted 1 line

2. Can child be another array?

Yes, but we don't want to recurse (or at least need to bound the recursion).
-Chris

Signature · 5 lines
Chris Wilson, Intel Open Source Technology Centre
_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
#11 lore raw

Re: [PATCH 06/42] drm/i915: Support asynchronous waits on struct fence from i915_gem_request

Quoted 58 lines · Chris Wilson
Quoted 42 lines · Tvrtko Ursulin
Quoted 41 lines · Chris Wilson

We will need to wait on DMA completion (as signaled via struct fence) before executing our i915_gem_request. Therefore we want to expose a method for adding the await on the fence itself to the request.

---
  drivers/gpu/drm/i915/i915_gem_request.c | 40 +++++++++++++++++++++++++++++++++
  drivers/gpu/drm/i915/i915_gem_request.h |  2 ++
  2 files changed, 42 insertions(+)
drivers/gpu/drm/i915/i915_gem_request.c +20 -0
index 8832f8ec1583..e1f7a32d4080 100644
@@ -23,6 +23,7 @@
   */
  #include <linux/prefetch.h>
+#include <linux/fence-array.h>
  #include "i915_drv.h"
@@ -495,6 +496,45 @@ i915_gem_request_await_request(struct drm_i915_gem_request *to,
  	return 0;
  }
+int
+i915_gem_request_await_fence(struct drm_i915_gem_request *req,
+			     struct fence *fence)
+{
+	struct fence_array *array;
+	int ret;
+	int i;
+
+	if (test_bit(FENCE_FLAG_SIGNALED_BIT, &fence->flags))
+		return 0;
+
+	if (fence_is_i915(fence))
+		return i915_gem_request_await_request(req, to_request(fence));
+
+	if (!fence_is_array(fence)) {
+		ret = i915_sw_fence_await_dma_fence(&req->submit,
+						    fence, 10*HZ,
+						    GFP_KERNEL);
+		return ret < 0 ? ret : 0;
Quoted 8 lines

drivers/gpu/drm/i915/i915_gem_request.c i915_gem_request_await_request(struct drm_i915_gem_request *to, · line 522

Quoted 5 lines
drivers/gpu/drm/i915/i915_gem_request.c +5 -0
+	}
+
+	array = to_fence_array(fence);
+	for (i = 0; i < array->num_fences; i++) {
+		struct fence *child = array->fences[i];

1. What is the advantage in manually waiting on array elements rather than just the array? Is the existance of signal_on_any a problem?

We can wait on native fences much more efficiently. In the typical case, we don't even need the wait at all since the ordering is imposed by the timeline. Or we can hook into the submission fence to avoid the interrupt. Failing that we have to respond to the interrupt. Also note that foriegn fences handle autosignaling differently to native fences, as we cannot rely on third parties having a working hangcheck.

Quoted 3 lines
Quoted 1 line

2. Can child be another array?

Yes, but we don't want to recurse (or at least need to bound the recursion).

In that case could the array have been created in the signal_on_any mode and how would we handle that?

Regards,

Tvrtko

_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx

#12 lore raw

Re: [PATCH 06/42] drm/i915: Support asynchronous waits on struct fence from i915_gem_request

Quoted 8 lines · Tvrtko Ursulin
Quoted 3 lines · Chris Wilson
Quoted 1 line

2. Can child be another array?

Yes, but we don't want to recurse (or at least need to bound the recursion).

In that case could the array have been created in the signal_on_any mode and how would we handle that?

Hmm. Not along our paths, I think. The only fence-array we have are from sync-file which are signal-on-all (except... in the case of where it wraps a single fence, that fence could be a composite). signal-on-any is icky, to be complete we should not decompose it into its elements - however, whether or not a fence-array is operating in signal_on_any mode is not stored. So at the moment, the best I can do is offer a comment.

The only user of it at the moment is amdgpu waiting for the first of many VM manager to become available, not something we'll see immediately via sync-file.
-Chris

Signature · 5 lines
Chris Wilson, Intel Open Source Technology Centre
_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
#13 lore raw

Re: [PATCH 06/42] drm/i915: Support asynchronous waits on struct fence from i915_gem_request

Quoted 5 lines · Chris Wilson
Quoted 4 lines · Tvrtko Ursulin
Quoted 3 lines · Chris Wilson
Quoted 1 line

2. Can child be another array?

Yes, but we don't want to recurse (or at least need to bound the recursion).

Quoted 8 lines
Quoted 2 lines

In that case could the array have been created in the signal_on_any mode and how would we handle that?

Hmm. Not along our paths, I think. The only fence-array we have are from sync-file which are signal-on-all (except... in the case of where it wraps a single fence, that fence could be a composite). signal-on-any is icky, to be complete we should not decompose it into its elements - however, whether or not a fence-array is operating in signal_on_any mode is not stored. So at the moment, the best I can do is offer a comment.

Yes signal-on-any sounds extremely weird. It mandates you decompose 3rd party fences in all cases, with recursion.

Quoted 3 lines

The only user of it at the moment is amdgpu waiting for the first of many VM manager to become available, not something we'll see immediately via sync-file.

So userspace cannot engineer it to happen with some funky operations before giving us a fence?

Regards,

Tvrtko

_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx

#14 lore raw

Re: [PATCH 06/42] drm/i915: Support asynchronous waits on struct fence from i915_gem_request

Quoted 19 lines · Tvrtko Ursulin
Quoted 5 lines · Chris Wilson
Quoted 4 lines · Tvrtko Ursulin
Quoted 3 lines · Chris Wilson
Quoted 1 line

2. Can child be another array?

Yes, but we don't want to recurse (or at least need to bound the recursion).

Quoted 8 lines
Quoted 2 lines

In that case could the array have been created in the signal_on_any mode and how would we handle that?

Hmm. Not along our paths, I think. The only fence-array we have are from sync-file which are signal-on-all (except... in the case of where it wraps a single fence, that fence could be a composite). signal-on-any is icky, to be complete we should not decompose it into its elements - however, whether or not a fence-array is operating in signal_on_any mode is not stored. So at the moment, the best I can do is offer a comment.

Yes signal-on-any sounds extremely weird. It mandates you decompose 3rd party fences in all cases, with recursion.

Quoted 6 lines
Quoted 3 lines

The only user of it at the moment is amdgpu waiting for the first of many VM manager to become available, not something we'll see immediately via sync-file.

So userspace cannot engineer it to happen with some funky operations before giving us a fence?

No. sync_file is a fence-array (in normal signal-on-all mode) or a single fence (which via merging sync_files can not be another fence-array, though the sync_file could be constructed pointing to a single fence-array fence). If that is a signal-on-any fence, it is broken if the sync_file is ever merged.... And reviewing the existing code, no one should be creating sync_file from a signal-on-any fence-array. There is no ABI to allow the user to create signal-on-any sync_file. I'm feeling safer that there are quite a few issues to resolve before signal-on-any fences are allowed to pollute the sync_file ABI.
-Chris

Signature · 5 lines
Chris Wilson, Intel Open Source Technology Centre
_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
#15 lore raw

PATCH 7/42 drm/i915: Allow i915_sw_fence_await_sw_fence() to allocate

In reply to Chris Wilson

In forthcoming patches, we want to be able to dynamically allocate the wait_queue_t used whilst awaiting. This is more convenient if we extend the i915_sw_fence_await_sw_fence() to perform the allocation for us if we pass in a gfp mask as an alternative than a preallocated struct.

---
 drivers/gpu/drm/i915/i915_sw_fence.c | 40 ++++++++++++++++++++++++++++++++----
 drivers/gpu/drm/i915/i915_sw_fence.h |  8 ++++++++
 2 files changed, 44 insertions(+), 4 deletions(-)
drivers/gpu/drm/i915/i915_sw_fence.c +36 -4
index 1e5cbc585ca2..dfb59dced0ce 100644
@@ -13,6 +13,8 @@
 
 #include "i915_sw_fence.h"
 
+#define I915_SW_FENCE_FLAG_ALLOC BIT(0)
+
 static DEFINE_SPINLOCK(i915_sw_fence_lock);
 
 static int __i915_sw_fence_notify(struct i915_sw_fence *fence,
@@ -135,6 +137,8 @@ static int i915_sw_fence_wake(wait_queue_t *wq, unsigned mode, int flags, void *
 	list_del(&wq->task_list);
 	__i915_sw_fence_complete(wq->private, key);
 	i915_sw_fence_put(wq->private);
+	if (wq->flags & I915_SW_FENCE_FLAG_ALLOC)
+		kfree(wq);
 	return 0;
 }
 
@@ -192,9 +196,9 @@ static bool i915_sw_fence_check_if_after(struct i915_sw_fence *fence,
 	return err;
 }
 
-int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence,
-				 struct i915_sw_fence *signaler,
-				 wait_queue_t *wq)
+static int __i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence,
+					  struct i915_sw_fence *signaler,
+					  wait_queue_t *wq, gfp_t gfp)
 {
 	unsigned long flags;
 	int pending;
@@ -206,8 +210,22 @@ int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence,
 	if (unlikely(i915_sw_fence_check_if_after(fence, signaler)))
 		return -EINVAL;
 
+	pending = 0;
+	if (!wq) {
+		wq = kmalloc(sizeof(*wq), gfp);
+		if (!wq) {
+			if (!gfpflags_allow_blocking(gfp))
+				return -ENOMEM;
+
+			i915_sw_fence_wait(signaler);
+			return 0;
+		}
+
+		pending |= I915_SW_FENCE_FLAG_ALLOC;
+	}
+
 	INIT_LIST_HEAD(&wq->task_list);
-	wq->flags = 0;
+	wq->flags = pending;
 	wq->func = i915_sw_fence_wake;
 	wq->private = i915_sw_fence_get(fence);
 
@@ -226,6 +244,20 @@ int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence,
 	return pending;
 }
 
+int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence,
+				 struct i915_sw_fence *signaler,
+				 wait_queue_t *wq)
+{
+	return __i915_sw_fence_await_sw_fence(fence, signaler, wq, 0);
+}
+
+int i915_sw_fence_await_sw_fence_gfp(struct i915_sw_fence *fence,
+				     struct i915_sw_fence *signaler,
+				     gfp_t gfp)
+{
+	return __i915_sw_fence_await_sw_fence(fence, signaler, NULL, gfp);
+}
+
 struct dma_fence_cb {
 	struct fence_cb base;
 	struct i915_sw_fence *fence;
drivers/gpu/drm/i915/i915_sw_fence.h +8 -0
index 373141602ca4..5861c69f24d4 100644
@@ -46,6 +46,9 @@ void i915_sw_fence_commit(struct i915_sw_fence *fence);
 int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence,
 				 struct i915_sw_fence *after,
 				 wait_queue_t *wq);
+int i915_sw_fence_await_sw_fence_gfp(struct i915_sw_fence *fence,
+				     struct i915_sw_fence *after,
+				     gfp_t gfp);
 int i915_sw_fence_await_dma_fence(struct i915_sw_fence *fence,
 				  struct fence *dma,
 				  unsigned long timeout,
@@ -62,4 +65,9 @@ static inline bool i915_sw_fence_done(const struct i915_sw_fence *fence)
 	return atomic_read(&fence->pending) < 0;
 }
 
+static inline void i915_sw_fence_wait(struct i915_sw_fence *fence)
+{
+	wait_event(fence->wait, i915_sw_fence_done(fence));
+}
+
 #endif /* _I915_SW_FENCE_H_ */
Signature · 6 lines
2.9.3

_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
#16 lore raw

Re: [PATCH 07/42] drm/i915: Allow i915_sw_fence_await_sw_fence() to allocate

Quoted 17 lines · Chris Wilson

In forthcoming patches, we want to be able to dynamically allocate the wait_queue_t used whilst awaiting. This is more convenient if we extend the i915_sw_fence_await_sw_fence() to perform the allocation for us if we pass in a gfp mask as an alternative than a preallocated struct.

---
  drivers/gpu/drm/i915/i915_sw_fence.c | 40 ++++++++++++++++++++++++++++++++----
  drivers/gpu/drm/i915/i915_sw_fence.h |  8 ++++++++
  2 files changed, 44 insertions(+), 4 deletions(-)
drivers/gpu/drm/i915/i915_sw_fence.c +0 -0
index 1e5cbc585ca2..dfb59dced0ce 100644

drivers/gpu/drm/i915/i915_sw_fence.c line 17

Quoted 5 lines
drivers/gpu/drm/i915/i915_sw_fence.c +2 -0
@@ -13,6 +13,8 @@
 
  #include "i915_sw_fence.h"
 
+#define I915_SW_FENCE_FLAG_ALLOC BIT(0)
+

You collide with WQ_FLAG_EXCLUSIVE here, so I assume i915 will have complete control of the wq in question?

Quoted 27 lines
drivers/gpu/drm/i915/i915_sw_fence.c +5 -3
  static DEFINE_SPINLOCK(i915_sw_fence_lock);
 
  static int __i915_sw_fence_notify(struct i915_sw_fence *fence,
@@ -135,6 +137,8 @@ static int i915_sw_fence_wake(wait_queue_t *wq, unsigned mode, int flags, void *
  	list_del(&wq->task_list);
  	__i915_sw_fence_complete(wq->private, key);
  	i915_sw_fence_put(wq->private);
+	if (wq->flags & I915_SW_FENCE_FLAG_ALLOC)
+		kfree(wq);
  	return 0;
  }
 
@@ -192,9 +196,9 @@ static bool i915_sw_fence_check_if_after(struct i915_sw_fence *fence,
  	return err;
  }
 
-int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence,
-				 struct i915_sw_fence *signaler,
-				 wait_queue_t *wq)
+static int __i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence,
+					  struct i915_sw_fence *signaler,
+					  wait_queue_t *wq, gfp_t gfp)
  {
  	unsigned long flags;
  	int pending;
@@ -206,8 +210,22 @@ int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence,
  	if (unlikely(i915_sw_fence_check_if_after(fence, signaler)))

drivers/gpu/drm/i915/i915_sw_fence.c int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence, · line 220

Quoted 10 lines
drivers/gpu/drm/i915/i915_sw_fence.c +8 -0
  		return -EINVAL;
 
+	pending = 0;
+	if (!wq) {
+		wq = kmalloc(sizeof(*wq), gfp);
+		if (!wq) {
+			if (!gfpflags_allow_blocking(gfp))
+				return -ENOMEM;
+
+			i915_sw_fence_wait(signaler);

This looks like a strange API, unless I am missing something - it normally sets things up for waiting, unless the allocation fails when it actually does the wait?

drivers/gpu/drm/i915/i915_sw_fence.c int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence, · line 224

Quoted 4 lines
drivers/gpu/drm/i915/i915_sw_fence.c +4 -0
+			return 0;
+		}
+
+		pending |= I915_SW_FENCE_FLAG_ALLOC;

Why is this stored in pending? Just because it is around as re-used?

Quoted 44 lines
drivers/gpu/drm/i915/i915_sw_fence.c +17 -1
+	}
+
  	INIT_LIST_HEAD(&wq->task_list);
-	wq->flags = 0;
+	wq->flags = pending;
  	wq->func = i915_sw_fence_wake;
  	wq->private = i915_sw_fence_get(fence);
 
@@ -226,6 +244,20 @@ int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence,
  	return pending;
  }
 
+int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence,
+				 struct i915_sw_fence *signaler,
+				 wait_queue_t *wq)
+{
+	return __i915_sw_fence_await_sw_fence(fence, signaler, wq, 0);
+}
+
+int i915_sw_fence_await_sw_fence_gfp(struct i915_sw_fence *fence,
+				     struct i915_sw_fence *signaler,
+				     gfp_t gfp)
+{
+	return __i915_sw_fence_await_sw_fence(fence, signaler, NULL, gfp);
+}
+
  struct dma_fence_cb {
  	struct fence_cb base;
  	struct i915_sw_fence *fence;
drivers/gpu/drm/i915/i915_sw_fence.h +3 -0
index 373141602ca4..5861c69f24d4 100644
@@ -46,6 +46,9 @@ void i915_sw_fence_commit(struct i915_sw_fence *fence);
  int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence,
  				 struct i915_sw_fence *after,
  				 wait_queue_t *wq);
+int i915_sw_fence_await_sw_fence_gfp(struct i915_sw_fence *fence,
+				     struct i915_sw_fence *after,
+				     gfp_t gfp);
  int i915_sw_fence_await_dma_fence(struct i915_sw_fence *fence,
  				  struct fence *dma,
  				  unsigned long timeout,

drivers/gpu/drm/i915/i915_sw_fence.h static inline bool i915_sw_fence_done(const struct i915_sw_fence *fence) · line 73

Quoted 9 lines
drivers/gpu/drm/i915/i915_sw_fence.h +5 -0
@@ -62,4 +65,9 @@ static inline bool i915_sw_fence_done(const struct i915_sw_fence *fence)
  	return atomic_read(&fence->pending) < 0;
  }
 
+static inline void i915_sw_fence_wait(struct i915_sw_fence *fence)
+{
+	wait_event(fence->wait, i915_sw_fence_done(fence));
+}
+
  #endif /* _I915_SW_FENCE_H_ */

Regards,

Tvrtko
_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx

#17 lore raw

Re: [PATCH 07/42] drm/i915: Allow i915_sw_fence_await_sw_fence() to allocate

Quoted 14 lines · Tvrtko Ursulin
Quoted 12 lines · Chris Wilson

In forthcoming patches, we want to be able to dynamically allocate the wait_queue_t used whilst awaiting. This is more convenient if we extend the i915_sw_fence_await_sw_fence() to perform the allocation for us if we pass in a gfp mask as an alternative than a preallocated struct.

---
 drivers/gpu/drm/i915/i915_sw_fence.c | 40 ++++++++++++++++++++++++++++++++----
 drivers/gpu/drm/i915/i915_sw_fence.h |  8 ++++++++
 2 files changed, 44 insertions(+), 4 deletions(-)
Quoted 11 lines

drivers/gpu/drm/i915/i915_sw_fence.c line 15

Quoted 8 lines
drivers/gpu/drm/i915/i915_sw_fence.c +2 -0
index 1e5cbc585ca2..dfb59dced0ce 100644
@@ -13,6 +13,8 @@
 #include "i915_sw_fence.h"
+#define I915_SW_FENCE_FLAG_ALLOC BIT(0)
+

You collide with WQ_FLAG_EXCLUSIVE here, so I assume i915 will have complete control of the wq in question?

Hmm, we should move bit as well, but yes the flag is atm private to the wake up function for which we use a custom function.

Quoted 10 lines

drivers/gpu/drm/i915/i915_sw_fence.c int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence, · line 220

Quoted 6 lines
drivers/gpu/drm/i915/i915_sw_fence.c +6 -0
+		wq = kmalloc(sizeof(*wq), gfp);
+		if (!wq) {
+			if (!gfpflags_allow_blocking(gfp))
+				return -ENOMEM;
+
+			i915_sw_fence_wait(signaler);

This looks like a strange API, unless I am missing something - it normally sets things up for waiting, unless the allocation fails when it actually does the wait?

Yes. If we blocked on the malloc and failed, we block a little more for the completion. iirc, the idea came from the async task.

Quoted 7 lines

drivers/gpu/drm/i915/i915_sw_fence.c int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence, · line 224

Quoted 5 lines
drivers/gpu/drm/i915/i915_sw_fence.c +4 -0
+			return 0;
+		}
+
+		pending |= I915_SW_FENCE_FLAG_ALLOC;

Why is this stored in pending? Just because it is around as re-used?

Urm, yes. s/pending/tmp/
-Chris

Signature · 5 lines
Chris Wilson, Intel Open Source Technology Centre
_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
#18 lore raw

Re: [PATCH 07/42] drm/i915: Allow i915_sw_fence_await_sw_fence() to allocate

Quoted 36 lines · Chris Wilson
Quoted 13 lines · Tvrtko Ursulin
Quoted 12 lines · Chris Wilson

In forthcoming patches, we want to be able to dynamically allocate the wait_queue_t used whilst awaiting. This is more convenient if we extend the i915_sw_fence_await_sw_fence() to perform the allocation for us if we pass in a gfp mask as an alternative than a preallocated struct.

---
  drivers/gpu/drm/i915/i915_sw_fence.c | 40 ++++++++++++++++++++++++++++++++----
  drivers/gpu/drm/i915/i915_sw_fence.h |  8 ++++++++
  2 files changed, 44 insertions(+), 4 deletions(-)
Quoted 10 lines

drivers/gpu/drm/i915/i915_sw_fence.c line 15

Quoted 8 lines
drivers/gpu/drm/i915/i915_sw_fence.c +2 -0
index 1e5cbc585ca2..dfb59dced0ce 100644
@@ -13,6 +13,8 @@
  #include "i915_sw_fence.h"
+#define I915_SW_FENCE_FLAG_ALLOC BIT(0)
+

You collide with WQ_FLAG_EXCLUSIVE here, so I assume i915 will have complete control of the wq in question?

Hmm, we should move bit as well, but yes the flag is atm private to the wake up function for which we use a custom function.

Quoted 9 lines

drivers/gpu/drm/i915/i915_sw_fence.c int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence, · line 220

Quoted 6 lines
drivers/gpu/drm/i915/i915_sw_fence.c +6 -0
+		wq = kmalloc(sizeof(*wq), gfp);
+		if (!wq) {
+			if (!gfpflags_allow_blocking(gfp))
+				return -ENOMEM;
+
+			i915_sw_fence_wait(signaler);

This looks like a strange API, unless I am missing something - it normally sets things up for waiting, unless the allocation fails when it actually does the wait?

Quoted 2 lines

Yes. If we blocked on the malloc and failed, we block a little more for the completion. iirc, the idea came from the async task.

Still sounds strange to me. I understand it is a unlikely corner case but sounds like that sort of decision should happen in the caller. I will have to revisit the i915_sw_fence implementation and usage in more detail it seems.

Regards,

Tvrtko

_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx

#19 lore raw

PATCH 8/42 drm/i915: Rearrange i915_wait_request() accounting with callers

In reply to Chris Wilson

Our low-level wait routine has evolved from our generic wait interface that handled unlocked, RPS boosting, waits with time tracking. If we push our GEM fence tracking to use reservation_objects (required for handling multiple timelines), we lose the ability to pass the required information down to i915_wait_request(). However, if we push the extra functionality from i915_wait_request() to the individual callsites (i915_gem_object_wait_rendering and i915_gem_wait_ioctl) that make use of those extras, we can both simplify our low level wait and prepare for extending the GEM interface for use of reservation_objects.

---
 drivers/gpu/drm/i915/i915_drv.h         |   7 +-
 drivers/gpu/drm/i915/i915_gem.c         | 301 +++++++++++++++++++++++---------
 drivers/gpu/drm/i915/i915_gem_request.c | 119 ++-----------
 drivers/gpu/drm/i915/i915_gem_request.h |  32 ++--
 drivers/gpu/drm/i915/i915_gem_userptr.c |  12 +-
 drivers/gpu/drm/i915/intel_display.c    |  27 +--
 drivers/gpu/drm/i915/intel_ringbuffer.c |  14 +-
 drivers/gpu/drm/i915/intel_ringbuffer.h |   3 +-
 8 files changed, 283 insertions(+), 232 deletions(-)
drivers/gpu/drm/i915/i915_drv.h +4 -3
index 54b1b931573c..ee25e265416f 100644
@@ -3301,9 +3301,10 @@ int __must_check i915_gem_wait_for_idle(struct drm_i915_private *dev_priv,
 int __must_check i915_gem_suspend(struct drm_device *dev);
 void i915_gem_resume(struct drm_device *dev);
 int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf);
-int __must_check
-i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
-			       bool readonly);
+int i915_gem_object_wait(struct drm_i915_gem_object *obj,
+			 unsigned int flags,
+			 long timeout,
+			 struct intel_rps_client *rps);
 int __must_check
 i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj,
 				  bool write);
drivers/gpu/drm/i915/i915_gem.c +219 -82

443 lines not on this page · raw message

drivers/gpu/drm/i915/i915_gem_request.c +18 -101

179 lines not on this page · raw message

drivers/gpu/drm/i915/i915_gem_request.h +16 -16
index c85a3d82febf..f1ce390c9244 100644
@@ -228,13 +228,13 @@ struct intel_rps_client;
 #define IS_RPS_CLIENT(p) (!IS_ERR(p))
 #define IS_RPS_USER(p) (!IS_ERR_OR_NULL(p))
 
-int i915_wait_request(struct drm_i915_gem_request *req,
-		      unsigned int flags,
-		      s64 *timeout,
-		      struct intel_rps_client *rps)
+long i915_wait_request(struct drm_i915_gem_request *req,
+		       unsigned int flags,
+		       long timeout)
 	__attribute__((nonnull(1)));
 #define I915_WAIT_INTERRUPTIBLE	BIT(0)
 #define I915_WAIT_LOCKED	BIT(1) /* struct_mutex held, handle GPU reset */
+#define I915_WAIT_ALL		BIT(2) /* used by i915_gem_object_wait() */
 
 static inline u32 intel_engine_get_seqno(struct intel_engine_cs *engine);
 
@@ -583,14 +583,16 @@ static inline int __must_check
 i915_gem_active_wait(const struct i915_gem_active *active, struct mutex *mutex)
 {
 	struct drm_i915_gem_request *request;
+	long ret;
 
 	request = i915_gem_active_peek(active, mutex);
 	if (!request)
 		return 0;
 
-	return i915_wait_request(request,
-				 I915_WAIT_INTERRUPTIBLE | I915_WAIT_LOCKED,
-				 NULL, NULL);
+	ret = i915_wait_request(request,
+				I915_WAIT_INTERRUPTIBLE | I915_WAIT_LOCKED,
+				MAX_SCHEDULE_TIMEOUT);
+	return ret < 0 ? ret : 0;
 }
 
 /**
@@ -617,20 +619,18 @@ i915_gem_active_wait(const struct i915_gem_active *active, struct mutex *mutex)
  */
 static inline int
 i915_gem_active_wait_unlocked(const struct i915_gem_active *active,
-			      unsigned int flags,
-			      s64 *timeout,
-			      struct intel_rps_client *rps)
+			      unsigned int flags)
 {
 	struct drm_i915_gem_request *request;
-	int ret = 0;
+	long ret = 0;
 
 	request = i915_gem_active_get_unlocked(active);
 	if (request) {
-		ret = i915_wait_request(request, flags, timeout, rps);
+		ret = i915_wait_request(request, flags, MAX_SCHEDULE_TIMEOUT);
 		i915_gem_request_put(request);
 	}
 
-	return ret;
+	return ret < 0 ? ret : 0;
 }
 
 /**
@@ -647,7 +647,7 @@ i915_gem_active_retire(struct i915_gem_active *active,
 		       struct mutex *mutex)
 {
 	struct drm_i915_gem_request *request;
-	int ret;
+	long ret;
 
 	request = i915_gem_active_raw(active, mutex);
 	if (!request)
@@ -655,8 +655,8 @@ i915_gem_active_retire(struct i915_gem_active *active,
 
 	ret = i915_wait_request(request,
 				I915_WAIT_INTERRUPTIBLE | I915_WAIT_LOCKED,
-				NULL, NULL);
-	if (ret)
+				MAX_SCHEDULE_TIMEOUT);
+	if (ret < 0)
 		return ret;
 
 	list_del_init(&active->link);
drivers/gpu/drm/i915/i915_gem_userptr.c +1 -11
index e537930c64b5..1c891b92ac80 100644
@@ -61,23 +61,13 @@ struct i915_mmu_object {
 	bool attached;
 };
 
-static void wait_rendering(struct drm_i915_gem_object *obj)
-{
-	unsigned long active = __I915_BO_ACTIVE(obj);
-	int idx;
-
-	for_each_active(active, idx)
-		i915_gem_active_wait_unlocked(&obj->last_read[idx],
-					      0, NULL, NULL);
-}
-
 static void cancel_userptr(struct work_struct *work)
 {
 	struct i915_mmu_object *mo = container_of(work, typeof(*mo), work);
 	struct drm_i915_gem_object *obj = mo->obj;
 	struct drm_device *dev = obj->base.dev;
 
-	wait_rendering(obj);
+	i915_gem_object_wait(obj, I915_WAIT_ALL, MAX_SCHEDULE_TIMEOUT, NULL);
 
 	mutex_lock(&dev->struct_mutex);
 	/* Cancel any active worker and force us to re-evaluate gup */
drivers/gpu/drm/i915/intel_display.c +16 -11
index 4b3d315c5254..5ee7bab6e560 100644
@@ -12042,7 +12042,7 @@ static void intel_mmio_flip_work_func(struct work_struct *w)
 
 	if (work->flip_queued_req)
 		WARN_ON(i915_wait_request(work->flip_queued_req,
-					  0, NULL, NO_WAITBOOST));
+					  0, MAX_SCHEDULE_TIMEOUT) < 0);
 
 	/* For framebuffer backed by dmabuf, wait for fence */
 	resv = i915_gem_object_get_dmabuf_resv(obj);
@@ -14089,19 +14089,21 @@ static int intel_atomic_prepare_commit(struct drm_device *dev,
 		for_each_plane_in_state(state, plane, plane_state, i) {
 			struct intel_plane_state *intel_plane_state =
 				to_intel_plane_state(plane_state);
+			long timeout;
 
 			if (!intel_plane_state->wait_req)
 				continue;
 
-			ret = i915_wait_request(intel_plane_state->wait_req,
-						I915_WAIT_INTERRUPTIBLE,
-						NULL, NULL);
-			if (ret) {
+			timeout = i915_wait_request(intel_plane_state->wait_req,
+						    I915_WAIT_INTERRUPTIBLE,
+						    MAX_SCHEDULE_TIMEOUT);
+			if (timeout < 0) {
 				/* Any hang should be swallowed by the wait */
-				WARN_ON(ret == -EIO);
+				WARN_ON(timeout == -EIO);
 				mutex_lock(&dev->struct_mutex);
 				drm_atomic_helper_cleanup_planes(dev, state);
 				mutex_unlock(&dev->struct_mutex);
+				ret = timeout;
 				break;
 			}
 		}
@@ -14303,7 +14305,7 @@ static void intel_atomic_commit_tail(struct drm_atomic_state *state)
 	bool hw_check = intel_state->modeset;
 	unsigned long put_domains[I915_MAX_PIPES] = {};
 	unsigned crtc_vblank_mask = 0;
-	int i, ret;
+	int i;
 
 	for_each_plane_in_state(state, plane, plane_state, i) {
 		struct intel_plane_state *intel_plane_state =
@@ -14312,11 +14314,10 @@ static void intel_atomic_commit_tail(struct drm_atomic_state *state)
 		if (!intel_plane_state->wait_req)
 			continue;
 
-		ret = i915_wait_request(intel_plane_state->wait_req,
-					0, NULL, NULL);
 		/* EIO should be eaten, and we can't get interrupted in the
 		 * worker, and blocking commits have waited already. */
-		WARN_ON(ret);
+		WARN_ON(i915_wait_request(intel_plane_state->wait_req,
+					  0, MAX_SCHEDULE_TIMEOUT) < 0);
 	}
 
 	drm_atomic_helper_wait_for_dependencies(state);
@@ -14679,7 +14680,11 @@ intel_prepare_plane_fb(struct drm_plane *plane,
 		 * can safely continue.
 		 */
 		if (needs_modeset(crtc_state))
-			ret = i915_gem_object_wait_rendering(old_obj, true);
+			ret = i915_gem_object_wait(old_obj,
+						   I915_WAIT_INTERRUPTIBLE |
+						   I915_WAIT_LOCKED,
+						   MAX_SCHEDULE_TIMEOUT,
+						   NULL);
 		if (ret) {
 			/* GPU hangs should have been swallowed by the wait */
 			WARN_ON(ret == -EIO);
drivers/gpu/drm/i915/intel_ringbuffer.c +8 -6
index 729f373782e2..b60c6f09fbfd 100644
@@ -2156,7 +2156,9 @@ static int wait_for_space(struct drm_i915_gem_request *req, int bytes)
 {
 	struct intel_ring *ring = req->ring;
 	struct drm_i915_gem_request *target;
-	int ret;
+	long timeout;
+
+	lockdep_assert_held(&req->i915->drm.struct_mutex);
 
 	intel_ring_update_space(ring);
 	if (ring->space >= bytes)
@@ -2186,11 +2188,11 @@ static int wait_for_space(struct drm_i915_gem_request *req, int bytes)
 	if (WARN_ON(&target->ring_link == &ring->request_list))
 		return -ENOSPC;
 
-	ret = i915_wait_request(target,
-				I915_WAIT_INTERRUPTIBLE | I915_WAIT_LOCKED,
-				NULL, NO_WAITBOOST);
-	if (ret)
-		return ret;
+	timeout = i915_wait_request(target,
+				    I915_WAIT_INTERRUPTIBLE | I915_WAIT_LOCKED,
+				    MAX_SCHEDULE_TIMEOUT);
+	if (timeout < 0)
+		return timeout;
 
 	i915_gem_request_retire_upto(target);
 
drivers/gpu/drm/i915/intel_ringbuffer.h +1 -2
index 124f4646958d..dbb7176534af 100644
@@ -530,8 +530,7 @@ static inline int intel_engine_idle(struct intel_engine_cs *engine,
 				    unsigned int flags)
 {
 	/* Wait upon the last request to be completed */
-	return i915_gem_active_wait_unlocked(&engine->last_request,
-					     flags, NULL, NULL);
+	return i915_gem_active_wait_unlocked(&engine->last_request, flags);
 }
 
 int intel_init_render_ring_buffer(struct intel_engine_cs *engine);
Signature · 6 lines
2.9.3

_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
maintainer
#20 lore raw

Re: [PATCH 08/42] drm/i915: Rearrange i915_wait_request() accounting with callers

Quoted 9 lines · Chris Wilson

Our low-level wait routine has evolved from our generic wait interface that handled unlocked, RPS boosting, waits with time tracking. If we push our GEM fence tracking to use reservation_objects (required for handling multiple timelines), we lose the ability to pass the required information down to i915_wait_request(). However, if we push the extra functionality from i915_wait_request() to the individual callsites (i915_gem_object_wait_rendering and i915_gem_wait_ioctl) that make use of those extras, we can both simplify our low level wait and prepare for extending the GEM interface for use of reservation_objects.

No changelog, no re-review.

Regards, Joonas

Signature · 7 lines
Joonas Lahtinen
Open Source Technology Center
Intel Corporation
_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
#21 lore raw

PATCH 9/42 drm/i915: Remove unused i915_gem_active_wait() in favour of _unlocked()

In reply to Chris Wilson

Since we only use the more generic unlocked variant, just rename it as the normal i915_gem_active_wait(). The temporary cost is that we need to always acquire the reference in a RCU safe manner, but the benefit is that we will combine the common paths.

---
 drivers/gpu/drm/i915/i915_gem_request.h | 34 +++------------------------------
 drivers/gpu/drm/i915/intel_ringbuffer.h |  2 +-
 2 files changed, 4 insertions(+), 32 deletions(-)
drivers/gpu/drm/i915/i915_gem_request.h +3 -31
index f1ce390c9244..45998eedda2c 100644
@@ -569,40 +569,13 @@ i915_gem_active_is_idle(const struct i915_gem_active *active,
 }
 
 /**
- * i915_gem_active_wait - waits until the request is completed
- * @active - the active request on which to wait
- *
- * i915_gem_active_wait() waits until the request is completed before
- * returning. Note that it does not guarantee that the request is
- * retired first, see i915_gem_active_retire().
- *
- * i915_gem_active_wait() returns immediately if the active
- * request is already complete.
- */
-static inline int __must_check
-i915_gem_active_wait(const struct i915_gem_active *active, struct mutex *mutex)
-{
-	struct drm_i915_gem_request *request;
-	long ret;
-
-	request = i915_gem_active_peek(active, mutex);
-	if (!request)
-		return 0;
-
-	ret = i915_wait_request(request,
-				I915_WAIT_INTERRUPTIBLE | I915_WAIT_LOCKED,
-				MAX_SCHEDULE_TIMEOUT);
-	return ret < 0 ? ret : 0;
-}
-
-/**
- * i915_gem_active_wait_unlocked - waits until the request is completed
+ * i915_gem_active_wait- waits until the request is completed
  * @active - the active request on which to wait
  * @flags - how to wait
  * @timeout - how long to wait at most
  * @rps - userspace client to charge for a waitboost
  *
- * i915_gem_active_wait_unlocked() waits until the request is completed before
+ * i915_gem_active_wait() waits until the request is completed before
  * returning, without requiring any locks to be held. Note that it does not
  * retire any requests before returning.
  *
@@ -618,8 +591,7 @@ i915_gem_active_wait(const struct i915_gem_active *active, struct mutex *mutex)
  * Returns 0 if successful, or a negative error code.
  */
 static inline int
-i915_gem_active_wait_unlocked(const struct i915_gem_active *active,
-			      unsigned int flags)
+i915_gem_active_wait(const struct i915_gem_active *active, unsigned int flags)
 {
 	struct drm_i915_gem_request *request;
 	long ret = 0;
drivers/gpu/drm/i915/intel_ringbuffer.h +1 -1
index dbb7176534af..a0510d4940b2 100644
@@ -530,7 +530,7 @@ static inline int intel_engine_idle(struct intel_engine_cs *engine,
 				    unsigned int flags)
 {
 	/* Wait upon the last request to be completed */
-	return i915_gem_active_wait_unlocked(&engine->last_request, flags);
+	return i915_gem_active_wait(&engine->last_request, flags);
 }
 
 int intel_init_render_ring_buffer(struct intel_engine_cs *engine);
Signature · 6 lines
2.9.3

_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
#22 lore raw

PATCH 10/42 drm/i915: Defer active reference until required

In reply to Chris Wilson

We only need the active reference to keep the object alive after the handle has been deleted (so as to prevent a synchronous gem_close). Why then pay the price of a kref on every execbuf when we can insert that final active ref just in time for the handle deletion?

---
 drivers/gpu/drm/i915/i915_drv.h              | 28 ++++++++++++++++++++++++++++
 drivers/gpu/drm/i915/i915_gem.c              | 22 +++++++++++++++++++++-
 drivers/gpu/drm/i915/i915_gem_batch_pool.c   |  2 +-
 drivers/gpu/drm/i915/i915_gem_context.c      |  2 +-
 drivers/gpu/drm/i915/i915_gem_execbuffer.c   |  2 --
 drivers/gpu/drm/i915/i915_gem_gtt.c          |  7 ++++++-
 drivers/gpu/drm/i915/i915_gem_render_state.c |  3 ++-
 drivers/gpu/drm/i915/intel_ringbuffer.c      | 15 ++++++++++++---
 8 files changed, 71 insertions(+), 10 deletions(-)
drivers/gpu/drm/i915/i915_drv.h +28 -0
index ee25e265416f..fee5cc92e2f2 100644
@@ -2232,6 +2232,12 @@ struct drm_i915_gem_object {
 	((READ_ONCE((bo)->flags) >> I915_BO_ACTIVE_SHIFT) & I915_BO_ACTIVE_MASK)
 
 	/**
+	 * Have we taken a reference for the object for incomplete GPU
+	 * activity?
+	 */
+#define I915_BO_ACTIVE_REF (I915_BO_ACTIVE_SHIFT + I915_NUM_ENGINES)
+
+	/**
 	 * This is set if the object has been written to since last bound
 	 * to the GTT
 	 */
@@ -2399,6 +2405,28 @@ i915_gem_object_has_active_engine(const struct drm_i915_gem_object *obj,
 	return obj->flags & BIT(engine + I915_BO_ACTIVE_SHIFT);
 }
 
+static inline bool
+i915_gem_object_has_active_reference(const struct drm_i915_gem_object *obj)
+{
+	return test_bit(I915_BO_ACTIVE_REF, &obj->flags);
+}
+
+static inline void
+i915_gem_object_set_active_reference(struct drm_i915_gem_object *obj)
+{
+	lockdep_assert_held(&obj->base.dev->struct_mutex);
+	__set_bit(I915_BO_ACTIVE_REF, &obj->flags);
+}
+
+static inline void
+i915_gem_object_clear_active_reference(struct drm_i915_gem_object *obj)
+{
+	lockdep_assert_held(&obj->base.dev->struct_mutex);
+	__clear_bit(I915_BO_ACTIVE_REF, &obj->flags);
+}
+
+void __i915_gem_object_release_unless_active(struct drm_i915_gem_object *obj);
+
 static inline unsigned int
 i915_gem_object_get_tiling(struct drm_i915_gem_object *obj)
 {
drivers/gpu/drm/i915/i915_gem.c +21 -1
index 7fa5cb764739..b560263bf446 100644
@@ -2618,7 +2618,10 @@ i915_gem_object_retire__read(struct i915_gem_active *active,
 		list_move_tail(&obj->global_list,
 			       &request->i915->mm.bound_list);
 
-	i915_gem_object_put(obj);
+	if (i915_gem_object_has_active_reference(obj)) {
+		i915_gem_object_clear_active_reference(obj);
+		i915_gem_object_put(obj);
+	}
 }
 
 static bool i915_context_is_banned(const struct i915_gem_context *ctx)
@@ -2889,6 +2892,12 @@ void i915_gem_close_object(struct drm_gem_object *gem, struct drm_file *file)
 	list_for_each_entry_safe(vma, vn, &obj->vma_list, obj_link)
 		if (vma->vm->file == fpriv)
 			i915_vma_close(vma);
+
+	if (i915_gem_object_is_active(obj) &&
+	    !i915_gem_object_has_active_reference(obj)) {
+		i915_gem_object_set_active_reference(obj);
+		i915_gem_object_get(obj);
+	}
 	mutex_unlock(&obj->base.dev->struct_mutex);
 }
 
@@ -4365,6 +4374,17 @@ void i915_gem_free_object(struct drm_gem_object *gem_obj)
 	intel_runtime_pm_put(dev_priv);
 }
 
+void __i915_gem_object_release_unless_active(struct drm_i915_gem_object *obj)
+{
+	lockdep_assert_held(&obj->base.dev->struct_mutex);
+
+	GEM_BUG_ON(i915_gem_object_has_active_reference(obj));
+	if (i915_gem_object_is_active(obj))
+		i915_gem_object_set_active_reference(obj);
+	else
+		i915_gem_object_put(obj);
+}
+
 int i915_gem_suspend(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = to_i915(dev);
drivers/gpu/drm/i915/i915_gem_batch_pool.c +1 -1
index ed989596d9a3..cb25cad3318c 100644
@@ -73,7 +73,7 @@ void i915_gem_batch_pool_fini(struct i915_gem_batch_pool *pool)
 		list_for_each_entry_safe(obj, next,
 					 &pool->cache_list[n],
 					 batch_pool_link)
-			i915_gem_object_put(obj);
+			__i915_gem_object_release_unless_active(obj);
 
 		INIT_LIST_HEAD(&pool->cache_list[n]);
 	}
drivers/gpu/drm/i915/i915_gem_context.c +1 -1
index df10f4e95736..1d2ab73a8f43 100644
@@ -155,7 +155,7 @@ void i915_gem_context_free(struct kref *ctx_ref)
 		if (ce->ring)
 			intel_ring_free(ce->ring);
 
-		i915_vma_put(ce->state);
+		__i915_gem_object_release_unless_active(ce->state->obj);
 	}
 
 	put_pid(ctx->pid);
drivers/gpu/drm/i915/i915_gem_execbuffer.c +0 -2
index 72c7c1855e70..0deecd4e3b6c 100644
@@ -1299,8 +1299,6 @@ void i915_vma_move_to_active(struct i915_vma *vma,
 	 * add the active reference first and queue for it to be dropped
 	 * *last*.
 	 */
-	if (!i915_gem_object_is_active(obj))
-		i915_gem_object_get(obj);
 	i915_gem_object_set_active(obj, idx);
 	i915_gem_active_set(&obj->last_read[idx], req);
 
drivers/gpu/drm/i915/i915_gem_gtt.c +6 -1
index 2d846aa39ca5..1c95da8424cb 100644
@@ -3712,11 +3712,16 @@ void __iomem *i915_vma_pin_iomap(struct i915_vma *vma)
 void i915_vma_unpin_and_release(struct i915_vma **p_vma)
 {
 	struct i915_vma *vma;
+	struct drm_i915_gem_object *obj;
 
 	vma = fetch_and_zero(p_vma);
 	if (!vma)
 		return;
 
+	obj = vma->obj;
+
 	i915_vma_unpin(vma);
-	i915_vma_put(vma);
+	i915_vma_close(vma);
+
+	__i915_gem_object_release_unless_active(obj);
 }
drivers/gpu/drm/i915/i915_gem_render_state.c +2 -1
index 95b7e9afd5f8..09cf4874c45f 100644
@@ -224,7 +224,8 @@ int i915_gem_render_state_init(struct drm_i915_gem_request *req)
 	i915_vma_move_to_active(so.vma, req, 0);
 err_unpin:
 	i915_vma_unpin(so.vma);
+	i915_vma_close(so.vma);
 err_obj:
-	i915_gem_object_put(obj);
+	__i915_gem_object_release_unless_active(obj);
 	return ret;
 }
drivers/gpu/drm/i915/intel_ringbuffer.c +12 -3
index b60c6f09fbfd..f3dfb7ca625d 100644
@@ -1763,14 +1763,19 @@ static void cleanup_phys_status_page(struct intel_engine_cs *engine)
 static void cleanup_status_page(struct intel_engine_cs *engine)
 {
 	struct i915_vma *vma;
+	struct drm_i915_gem_object *obj;
 
 	vma = fetch_and_zero(&engine->status_page.vma);
 	if (!vma)
 		return;
 
+	obj = vma->obj;
+
 	i915_vma_unpin(vma);
-	i915_gem_object_unpin_map(vma->obj);
-	i915_vma_put(vma);
+	i915_vma_close(vma);
+
+	i915_gem_object_unpin_map(obj);
+	__i915_gem_object_release_unless_active(obj);
 }
 
 static int init_status_page(struct intel_engine_cs *engine)
@@ -1968,7 +1973,11 @@ intel_engine_create_ring(struct intel_engine_cs *engine, int size)
 void
 intel_ring_free(struct intel_ring *ring)
 {
-	i915_vma_put(ring->vma);
+	struct drm_i915_gem_object *obj = ring->vma->obj;
+
+	i915_vma_close(ring->vma);
+	__i915_gem_object_release_unless_active(obj);
+
 	kfree(ring);
 }
 
Signature · 6 lines
2.9.3

_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
#23 lore raw

Re: [PATCH 10/42] drm/i915: Defer active reference until required

Quoted 4 lines · Chris Wilson

We only need the active reference to keep the object alive after the handle has been deleted (so as to prevent a synchronous gem_close). Why then pay the price of a kref on every execbuf when we can insert that final active ref just in time for the handle deletion?

I really dislike this. Where there was elegance with obj/vma_put, it is now replaced with out of place looking __i915_gem_object_release_unless_active. I don't see why would higher level layers have to concern themselves with calling something with such a low-level sounding name.

How much does this influence performance and in what cases? If significant, could we try to come up with something similar but more elegant?

Regards,

Tvrtko

Quoted 210 lines
---
  drivers/gpu/drm/i915/i915_drv.h              | 28 ++++++++++++++++++++++++++++
  drivers/gpu/drm/i915/i915_gem.c              | 22 +++++++++++++++++++++-
  drivers/gpu/drm/i915/i915_gem_batch_pool.c   |  2 +-
  drivers/gpu/drm/i915/i915_gem_context.c      |  2 +-
  drivers/gpu/drm/i915/i915_gem_execbuffer.c   |  2 --
  drivers/gpu/drm/i915/i915_gem_gtt.c          |  7 ++++++-
  drivers/gpu/drm/i915/i915_gem_render_state.c |  3 ++-
  drivers/gpu/drm/i915/intel_ringbuffer.c      | 15 ++++++++++++---
  8 files changed, 71 insertions(+), 10 deletions(-)
drivers/gpu/drm/i915/i915_drv.h +28 -0
index ee25e265416f..fee5cc92e2f2 100644
@@ -2232,6 +2232,12 @@ struct drm_i915_gem_object {
  	((READ_ONCE((bo)->flags) >> I915_BO_ACTIVE_SHIFT) & I915_BO_ACTIVE_MASK)
 
  	/**
+	 * Have we taken a reference for the object for incomplete GPU
+	 * activity?
+	 */
+#define I915_BO_ACTIVE_REF (I915_BO_ACTIVE_SHIFT + I915_NUM_ENGINES)
+
+	/**
  	 * This is set if the object has been written to since last bound
  	 * to the GTT
  	 */
@@ -2399,6 +2405,28 @@ i915_gem_object_has_active_engine(const struct drm_i915_gem_object *obj,
  	return obj->flags & BIT(engine + I915_BO_ACTIVE_SHIFT);
  }
 
+static inline bool
+i915_gem_object_has_active_reference(const struct drm_i915_gem_object *obj)
+{
+	return test_bit(I915_BO_ACTIVE_REF, &obj->flags);
+}
+
+static inline void
+i915_gem_object_set_active_reference(struct drm_i915_gem_object *obj)
+{
+	lockdep_assert_held(&obj->base.dev->struct_mutex);
+	__set_bit(I915_BO_ACTIVE_REF, &obj->flags);
+}
+
+static inline void
+i915_gem_object_clear_active_reference(struct drm_i915_gem_object *obj)
+{
+	lockdep_assert_held(&obj->base.dev->struct_mutex);
+	__clear_bit(I915_BO_ACTIVE_REF, &obj->flags);
+}
+
+void __i915_gem_object_release_unless_active(struct drm_i915_gem_object *obj);
+
  static inline unsigned int
  i915_gem_object_get_tiling(struct drm_i915_gem_object *obj)
  {
drivers/gpu/drm/i915/i915_gem.c +21 -1
index 7fa5cb764739..b560263bf446 100644
@@ -2618,7 +2618,10 @@ i915_gem_object_retire__read(struct i915_gem_active *active,
  		list_move_tail(&obj->global_list,
  			       &request->i915->mm.bound_list);
 
-	i915_gem_object_put(obj);
+	if (i915_gem_object_has_active_reference(obj)) {
+		i915_gem_object_clear_active_reference(obj);
+		i915_gem_object_put(obj);
+	}
  }
 
  static bool i915_context_is_banned(const struct i915_gem_context *ctx)
@@ -2889,6 +2892,12 @@ void i915_gem_close_object(struct drm_gem_object *gem, struct drm_file *file)
  	list_for_each_entry_safe(vma, vn, &obj->vma_list, obj_link)
  		if (vma->vm->file == fpriv)
  			i915_vma_close(vma);
+
+	if (i915_gem_object_is_active(obj) &&
+	    !i915_gem_object_has_active_reference(obj)) {
+		i915_gem_object_set_active_reference(obj);
+		i915_gem_object_get(obj);
+	}
  	mutex_unlock(&obj->base.dev->struct_mutex);
  }
 
@@ -4365,6 +4374,17 @@ void i915_gem_free_object(struct drm_gem_object *gem_obj)
  	intel_runtime_pm_put(dev_priv);
  }
 
+void __i915_gem_object_release_unless_active(struct drm_i915_gem_object *obj)
+{
+	lockdep_assert_held(&obj->base.dev->struct_mutex);
+
+	GEM_BUG_ON(i915_gem_object_has_active_reference(obj));
+	if (i915_gem_object_is_active(obj))
+		i915_gem_object_set_active_reference(obj);
+	else
+		i915_gem_object_put(obj);
+}
+
  int i915_gem_suspend(struct drm_device *dev)
  {
  	struct drm_i915_private *dev_priv = to_i915(dev);
drivers/gpu/drm/i915/i915_gem_batch_pool.c +1 -1
index ed989596d9a3..cb25cad3318c 100644
@@ -73,7 +73,7 @@ void i915_gem_batch_pool_fini(struct i915_gem_batch_pool *pool)
  		list_for_each_entry_safe(obj, next,
  					 &pool->cache_list[n],
  					 batch_pool_link)
-			i915_gem_object_put(obj);
+			__i915_gem_object_release_unless_active(obj);
 
  		INIT_LIST_HEAD(&pool->cache_list[n]);
  	}
drivers/gpu/drm/i915/i915_gem_context.c +1 -1
index df10f4e95736..1d2ab73a8f43 100644
@@ -155,7 +155,7 @@ void i915_gem_context_free(struct kref *ctx_ref)
  		if (ce->ring)
  			intel_ring_free(ce->ring);
 
-		i915_vma_put(ce->state);
+		__i915_gem_object_release_unless_active(ce->state->obj);
  	}
 
  	put_pid(ctx->pid);
drivers/gpu/drm/i915/i915_gem_execbuffer.c +0 -2
index 72c7c1855e70..0deecd4e3b6c 100644
@@ -1299,8 +1299,6 @@ void i915_vma_move_to_active(struct i915_vma *vma,
  	 * add the active reference first and queue for it to be dropped
  	 * *last*.
  	 */
-	if (!i915_gem_object_is_active(obj))
-		i915_gem_object_get(obj);
  	i915_gem_object_set_active(obj, idx);
  	i915_gem_active_set(&obj->last_read[idx], req);
 
drivers/gpu/drm/i915/i915_gem_gtt.c +6 -1
index 2d846aa39ca5..1c95da8424cb 100644
@@ -3712,11 +3712,16 @@ void __iomem *i915_vma_pin_iomap(struct i915_vma *vma)
  void i915_vma_unpin_and_release(struct i915_vma **p_vma)
  {
  	struct i915_vma *vma;
+	struct drm_i915_gem_object *obj;
 
  	vma = fetch_and_zero(p_vma);
  	if (!vma)
  		return;
 
+	obj = vma->obj;
+
  	i915_vma_unpin(vma);
-	i915_vma_put(vma);
+	i915_vma_close(vma);
+
+	__i915_gem_object_release_unless_active(obj);
  }
drivers/gpu/drm/i915/i915_gem_render_state.c +2 -1
index 95b7e9afd5f8..09cf4874c45f 100644
@@ -224,7 +224,8 @@ int i915_gem_render_state_init(struct drm_i915_gem_request *req)
  	i915_vma_move_to_active(so.vma, req, 0);
  err_unpin:
  	i915_vma_unpin(so.vma);
+	i915_vma_close(so.vma);
  err_obj:
-	i915_gem_object_put(obj);
+	__i915_gem_object_release_unless_active(obj);
  	return ret;
  }
drivers/gpu/drm/i915/intel_ringbuffer.c +7 -2
index b60c6f09fbfd..f3dfb7ca625d 100644
@@ -1763,14 +1763,19 @@ static void cleanup_phys_status_page(struct intel_engine_cs *engine)
  static void cleanup_status_page(struct intel_engine_cs *engine)
  {
  	struct i915_vma *vma;
+	struct drm_i915_gem_object *obj;
 
  	vma = fetch_and_zero(&engine->status_page.vma);
  	if (!vma)
  		return;
 
+	obj = vma->obj;
+
  	i915_vma_unpin(vma);
-	i915_gem_object_unpin_map(vma->obj);
-	i915_vma_put(vma);
+	i915_vma_close(vma);
+
+	i915_gem_object_unpin_map(obj);
+	__i915_gem_object_release_unless_active(obj);
  }
 
  static int init_status_page(struct intel_engine_cs *engine)
@@ -1968,7 +1973,11 @@ intel_engine_create_ring(struct intel_engine_cs *engine, int size)
  void
  intel_ring_free(struct intel_ring *ring)

drivers/gpu/drm/i915/intel_ringbuffer.c intel_engine_create_ring(struct intel_engine_cs *engine, int size) · line 1983

Quoted 10 lines
drivers/gpu/drm/i915/intel_ringbuffer.c +5 -1
  {
-	i915_vma_put(ring->vma);
+	struct drm_i915_gem_object *obj = ring->vma->obj;
+
+	i915_vma_close(ring->vma);
+	__i915_gem_object_release_unless_active(obj);
+
  	kfree(ring);
  }
 

_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx

#24 lore raw

Re: [PATCH 10/42] drm/i915: Defer active reference until required

Quoted 13 lines · Tvrtko Ursulin
Quoted 4 lines · Chris Wilson

We only need the active reference to keep the object alive after the handle has been deleted (so as to prevent a synchronous gem_close). Why then pay the price of a kref on every execbuf when we can insert that final active ref just in time for the handle deletion?

I really dislike this. Where there was elegance with obj/vma_put, it is now replaced with out of place looking __i915_gem_object_release_unless_active. I don't see why would higher level layers have to concern themselves with calling something with such a low-level sounding name.

Quoted 3 lines

How much does this influence performance and in what cases? If significant, could we try to come up with something similar but more elegant?

Back in the day, this was one of the most frequent atomic operations we did. And whilst perf overemphasizes the stalls from locked instructions, the sheer numbers of them we do are significant (since we do one at the start and end of every execbuf for every object in typical conditions). Whilst it is less significant in the face of obj->resv undoing all of the gains, it is still a deep paper cut. (At the GL level, consider about 100 objects per batch, several thousand times a second x 2, these ops are low hanging fruit.)

What's needed is a function to take the place of the close_object for internally allocated objects. It is also worth noting that they are either already part of a cache, or are suitable for caching.... -Chris

Quoted 210 lines

Regards,

Tvrtko

Quoted 204 lines
---
 drivers/gpu/drm/i915/i915_drv.h              | 28 ++++++++++++++++++++++++++++
 drivers/gpu/drm/i915/i915_gem.c              | 22 +++++++++++++++++++++-
 drivers/gpu/drm/i915/i915_gem_batch_pool.c   |  2 +-
 drivers/gpu/drm/i915/i915_gem_context.c      |  2 +-
 drivers/gpu/drm/i915/i915_gem_execbuffer.c   |  2 --
 drivers/gpu/drm/i915/i915_gem_gtt.c          |  7 ++++++-
 drivers/gpu/drm/i915/i915_gem_render_state.c |  3 ++-
 drivers/gpu/drm/i915/intel_ringbuffer.c      | 15 ++++++++++++---
 8 files changed, 71 insertions(+), 10 deletions(-)
drivers/gpu/drm/i915/i915_drv.h +28 -0
index ee25e265416f..fee5cc92e2f2 100644
@@ -2232,6 +2232,12 @@ struct drm_i915_gem_object {
 	((READ_ONCE((bo)->flags) >> I915_BO_ACTIVE_SHIFT) & I915_BO_ACTIVE_MASK)
 	/**
+	 * Have we taken a reference for the object for incomplete GPU
+	 * activity?
+	 */
+#define I915_BO_ACTIVE_REF (I915_BO_ACTIVE_SHIFT + I915_NUM_ENGINES)
+
+	/**
 	 * This is set if the object has been written to since last bound
 	 * to the GTT
 	 */
@@ -2399,6 +2405,28 @@ i915_gem_object_has_active_engine(const struct drm_i915_gem_object *obj,
 	return obj->flags & BIT(engine + I915_BO_ACTIVE_SHIFT);
 }
+static inline bool
+i915_gem_object_has_active_reference(const struct drm_i915_gem_object *obj)
+{
+	return test_bit(I915_BO_ACTIVE_REF, &obj->flags);
+}
+
+static inline void
+i915_gem_object_set_active_reference(struct drm_i915_gem_object *obj)
+{
+	lockdep_assert_held(&obj->base.dev->struct_mutex);
+	__set_bit(I915_BO_ACTIVE_REF, &obj->flags);
+}
+
+static inline void
+i915_gem_object_clear_active_reference(struct drm_i915_gem_object *obj)
+{
+	lockdep_assert_held(&obj->base.dev->struct_mutex);
+	__clear_bit(I915_BO_ACTIVE_REF, &obj->flags);
+}
+
+void __i915_gem_object_release_unless_active(struct drm_i915_gem_object *obj);
+
 static inline unsigned int
 i915_gem_object_get_tiling(struct drm_i915_gem_object *obj)
 {
drivers/gpu/drm/i915/i915_gem.c +21 -1
index 7fa5cb764739..b560263bf446 100644
@@ -2618,7 +2618,10 @@ i915_gem_object_retire__read(struct i915_gem_active *active,
 		list_move_tail(&obj->global_list,
 			       &request->i915->mm.bound_list);
-	i915_gem_object_put(obj);
+	if (i915_gem_object_has_active_reference(obj)) {
+		i915_gem_object_clear_active_reference(obj);
+		i915_gem_object_put(obj);
+	}
 }
 static bool i915_context_is_banned(const struct i915_gem_context *ctx)
@@ -2889,6 +2892,12 @@ void i915_gem_close_object(struct drm_gem_object *gem, struct drm_file *file)
 	list_for_each_entry_safe(vma, vn, &obj->vma_list, obj_link)
 		if (vma->vm->file == fpriv)
 			i915_vma_close(vma);
+
+	if (i915_gem_object_is_active(obj) &&
+	    !i915_gem_object_has_active_reference(obj)) {
+		i915_gem_object_set_active_reference(obj);
+		i915_gem_object_get(obj);
+	}
 	mutex_unlock(&obj->base.dev->struct_mutex);
 }
@@ -4365,6 +4374,17 @@ void i915_gem_free_object(struct drm_gem_object *gem_obj)
 	intel_runtime_pm_put(dev_priv);
 }
+void __i915_gem_object_release_unless_active(struct drm_i915_gem_object *obj)
+{
+	lockdep_assert_held(&obj->base.dev->struct_mutex);
+
+	GEM_BUG_ON(i915_gem_object_has_active_reference(obj));
+	if (i915_gem_object_is_active(obj))
+		i915_gem_object_set_active_reference(obj);
+	else
+		i915_gem_object_put(obj);
+}
+
 int i915_gem_suspend(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = to_i915(dev);
drivers/gpu/drm/i915/i915_gem_batch_pool.c +1 -1
index ed989596d9a3..cb25cad3318c 100644
@@ -73,7 +73,7 @@ void i915_gem_batch_pool_fini(struct i915_gem_batch_pool *pool)
 		list_for_each_entry_safe(obj, next,
 					 &pool->cache_list[n],
 					 batch_pool_link)
-			i915_gem_object_put(obj);
+			__i915_gem_object_release_unless_active(obj);
 		INIT_LIST_HEAD(&pool->cache_list[n]);
 	}
drivers/gpu/drm/i915/i915_gem_context.c +1 -1
index df10f4e95736..1d2ab73a8f43 100644
@@ -155,7 +155,7 @@ void i915_gem_context_free(struct kref *ctx_ref)
 		if (ce->ring)
 			intel_ring_free(ce->ring);
-		i915_vma_put(ce->state);
+		__i915_gem_object_release_unless_active(ce->state->obj);
 	}
 	put_pid(ctx->pid);
drivers/gpu/drm/i915/i915_gem_execbuffer.c +0 -2
index 72c7c1855e70..0deecd4e3b6c 100644
@@ -1299,8 +1299,6 @@ void i915_vma_move_to_active(struct i915_vma *vma,
 	 * add the active reference first and queue for it to be dropped
 	 * *last*.
 	 */
-	if (!i915_gem_object_is_active(obj))
-		i915_gem_object_get(obj);
 	i915_gem_object_set_active(obj, idx);
 	i915_gem_active_set(&obj->last_read[idx], req);
drivers/gpu/drm/i915/i915_gem_gtt.c +6 -1
index 2d846aa39ca5..1c95da8424cb 100644
@@ -3712,11 +3712,16 @@ void __iomem *i915_vma_pin_iomap(struct i915_vma *vma)
 void i915_vma_unpin_and_release(struct i915_vma **p_vma)
 {
 	struct i915_vma *vma;
+	struct drm_i915_gem_object *obj;
 	vma = fetch_and_zero(p_vma);
 	if (!vma)
 		return;
+	obj = vma->obj;
+
 	i915_vma_unpin(vma);
-	i915_vma_put(vma);
+	i915_vma_close(vma);
+
+	__i915_gem_object_release_unless_active(obj);
 }
drivers/gpu/drm/i915/i915_gem_render_state.c +2 -1
index 95b7e9afd5f8..09cf4874c45f 100644
@@ -224,7 +224,8 @@ int i915_gem_render_state_init(struct drm_i915_gem_request *req)
 	i915_vma_move_to_active(so.vma, req, 0);
 err_unpin:
 	i915_vma_unpin(so.vma);
+	i915_vma_close(so.vma);
 err_obj:
-	i915_gem_object_put(obj);
+	__i915_gem_object_release_unless_active(obj);
 	return ret;
 }
drivers/gpu/drm/i915/intel_ringbuffer.c +12 -3
index b60c6f09fbfd..f3dfb7ca625d 100644
@@ -1763,14 +1763,19 @@ static void cleanup_phys_status_page(struct intel_engine_cs *engine)
 static void cleanup_status_page(struct intel_engine_cs *engine)
 {
 	struct i915_vma *vma;
+	struct drm_i915_gem_object *obj;
 	vma = fetch_and_zero(&engine->status_page.vma);
 	if (!vma)
 		return;
+	obj = vma->obj;
+
 	i915_vma_unpin(vma);
-	i915_gem_object_unpin_map(vma->obj);
-	i915_vma_put(vma);
+	i915_vma_close(vma);
+
+	i915_gem_object_unpin_map(obj);
+	__i915_gem_object_release_unless_active(obj);
 }
 static int init_status_page(struct intel_engine_cs *engine)
@@ -1968,7 +1973,11 @@ intel_engine_create_ring(struct intel_engine_cs *engine, int size)
 void
 intel_ring_free(struct intel_ring *ring)
 {
-	i915_vma_put(ring->vma);
+	struct drm_i915_gem_object *obj = ring->vma->obj;
+
+	i915_vma_close(ring->vma);
+	__i915_gem_object_release_unless_active(obj);
+
 	kfree(ring);
 }
Signature · 5 lines
Chris Wilson, Intel Open Source Technology Centre
_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
#25 lore raw

Re: [PATCH 10/42] drm/i915: Defer active reference until required

Quoted 15 lines · Chris Wilson
Quoted 11 lines · Tvrtko Ursulin
Quoted 4 lines · Chris Wilson

We only need the active reference to keep the object alive after the handle has been deleted (so as to prevent a synchronous gem_close). Why then pay the price of a kref on every execbuf when we can insert that final active ref just in time for the handle deletion?

I really dislike this. Where there was elegance with obj/vma_put, it is now replaced with out of place looking __i915_gem_object_release_unless_active. I don't see why would higher level layers have to concern themselves with calling something with such a low-level sounding name.

Quoted 3 lines

How much does this influence performance and in what cases? If significant, could we try to come up with something similar but more elegant?

Quoted 8 lines

Back in the day, this was one of the most frequent atomic operations we did. And whilst perf overemphasizes the stalls from locked instructions, the sheer numbers of them we do are significant (since we do one at the start and end of every execbuf for every object in typical conditions). Whilst it is less significant in the face of obj->resv undoing all of the gains, it is still a deep paper cut. (At the GL level, consider about 100 objects per batch, several thousand times a second x 2, these ops are low hanging fruit.)

I understand there is a lot of them (same is true for many other operations we do), but how does that translate to some benchmarks?

Quoted 3 lines

What's needed is a function to take the place of the close_object for internally allocated objects. It is also worth noting that they are either already part of a cache, or are suitable for caching....

Ok but those are not x 100 per batch.

Regards,

Tvrtko

_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx

#26 lore raw

PATCH 11/42 drm/i915: Introduce an internal allocator for disposable private objects

In reply to Chris Wilson

Quite a few of our objects used for internal hardware programming do not benefit from being swappable or from being zero initialised. As such they do not benefit from using a shmemfs backing storage and since they are internal and never directly exposed to the user, we do not need to worry about providing a filp. For these we can use an drm_i915_gem_object wrapper around a sg_table of plain struct page. They are not swap backed and not automatically pinned. If they are reaped by the shrinker, the pages are released and the contents discarded. For the internal use case, this is fine as for example, ringbuffers are pinned from being written by a request to be read by the hardware. Once they are idle, they can be discarded entirely. As such they are a good match for execlist ringbuffers and a small variety of other internal objects.

In the first iteration, this is limited to the scratch batch buffers we use (for command parsing and state initialisation).

---
 drivers/gpu/drm/i915/Makefile                |   1 +
 drivers/gpu/drm/i915/i915_drv.h              |   5 +
 drivers/gpu/drm/i915/i915_gem_batch_pool.c   |  28 ++---
 drivers/gpu/drm/i915/i915_gem_internal.c     | 161 +++++++++++++++++++++++++++
 drivers/gpu/drm/i915/i915_gem_render_state.c |   2 +-
 drivers/gpu/drm/i915/intel_engine_cs.c       |   2 +-
 drivers/gpu/drm/i915/intel_ringbuffer.c      |  14 ++-
 7 files changed, 189 insertions(+), 24 deletions(-)
 create mode 100644 drivers/gpu/drm/i915/i915_gem_internal.c
drivers/gpu/drm/i915/Makefile +1 -0
index a998c2bce70a..b94a90f34d2d 100644
@@ -35,6 +35,7 @@ i915-y += i915_cmd_parser.o \
 	  i915_gem_execbuffer.o \
 	  i915_gem_fence.o \
 	  i915_gem_gtt.o \
+	  i915_gem_internal.o \
 	  i915_gem.o \
 	  i915_gem_render_state.o \
 	  i915_gem_request.o \
drivers/gpu/drm/i915/i915_drv.h +5 -0
index fee5cc92e2f2..bad97f1e5265 100644
@@ -3538,6 +3538,11 @@ i915_gem_object_create_stolen_for_preallocated(struct drm_device *dev,
 					       u32 gtt_offset,
 					       u32 size);
 
+/* i915_gem_internal.c */
+struct drm_i915_gem_object *
+i915_gem_object_create_internal(struct drm_device *dev,
+				unsigned int size);
+
 /* i915_gem_shrinker.c */
 unsigned long i915_gem_shrink(struct drm_i915_private *dev_priv,
 			      unsigned long target,
drivers/gpu/drm/i915/i915_gem_batch_pool.c +9 -19
index cb25cad3318c..3934c9103cf2 100644
@@ -97,9 +97,9 @@ i915_gem_batch_pool_get(struct i915_gem_batch_pool *pool,
 			size_t size)
 {
 	struct drm_i915_gem_object *obj = NULL;
-	struct drm_i915_gem_object *tmp, *next;
+	struct drm_i915_gem_object *tmp;
 	struct list_head *list;
-	int n;
+	int n, ret;
 
 	lockdep_assert_held(&pool->engine->i915->drm.struct_mutex);
 
@@ -112,19 +112,12 @@ i915_gem_batch_pool_get(struct i915_gem_batch_pool *pool,
 		n = ARRAY_SIZE(pool->cache_list) - 1;
 	list = &pool->cache_list[n];
 
-	list_for_each_entry_safe(tmp, next, list, batch_pool_link) {
+	list_for_each_entry(tmp, list, batch_pool_link) {
 		/* The batches are strictly LRU ordered */
 		if (!i915_gem_active_is_idle(&tmp->last_read[pool->engine->id],
 					     &tmp->base.dev->struct_mutex))
 			break;
 
-		/* While we're looping, do some clean up */
-		if (tmp->madv == __I915_MADV_PURGED) {
-			list_del(&tmp->batch_pool_link);
-			i915_gem_object_put(tmp);
-			continue;
-		}
-
 		if (tmp->base.size >= size) {
 			obj = tmp;
 			break;
@@ -132,19 +125,16 @@ i915_gem_batch_pool_get(struct i915_gem_batch_pool *pool,
 	}
 
 	if (obj == NULL) {
-		int ret;
-
-		obj = i915_gem_object_create(&pool->engine->i915->drm, size);
+		obj = i915_gem_object_create_internal(&pool->engine->i915->drm,
+						      size);
 		if (IS_ERR(obj))
 			return obj;
-
-		ret = i915_gem_object_get_pages(obj);
-		if (ret)
-			return ERR_PTR(ret);
-
-		obj->madv = I915_MADV_DONTNEED;
 	}
 
+	ret = i915_gem_object_get_pages(obj);
+	if (ret)
+		return ERR_PTR(ret);
+
 	list_move_tail(&obj->batch_pool_link, list);
 	i915_gem_object_pin_pages(obj);
 	return obj;
drivers/gpu/drm/i915/i915_gem_internal.c new file +161 -0

161 lines not on this page · raw message

drivers/gpu/drm/i915/i915_gem_render_state.c +1 -1
index 09cf4874c45f..c009ee1c27fd 100644
@@ -187,7 +187,7 @@ int i915_gem_render_state_init(struct drm_i915_gem_request *req)
 	if (so.rodata->batch_items * 4 > 4096)
 		return -EINVAL;
 
-	obj = i915_gem_object_create(&req->i915->drm, 4096);
+	obj = i915_gem_object_create_internal(&req->i915->drm, 4096);
 	if (IS_ERR(obj))
 		return PTR_ERR(obj);
 
drivers/gpu/drm/i915/intel_engine_cs.c +1 -1
index 480584c09306..99737b842d73 100644
@@ -251,7 +251,7 @@ int intel_engine_create_scratch(struct intel_engine_cs *engine, int size)
 
 	obj = i915_gem_object_create_stolen(&engine->i915->drm, size);
 	if (!obj)
-		obj = i915_gem_object_create(&engine->i915->drm, size);
+		obj = i915_gem_object_create_internal(&engine->i915->drm, size);
 	if (IS_ERR(obj)) {
 		DRM_ERROR("Failed to allocate scratch page\n");
 		return PTR_ERR(obj);
drivers/gpu/drm/i915/intel_ringbuffer.c +11 -3
index f3dfb7ca625d..b5d9c03f84ce 100644
@@ -1783,9 +1783,10 @@ static int init_status_page(struct intel_engine_cs *engine)
 	struct drm_i915_gem_object *obj;
 	struct i915_vma *vma;
 	unsigned int flags;
+	void *vaddr;
 	int ret;
 
-	obj = i915_gem_object_create(&engine->i915->drm, 4096);
+	obj = i915_gem_object_create_internal(&engine->i915->drm, 4096);
 	if (IS_ERR(obj)) {
 		DRM_ERROR("Failed to allocate status page\n");
 		return PTR_ERR(obj);
@@ -1818,15 +1819,22 @@ static int init_status_page(struct intel_engine_cs *engine)
 	if (ret)
 		goto err;
 
+	vaddr = i915_gem_object_pin_map(obj, I915_MAP_WB);
+	if (IS_ERR(vaddr)) {
+		ret = PTR_ERR(vaddr);
+		goto err_unpin;
+	}
+
 	engine->status_page.vma = vma;
 	engine->status_page.ggtt_offset = i915_ggtt_offset(vma);
-	engine->status_page.page_addr =
-		i915_gem_object_pin_map(obj, I915_MAP_WB);
+	engine->status_page.page_addr = memset(vaddr, 0, 4096);
 
 	DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n",
 			 engine->name, i915_ggtt_offset(vma));
 	return 0;
 
+err_unpin:
+	i915_vma_unpin(vma);
 err:
 	i915_gem_object_put(obj);
 	return ret;
Signature · 6 lines
2.9.3

_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
maintainer Reviewed-by on v1 11/42
#27 lore raw

Re: [PATCH 11/42] drm/i915: Introduce an internal allocator for disposable private objects

Quoted 14 lines · Chris Wilson

Quite a few of our objects used for internal hardware programming do not benefit from being swappable or from being zero initialised. As such they do not benefit from using a shmemfs backing storage and since they are internal and never directly exposed to the user, we do not need to worry about providing a filp. For these we can use an drm_i915_gem_object wrapper around a sg_table of plain struct page. They are not swap backed and not automatically pinned. If they are reaped by the shrinker, the pages are released and the contents discarded. For the internal use case, this is fine as for example, ringbuffers are pinned from being written by a request to be read by the hardware. Once they are idle, they can be discarded entirely. As such they are a good match for execlist ringbuffers and a small variety of other internal objects.

Quoted 3 lines

In the first iteration, this is limited to the scratch batch buffers we use (for command parsing and state initialisation).

No changelog, assuming otherwise intact but the typo fixed;

Regards, Joonas

Signature · 7 lines
Joonas Lahtinen
Open Source Technology Center
Intel Corporation
_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
#28 lore raw

Re: [PATCH 11/42] drm/i915: Introduce an internal allocator for disposable private objects

In reply to Chris Wilson
Quoted 46 lines · Chris Wilson

Quite a few of our objects used for internal hardware programming do not benefit from being swappable or from being zero initialised. As such they do not benefit from using a shmemfs backing storage and since they are internal and never directly exposed to the user, we do not need to worry about providing a filp. For these we can use an drm_i915_gem_object wrapper around a sg_table of plain struct page. They are not swap backed and not automatically pinned. If they are reaped by the shrinker, the pages are released and the contents discarded. For the internal use case, this is fine as for example, ringbuffers are pinned from being written by a request to be read by the hardware. Once they are idle, they can be discarded entirely. As such they are a good match for execlist ringbuffers and a small variety of other internal objects.

In the first iteration, this is limited to the scratch batch buffers we use (for command parsing and state initialisation).

---
  drivers/gpu/drm/i915/Makefile                |   1 +
  drivers/gpu/drm/i915/i915_drv.h              |   5 +
  drivers/gpu/drm/i915/i915_gem_batch_pool.c   |  28 ++---
  drivers/gpu/drm/i915/i915_gem_internal.c     | 161 +++++++++++++++++++++++++++
  drivers/gpu/drm/i915/i915_gem_render_state.c |   2 +-
  drivers/gpu/drm/i915/intel_engine_cs.c       |   2 +-
  drivers/gpu/drm/i915/intel_ringbuffer.c      |  14 ++-
  7 files changed, 189 insertions(+), 24 deletions(-)

create mode 100644 drivers/gpu/drm/i915/i915_gem_internal.c

drivers/gpu/drm/i915/Makefile +1 -0
index a998c2bce70a..b94a90f34d2d 100644
@@ -35,6 +35,7 @@ i915-y += i915_cmd_parser.o \
  	  i915_gem_execbuffer.o \
  	  i915_gem_fence.o \
  	  i915_gem_gtt.o \
+	  i915_gem_internal.o \
  	  i915_gem.o \
  	  i915_gem_render_state.o \
  	  i915_gem_request.o \
drivers/gpu/drm/i915/i915_drv.h +0 -0
index fee5cc92e2f2..bad97f1e5265 100644

drivers/gpu/drm/i915/i915_drv.h i915_gem_object_create_stolen_for_preallocated(struct drm_device *dev, · line 3545

Quoted 8 lines
drivers/gpu/drm/i915/i915_drv.h +5 -0
@@ -3538,6 +3538,11 @@ i915_gem_object_create_stolen_for_preallocated(struct drm_device *dev,
  					       u32 gtt_offset,
  					       u32 size);
 
+/* i915_gem_internal.c */
+struct drm_i915_gem_object *
+i915_gem_object_create_internal(struct drm_device *dev,
+				unsigned int size);
+

Wasn't size_t our convention for GEM objects?

Quoted 124 lines
drivers/gpu/drm/i915/i915_drv.h +0 -0
  /* i915_gem_shrinker.c */
  unsigned long i915_gem_shrink(struct drm_i915_private *dev_priv,
  			      unsigned long target,
drivers/gpu/drm/i915/i915_gem_batch_pool.c +9 -19
index cb25cad3318c..3934c9103cf2 100644
@@ -97,9 +97,9 @@ i915_gem_batch_pool_get(struct i915_gem_batch_pool *pool,
  			size_t size)
  {
  	struct drm_i915_gem_object *obj = NULL;
-	struct drm_i915_gem_object *tmp, *next;
+	struct drm_i915_gem_object *tmp;
  	struct list_head *list;
-	int n;
+	int n, ret;
 
  	lockdep_assert_held(&pool->engine->i915->drm.struct_mutex);
 
@@ -112,19 +112,12 @@ i915_gem_batch_pool_get(struct i915_gem_batch_pool *pool,
  		n = ARRAY_SIZE(pool->cache_list) - 1;
  	list = &pool->cache_list[n];
 
-	list_for_each_entry_safe(tmp, next, list, batch_pool_link) {
+	list_for_each_entry(tmp, list, batch_pool_link) {
  		/* The batches are strictly LRU ordered */
  		if (!i915_gem_active_is_idle(&tmp->last_read[pool->engine->id],
  					     &tmp->base.dev->struct_mutex))
  			break;
 
-		/* While we're looping, do some clean up */
-		if (tmp->madv == __I915_MADV_PURGED) {
-			list_del(&tmp->batch_pool_link);
-			i915_gem_object_put(tmp);
-			continue;
-		}
-
  		if (tmp->base.size >= size) {
  			obj = tmp;
  			break;
@@ -132,19 +125,16 @@ i915_gem_batch_pool_get(struct i915_gem_batch_pool *pool,
  	}
 
  	if (obj == NULL) {
-		int ret;
-
-		obj = i915_gem_object_create(&pool->engine->i915->drm, size);
+		obj = i915_gem_object_create_internal(&pool->engine->i915->drm,
+						      size);
  		if (IS_ERR(obj))
  			return obj;
-
-		ret = i915_gem_object_get_pages(obj);
-		if (ret)
-			return ERR_PTR(ret);
-
-		obj->madv = I915_MADV_DONTNEED;
  	}
 
+	ret = i915_gem_object_get_pages(obj);
+	if (ret)
+		return ERR_PTR(ret);
+
  	list_move_tail(&obj->batch_pool_link, list);
  	i915_gem_object_pin_pages(obj);
  	return obj;
drivers/gpu/drm/i915/i915_gem_internal.c new file +52 -0
new file mode 100644
index 000000000000..534a61c1aba2
@@ -0,0 +1,161 @@
+/*
+ * Copyright © 2014-2016 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ */
+
+#include <drm/drmP.h>
+#include <drm/i915_drm.h>
+#include "i915_drv.h"
+
+static void internal_free_pages(struct sg_table *st)
+{
+	struct sg_page_iter sg_iter;
+
+	for_each_sg_page(st->sgl, &sg_iter, st->nents, 0)
+		put_page(sg_page_iter_page(&sg_iter));
+
+	sg_free_table(st);
+	kfree(st);
+}
+
+static int i915_gem_object_get_pages_internal(struct drm_i915_gem_object *obj)
+{
+	const unsigned int npages = obj->base.size / PAGE_SIZE;
+	struct sg_table *st;
+	struct scatterlist *sg;
+	unsigned long last_pfn = 0;	/* suppress gcc warning */
+	gfp_t gfp;
+	int i;
+
+	st = kmalloc(sizeof(*st), GFP_KERNEL);
+	if (!st)
+		return -ENOMEM;
+

drivers/gpu/drm/i915/i915_gem_internal.c line 62

Quoted 10 lines
drivers/gpu/drm/i915/i915_gem_internal.c new file +10 -0
+	if (sg_alloc_table(st, npages, GFP_KERNEL)) {
+		kfree(st);
+		return -ENOMEM;
+	}
+
+	sg = st->sgl;
+	st->nents = 0;
+
+	gfp = GFP_KERNEL | __GFP_HIGHMEM | __GFP_RECLAIMABLE;
+	if (IS_CRESTLINE(obj->base.dev) || IS_BROADWATER(obj->base.dev)) {

/O\ Nooooo... dev_priv please! :)

Quoted 24 lines
drivers/gpu/drm/i915/i915_gem_internal.c new file +24 -0
+		/* 965gm cannot relocate objects above 4GiB. */
+		gfp &= ~__GFP_HIGHMEM;
+		gfp |= __GFP_DMA32;
+	}
+
+	for (i = 0; i < npages; i++) {
+		struct page *page;
+
+		page = alloc_page(gfp);
+		if (!page)
+			goto err;
+
+#ifdef CONFIG_SWIOTLB
+		if (swiotlb_nr_tbl()) {
+			st->nents++;
+			sg_set_page(sg, page, PAGE_SIZE, 0);
+			sg = sg_next(sg);
+			continue;
+		}
+#endif
+		if (!i || page_to_pfn(page) != last_pfn + 1) {
+			if (i)
+				sg = sg_next(sg);
+			st->nents++;

drivers/gpu/drm/i915/i915_gem_internal.c line 96

Quoted 10 lines
drivers/gpu/drm/i915/i915_gem_internal.c new file +10 -0
+			sg_set_page(sg, page, PAGE_SIZE, 0);
+		} else {
+			sg->length += PAGE_SIZE;
+		}
+		last_pfn = page_to_pfn(page);
+	}
+#ifdef CONFIG_SWIOTLB
+	if (!swiotlb_nr_tbl())
+#endif
+		sg_mark_end(sg);

Looks like the loop above could be moved into a helper and shared with i915_gem_object_get_pages_gtt. Maybe just a page-alloc and page-alloc-error callbacks would be required.

Quoted 39 lines
drivers/gpu/drm/i915/i915_gem_internal.c new file +39 -0
+	obj->pages = st;
+
+	if (i915_gem_gtt_prepare_object(obj)) {
+		obj->pages = NULL;
+		goto err;
+	}
+
+	/* Mark the pages as dontneed whilst they are still pinned. As soon
+	 * as they are unpinned they are allowed to be reaped by the shrinker,
+	 * and the caller is expected to repopulate - the contents of this
+	 * object are only valid whilst active and pinned.
+	 */
+	obj->madv = I915_MADV_DONTNEED;
+	return 0;
+
+err:
+	sg_mark_end(sg);
+	internal_free_pages(st);
+	return -ENOMEM;
+}
+
+static void i915_gem_object_put_pages_internal(struct drm_i915_gem_object *obj)
+{
+	internal_free_pages(obj->pages);
+
+	obj->dirty = 0;
+	obj->madv = I915_MADV_WILLNEED;
+}
+
+static const struct drm_i915_gem_object_ops i915_gem_object_internal_ops = {
+	.flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE,
+	.get_pages = i915_gem_object_get_pages_internal,
+	.put_pages = i915_gem_object_put_pages_internal,
+};
+
+/**
+ * Creates a new object that wraps some internal memory for private use.
+ * This object is not backed by swappable storage, and as such its contents
+ * are volatile and only valid whilst pinned. If the object is reaped by the

drivers/gpu/drm/i915/i915_gem_internal.c line 145

Quoted 10 lines
drivers/gpu/drm/i915/i915_gem_internal.c new file +10 -0
+ * shrinker, its pages and data will be discarded. Equally, it is not a full
+ * GEM object and so not valid for access from userspace. This makes it useful
+ * for hardware interfaces like ringbuffers (which are pinned from the time
+ * the request is written to the time the hardware stops accessing it), but
+ * not for contexts (which need to be preserved when not active for later
+ * reuse). Note that it is not cleared upon allocation.
+ */
+struct drm_i915_gem_object *
+i915_gem_object_create_internal(struct drm_device *dev,
+				unsigned int size)

size_t again.

drivers/gpu/drm/i915/i915_gem_internal.c line 158

Quoted 13 lines
drivers/gpu/drm/i915/i915_gem_internal.c new file +13 -0
+{
+	struct drm_i915_gem_object *obj;
+
+	obj = i915_gem_object_alloc(dev);
+	if (!obj)
+		return ERR_PTR(-ENOMEM);
+
+	drm_gem_private_object_init(dev, &obj->base, size);
+	i915_gem_object_init(obj, &i915_gem_object_internal_ops);
+
+	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
+	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
+	obj->cache_level = HAS_LLC(dev) ? I915_CACHE_LLC : I915_CACHE_NONE;

At least to_i915 would be helpful.

Quoted 46 lines
drivers/gpu/drm/i915/i915_gem_internal.c new file +3 -0
+
+	return obj;
+}
drivers/gpu/drm/i915/i915_gem_render_state.c +1 -1
index 09cf4874c45f..c009ee1c27fd 100644
@@ -187,7 +187,7 @@ int i915_gem_render_state_init(struct drm_i915_gem_request *req)
  	if (so.rodata->batch_items * 4 > 4096)
  		return -EINVAL;
 
-	obj = i915_gem_object_create(&req->i915->drm, 4096);
+	obj = i915_gem_object_create_internal(&req->i915->drm, 4096);
  	if (IS_ERR(obj))
  		return PTR_ERR(obj);
 
drivers/gpu/drm/i915/intel_engine_cs.c +1 -1
index 480584c09306..99737b842d73 100644
@@ -251,7 +251,7 @@ int intel_engine_create_scratch(struct intel_engine_cs *engine, int size)
 
  	obj = i915_gem_object_create_stolen(&engine->i915->drm, size);
  	if (!obj)
-		obj = i915_gem_object_create(&engine->i915->drm, size);
+		obj = i915_gem_object_create_internal(&engine->i915->drm, size);
  	if (IS_ERR(obj)) {
  		DRM_ERROR("Failed to allocate scratch page\n");
  		return PTR_ERR(obj);
drivers/gpu/drm/i915/intel_ringbuffer.c +2 -1
index f3dfb7ca625d..b5d9c03f84ce 100644
@@ -1783,9 +1783,10 @@ static int init_status_page(struct intel_engine_cs *engine)
  	struct drm_i915_gem_object *obj;
  	struct i915_vma *vma;
  	unsigned int flags;
+	void *vaddr;
  	int ret;
 
-	obj = i915_gem_object_create(&engine->i915->drm, 4096);
+	obj = i915_gem_object_create_internal(&engine->i915->drm, 4096);
  	if (IS_ERR(obj)) {
  		DRM_ERROR("Failed to allocate status page\n");
  		return PTR_ERR(obj);

drivers/gpu/drm/i915/intel_ringbuffer.c static int init_status_page(struct intel_engine_cs *engine) · line 1827

Quoted 9 lines
drivers/gpu/drm/i915/intel_ringbuffer.c +6 -0
@@ -1818,15 +1819,22 @@ static int init_status_page(struct intel_engine_cs *engine)
  	if (ret)
  		goto err;
 
+	vaddr = i915_gem_object_pin_map(obj, I915_MAP_WB);
+	if (IS_ERR(vaddr)) {
+		ret = PTR_ERR(vaddr);
+		goto err_unpin;
+	}
+

Unrelated fix, best if you could afford to extract it.

drivers/gpu/drm/i915/intel_ringbuffer.c static int init_status_page(struct intel_engine_cs *engine) · line 1831

Quoted 6 lines
drivers/gpu/drm/i915/intel_ringbuffer.c +1 -2
  	engine->status_page.vma = vma;
  	engine->status_page.ggtt_offset = i915_ggtt_offset(vma);
-	engine->status_page.page_addr =
-		i915_gem_object_pin_map(obj, I915_MAP_WB);
+	engine->status_page.page_addr = memset(vaddr, 0, 4096);
 

Very unusual, at least for me. :)

drivers/gpu/drm/i915/intel_ringbuffer.c static int init_status_page(struct intel_engine_cs *engine) · line 1840

Quoted 9 lines
drivers/gpu/drm/i915/intel_ringbuffer.c +2 -0
  	DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n",
  			 engine->name, i915_ggtt_offset(vma));
  	return 0;
 
+err_unpin:
+	i915_vma_unpin(vma);
  err:
  	i915_gem_object_put(obj);
  	return ret;

Regards,

Tvrtko

_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx

#29 lore raw

Re: [PATCH 11/42] drm/i915: Introduce an internal allocator for disposable private objects

Quoted 49 lines · Tvrtko Ursulin
Quoted 47 lines · Chris Wilson

Quite a few of our objects used for internal hardware programming do not benefit from being swappable or from being zero initialised. As such they do not benefit from using a shmemfs backing storage and since they are internal and never directly exposed to the user, we do not need to worry about providing a filp. For these we can use an drm_i915_gem_object wrapper around a sg_table of plain struct page. They are not swap backed and not automatically pinned. If they are reaped by the shrinker, the pages are released and the contents discarded. For the internal use case, this is fine as for example, ringbuffers are pinned from being written by a request to be read by the hardware. Once they are idle, they can be discarded entirely. As such they are a good match for execlist ringbuffers and a small variety of other internal objects.

In the first iteration, this is limited to the scratch batch buffers we use (for command parsing and state initialisation).

---
 drivers/gpu/drm/i915/Makefile                |   1 +
 drivers/gpu/drm/i915/i915_drv.h              |   5 +
 drivers/gpu/drm/i915/i915_gem_batch_pool.c   |  28 ++---
 drivers/gpu/drm/i915/i915_gem_internal.c     | 161 +++++++++++++++++++++++++++
 drivers/gpu/drm/i915/i915_gem_render_state.c |   2 +-
 drivers/gpu/drm/i915/intel_engine_cs.c       |   2 +-
 drivers/gpu/drm/i915/intel_ringbuffer.c      |  14 ++-
 7 files changed, 189 insertions(+), 24 deletions(-)
 create mode 100644 drivers/gpu/drm/i915/i915_gem_internal.c
drivers/gpu/drm/i915/Makefile +1 -0
index a998c2bce70a..b94a90f34d2d 100644
@@ -35,6 +35,7 @@ i915-y += i915_cmd_parser.o \
 	  i915_gem_execbuffer.o \
 	  i915_gem_fence.o \
 	  i915_gem_gtt.o \
+	  i915_gem_internal.o \
 	  i915_gem.o \
 	  i915_gem_render_state.o \
 	  i915_gem_request.o \
drivers/gpu/drm/i915/i915_drv.h +0 -0
index fee5cc92e2f2..bad97f1e5265 100644
@@ -3538,6 +3538,11 @@ i915_gem_object_create_stolen_for_preallocated(struct drm_device *dev,
 					       u32 gtt_offset,
Quoted 8 lines

drivers/gpu/drm/i915/i915_drv.h i915_gem_object_create_stolen_for_preallocated(struct drm_device *dev, · line 3545

Quoted 6 lines
drivers/gpu/drm/i915/i915_drv.h +5 -0
 					       u32 size);
+/* i915_gem_internal.c */
+struct drm_i915_gem_object *
+i915_gem_object_create_internal(struct drm_device *dev,
+				unsigned int size);
+

Wasn't size_t our convention for GEM objects?

Not our convention, no. Using size_t has caused too many bugs, that we started to erradicate it (in our usual piecemeal approach).

Quoted 30 lines
Quoted 30 lines
drivers/gpu/drm/i915/i915_drv.h +30 -0
+		/* 965gm cannot relocate objects above 4GiB. */
+		gfp &= ~__GFP_HIGHMEM;
+		gfp |= __GFP_DMA32;
+	}
+
+	for (i = 0; i < npages; i++) {
+		struct page *page;
+
+		page = alloc_page(gfp);
+		if (!page)
+			goto err;
+
+#ifdef CONFIG_SWIOTLB
+		if (swiotlb_nr_tbl()) {
+			st->nents++;
+			sg_set_page(sg, page, PAGE_SIZE, 0);
+			sg = sg_next(sg);
+			continue;
+		}
+#endif
+		if (!i || page_to_pfn(page) != last_pfn + 1) {
+			if (i)
+				sg = sg_next(sg);
+			st->nents++;
+			sg_set_page(sg, page, PAGE_SIZE, 0);
+		} else {
+			sg->length += PAGE_SIZE;
+		}
+		last_pfn = page_to_pfn(page);
+	}
Quoted 8 lines

drivers/gpu/drm/i915/i915_drv.h

Quoted 4 lines
drivers/gpu/drm/i915/i915_drv.h +4 -0
+#ifdef CONFIG_SWIOTLB
+	if (!swiotlb_nr_tbl())
+#endif
+		sg_mark_end(sg);

Looks like the loop above could be moved into a helper and shared with i915_gem_object_get_pages_gtt. Maybe just a page-alloc and page-alloc-error callbacks would be required.

So just the entire thing as a callback... I would have thought you might suggest trying high order allocations and falling back to low order. -Chirs

Signature · 5 lines
Chris Wilson, Intel Open Source Technology Centre
_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
#30 lore raw

Re: [PATCH 11/42] drm/i915: Introduce an internal allocator for disposable private objects

Quoted 56 lines · Chris Wilson
Quoted 47 lines · Tvrtko Ursulin
Quoted 46 lines · Chris Wilson

Quite a few of our objects used for internal hardware programming do not benefit from being swappable or from being zero initialised. As such they do not benefit from using a shmemfs backing storage and since they are internal and never directly exposed to the user, we do not need to worry about providing a filp. For these we can use an drm_i915_gem_object wrapper around a sg_table of plain struct page. They are not swap backed and not automatically pinned. If they are reaped by the shrinker, the pages are released and the contents discarded. For the internal use case, this is fine as for example, ringbuffers are pinned from being written by a request to be read by the hardware. Once they are idle, they can be discarded entirely. As such they are a good match for execlist ringbuffers and a small variety of other internal objects.

In the first iteration, this is limited to the scratch batch buffers we use (for command parsing and state initialisation).

---
  drivers/gpu/drm/i915/Makefile                |   1 +
  drivers/gpu/drm/i915/i915_drv.h              |   5 +
  drivers/gpu/drm/i915/i915_gem_batch_pool.c   |  28 ++---
  drivers/gpu/drm/i915/i915_gem_internal.c     | 161 +++++++++++++++++++++++++++
  drivers/gpu/drm/i915/i915_gem_render_state.c |   2 +-
  drivers/gpu/drm/i915/intel_engine_cs.c       |   2 +-
  drivers/gpu/drm/i915/intel_ringbuffer.c      |  14 ++-
  7 files changed, 189 insertions(+), 24 deletions(-)

create mode 100644 drivers/gpu/drm/i915/i915_gem_internal.c

drivers/gpu/drm/i915/Makefile +1 -0
index a998c2bce70a..b94a90f34d2d 100644
@@ -35,6 +35,7 @@ i915-y += i915_cmd_parser.o \
  	  i915_gem_execbuffer.o \
  	  i915_gem_fence.o \
  	  i915_gem_gtt.o \
+	  i915_gem_internal.o \
  	  i915_gem.o \
  	  i915_gem_render_state.o \
  	  i915_gem_request.o \
drivers/gpu/drm/i915/i915_drv.h +0 -0
index fee5cc92e2f2..bad97f1e5265 100644
@@ -3538,6 +3538,11 @@ i915_gem_object_create_stolen_for_preallocated(struct drm_device *dev,
Quoted 8 lines

drivers/gpu/drm/i915/i915_drv.h i915_gem_object_create_stolen_for_preallocated(struct drm_device *dev, · line 3545

Quoted 7 lines
drivers/gpu/drm/i915/i915_drv.h +5 -0
  					       u32 gtt_offset,
  					       u32 size);
+/* i915_gem_internal.c */
+struct drm_i915_gem_object *
+i915_gem_object_create_internal(struct drm_device *dev,
+				unsigned int size);
+

Wasn't size_t our convention for GEM objects?

Quoted 2 lines

Not our convention, no. Using size_t has caused too many bugs, that we started to erradicate it (in our usual piecemeal approach).

Oh wow, I missed that decision. Last thing I remember was that someone was trying to convert it all to size_t. Fine by me.

Quoted 37 lines
Quoted 29 lines
Quoted 29 lines
drivers/gpu/drm/i915/i915_drv.h +29 -0
+		/* 965gm cannot relocate objects above 4GiB. */
+		gfp &= ~__GFP_HIGHMEM;
+		gfp |= __GFP_DMA32;
+	}
+
+	for (i = 0; i < npages; i++) {
+		struct page *page;
+
+		page = alloc_page(gfp);
+		if (!page)
+			goto err;
+
+#ifdef CONFIG_SWIOTLB
+		if (swiotlb_nr_tbl()) {
+			st->nents++;
+			sg_set_page(sg, page, PAGE_SIZE, 0);
+			sg = sg_next(sg);
+			continue;
+		}
+#endif
+		if (!i || page_to_pfn(page) != last_pfn + 1) {
+			if (i)
+				sg = sg_next(sg);
+			st->nents++;
+			sg_set_page(sg, page, PAGE_SIZE, 0);
+		} else {
+			sg->length += PAGE_SIZE;
+		}
+		last_pfn = page_to_pfn(page);
Quoted 8 lines

drivers/gpu/drm/i915/i915_drv.h

Quoted 5 lines
drivers/gpu/drm/i915/i915_drv.h +5 -0
+	}
+#ifdef CONFIG_SWIOTLB
+	if (!swiotlb_nr_tbl())
+#endif
+		sg_mark_end(sg);

Looks like the loop above could be moved into a helper and shared with i915_gem_object_get_pages_gtt. Maybe just a page-alloc and page-alloc-error callbacks would be required.

Quoted 2 lines

So just the entire thing as a callback... I would have thought you might suggest trying high order allocations and falling back to low order.

I am not sure higher order attempts would be worth it. What I was thinking was to implement the loop above generically in one place, and then two (or even three with userptr) callers would call that with their own alloc-page and alloc-page-on-error callbacks. So that there is a single implementation of the coalescing logic etc.

Regards,

Tvrtko

_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx

#31 lore raw

Re: [PATCH 11/42] drm/i915: Introduce an internal allocator for disposable private objects

Quoted 58 lines · Tvrtko Ursulin
Quoted 56 lines · Chris Wilson
Quoted 55 lines · Tvrtko Ursulin
Quoted 46 lines · Chris Wilson

Quite a few of our objects used for internal hardware programming do not benefit from being swappable or from being zero initialised. As such they do not benefit from using a shmemfs backing storage and since they are internal and never directly exposed to the user, we do not need to worry about providing a filp. For these we can use an drm_i915_gem_object wrapper around a sg_table of plain struct page. They are not swap backed and not automatically pinned. If they are reaped by the shrinker, the pages are released and the contents discarded. For the internal use case, this is fine as for example, ringbuffers are pinned from being written by a request to be read by the hardware. Once they are idle, they can be discarded entirely. As such they are a good match for execlist ringbuffers and a small variety of other internal objects.

In the first iteration, this is limited to the scratch batch buffers we use (for command parsing and state initialisation).

---
 drivers/gpu/drm/i915/Makefile                |   1 +
 drivers/gpu/drm/i915/i915_drv.h              |   5 +
 drivers/gpu/drm/i915/i915_gem_batch_pool.c   |  28 ++---
 drivers/gpu/drm/i915/i915_gem_internal.c     | 161 +++++++++++++++++++++++++++
 drivers/gpu/drm/i915/i915_gem_render_state.c |   2 +-
 drivers/gpu/drm/i915/intel_engine_cs.c       |   2 +-
 drivers/gpu/drm/i915/intel_ringbuffer.c      |  14 ++-
 7 files changed, 189 insertions(+), 24 deletions(-)
 create mode 100644 drivers/gpu/drm/i915/i915_gem_internal.c
drivers/gpu/drm/i915/Makefile +1 -0
index a998c2bce70a..b94a90f34d2d 100644
@@ -35,6 +35,7 @@ i915-y += i915_cmd_parser.o \
 	  i915_gem_execbuffer.o \
 	  i915_gem_fence.o \
 	  i915_gem_gtt.o \
+	  i915_gem_internal.o \
 	  i915_gem.o \
 	  i915_gem_render_state.o \
 	  i915_gem_request.o \
drivers/gpu/drm/i915/i915_drv.h +0 -0
index fee5cc92e2f2..bad97f1e5265 100644

drivers/gpu/drm/i915/i915_drv.h i915_gem_object_create_stolen_for_preallocated(struct drm_device *dev, · line 3544

Quoted 7 lines
drivers/gpu/drm/i915/i915_drv.h +5 -0
@@ -3538,6 +3538,11 @@ i915_gem_object_create_stolen_for_preallocated(struct drm_device *dev,
 					       u32 gtt_offset,
 					       u32 size);
+/* i915_gem_internal.c */
+struct drm_i915_gem_object *
+i915_gem_object_create_internal(struct drm_device *dev,
+				unsigned int size);
+

Wasn't size_t our convention for GEM objects?

Quoted 5 lines
Quoted 2 lines

Not our convention, no. Using size_t has caused too many bugs, that we started to erradicate it (in our usual piecemeal approach).

Oh wow, I missed that decision. Last thing I remember was that someone was trying to convert it all to size_t. Fine by me.

Who? They probably didn't have to spend the time fixing size_t variables being wrong on 32bit.

Quoted 37 lines
Quoted 37 lines
Quoted 37 lines
Quoted 24 lines
drivers/gpu/drm/i915/i915_drv.h +24 -0
+		/* 965gm cannot relocate objects above 4GiB. */
+		gfp &= ~__GFP_HIGHMEM;
+		gfp |= __GFP_DMA32;
+	}
+
+	for (i = 0; i < npages; i++) {
+		struct page *page;
+
+		page = alloc_page(gfp);
+		if (!page)
+			goto err;
+
+#ifdef CONFIG_SWIOTLB
+		if (swiotlb_nr_tbl()) {
+			st->nents++;
+			sg_set_page(sg, page, PAGE_SIZE, 0);
+			sg = sg_next(sg);
+			continue;
+		}
+#endif
+		if (!i || page_to_pfn(page) != last_pfn + 1) {
+			if (i)
+				sg = sg_next(sg);
+			st->nents++;

drivers/gpu/drm/i915/i915_drv.h i915_gem_object_create_stolen_for_preallocated(struct drm_device *dev,

Quoted 10 lines
drivers/gpu/drm/i915/i915_drv.h +10 -0
+			sg_set_page(sg, page, PAGE_SIZE, 0);
+		} else {
+			sg->length += PAGE_SIZE;
+		}
+		last_pfn = page_to_pfn(page);
+	}
+#ifdef CONFIG_SWIOTLB
+	if (!swiotlb_nr_tbl())
+#endif
+		sg_mark_end(sg);

Looks like the loop above could be moved into a helper and shared with i915_gem_object_get_pages_gtt. Maybe just a page-alloc and page-alloc-error callbacks would be required.

Quoted 4 lines
Quoted 2 lines

So just the entire thing as a callback... I would have thought you might suggest trying high order allocations and falling back to low order.

I am not sure higher order attempts would be worth it.

Looks like it is. :)
-Chris

Signature · 5 lines
Chris Wilson, Intel Open Source Technology Centre
_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
#32 lore raw

PATCH v2 1/1 drm/i915: Introduce an internal allocator for disposable private objects

In reply to Tvrtko Ursulin

Quite a few of our objects used for internal hardware programming do not benefit from being swappable or from being zero initialised. As such they do not benefit from using a shmemfs backing storage and since they are internal and never directly exposed to the user, we do not need to worry about providing a filp. For these we can use an drm_i915_gem_object wrapper around a sg_table of plain struct page. They are not swap backed and not automatically pinned. If they are reaped by the shrinker, the pages are released and the contents discarded. For the internal use case, this is fine as for example, ringbuffers are pinned from being written by a request to be read by the hardware. Once they are idle, they can be discarded entirely. As such they are a good match for execlist ringbuffers and a small variety of other internal objects.

In the first iteration, this is limited to the scratch batch buffers we use (for command parsing and state initialisation).

v2: Allocate physically contiguous pages, where possible.

Cc Tvrtko Ursulin <[email protected]>
---
 drivers/gpu/drm/i915/Makefile                |   1 +
 drivers/gpu/drm/i915/i915_drv.h              |   5 +
 drivers/gpu/drm/i915/i915_gem_batch_pool.c   |  28 ++---
 drivers/gpu/drm/i915/i915_gem_internal.c     | 162 +++++++++++++++++++++++++++
 drivers/gpu/drm/i915/i915_gem_render_state.c |   2 +-
 drivers/gpu/drm/i915/intel_engine_cs.c       |   2 +-
 drivers/gpu/drm/i915/intel_ringbuffer.c      |  14 ++-
 7 files changed, 190 insertions(+), 24 deletions(-)
 create mode 100644 drivers/gpu/drm/i915/i915_gem_internal.c
drivers/gpu/drm/i915/Makefile +1 -0
index a998c2bce70a..b94a90f34d2d 100644
@@ -35,6 +35,7 @@ i915-y += i915_cmd_parser.o \
 	  i915_gem_execbuffer.o \
 	  i915_gem_fence.o \
 	  i915_gem_gtt.o \
+	  i915_gem_internal.o \
 	  i915_gem.o \
 	  i915_gem_render_state.o \
 	  i915_gem_request.o \
drivers/gpu/drm/i915/i915_drv.h +5 -0
index fee5cc92e2f2..bad97f1e5265 100644
@@ -3538,6 +3538,11 @@ i915_gem_object_create_stolen_for_preallocated(struct drm_device *dev,
 					       u32 gtt_offset,
 					       u32 size);
 
+/* i915_gem_internal.c */
+struct drm_i915_gem_object *
+i915_gem_object_create_internal(struct drm_device *dev,
+				unsigned int size);
+
 /* i915_gem_shrinker.c */
 unsigned long i915_gem_shrink(struct drm_i915_private *dev_priv,
 			      unsigned long target,
drivers/gpu/drm/i915/i915_gem_batch_pool.c +9 -19
index cb25cad3318c..3934c9103cf2 100644
@@ -97,9 +97,9 @@ i915_gem_batch_pool_get(struct i915_gem_batch_pool *pool,
 			size_t size)
 {
 	struct drm_i915_gem_object *obj = NULL;
-	struct drm_i915_gem_object *tmp, *next;
+	struct drm_i915_gem_object *tmp;
 	struct list_head *list;
-	int n;
+	int n, ret;
 
 	lockdep_assert_held(&pool->engine->i915->drm.struct_mutex);
 
@@ -112,19 +112,12 @@ i915_gem_batch_pool_get(struct i915_gem_batch_pool *pool,
 		n = ARRAY_SIZE(pool->cache_list) - 1;
 	list = &pool->cache_list[n];
 
-	list_for_each_entry_safe(tmp, next, list, batch_pool_link) {
+	list_for_each_entry(tmp, list, batch_pool_link) {
 		/* The batches are strictly LRU ordered */
 		if (!i915_gem_active_is_idle(&tmp->last_read[pool->engine->id],
 					     &tmp->base.dev->struct_mutex))
 			break;
 
-		/* While we're looping, do some clean up */
-		if (tmp->madv == __I915_MADV_PURGED) {
-			list_del(&tmp->batch_pool_link);
-			i915_gem_object_put(tmp);
-			continue;
-		}
-
 		if (tmp->base.size >= size) {
 			obj = tmp;
 			break;
@@ -132,19 +125,16 @@ i915_gem_batch_pool_get(struct i915_gem_batch_pool *pool,
 	}
 
 	if (obj == NULL) {
-		int ret;
-
-		obj = i915_gem_object_create(&pool->engine->i915->drm, size);
+		obj = i915_gem_object_create_internal(&pool->engine->i915->drm,
+						      size);
 		if (IS_ERR(obj))
 			return obj;
-
-		ret = i915_gem_object_get_pages(obj);
-		if (ret)
-			return ERR_PTR(ret);
-
-		obj->madv = I915_MADV_DONTNEED;
 	}
 
+	ret = i915_gem_object_get_pages(obj);
+	if (ret)
+		return ERR_PTR(ret);
+
 	list_move_tail(&obj->batch_pool_link, list);
 	i915_gem_object_pin_pages(obj);
 	return obj;
drivers/gpu/drm/i915/i915_gem_internal.c new file +162 -0

162 lines not on this page · raw message

drivers/gpu/drm/i915/i915_gem_render_state.c +1 -1
index 09cf4874c45f..c009ee1c27fd 100644
@@ -187,7 +187,7 @@ int i915_gem_render_state_init(struct drm_i915_gem_request *req)
 	if (so.rodata->batch_items * 4 > 4096)
 		return -EINVAL;
 
-	obj = i915_gem_object_create(&req->i915->drm, 4096);
+	obj = i915_gem_object_create_internal(&req->i915->drm, 4096);
 	if (IS_ERR(obj))
 		return PTR_ERR(obj);
 
drivers/gpu/drm/i915/intel_engine_cs.c +1 -1
index 480584c09306..99737b842d73 100644
@@ -251,7 +251,7 @@ int intel_engine_create_scratch(struct intel_engine_cs *engine, int size)
 
 	obj = i915_gem_object_create_stolen(&engine->i915->drm, size);
 	if (!obj)
-		obj = i915_gem_object_create(&engine->i915->drm, size);
+		obj = i915_gem_object_create_internal(&engine->i915->drm, size);
 	if (IS_ERR(obj)) {
 		DRM_ERROR("Failed to allocate scratch page\n");
 		return PTR_ERR(obj);
drivers/gpu/drm/i915/intel_ringbuffer.c +11 -3
index f3dfb7ca625d..b5d9c03f84ce 100644
@@ -1783,9 +1783,10 @@ static int init_status_page(struct intel_engine_cs *engine)
 	struct drm_i915_gem_object *obj;
 	struct i915_vma *vma;
 	unsigned int flags;
+	void *vaddr;
 	int ret;
 
-	obj = i915_gem_object_create(&engine->i915->drm, 4096);
+	obj = i915_gem_object_create_internal(&engine->i915->drm, 4096);
 	if (IS_ERR(obj)) {
 		DRM_ERROR("Failed to allocate status page\n");
 		return PTR_ERR(obj);
@@ -1818,15 +1819,22 @@ static int init_status_page(struct intel_engine_cs *engine)
 	if (ret)
 		goto err;
 
+	vaddr = i915_gem_object_pin_map(obj, I915_MAP_WB);
+	if (IS_ERR(vaddr)) {
+		ret = PTR_ERR(vaddr);
+		goto err_unpin;
+	}
+
 	engine->status_page.vma = vma;
 	engine->status_page.ggtt_offset = i915_ggtt_offset(vma);
-	engine->status_page.page_addr =
-		i915_gem_object_pin_map(obj, I915_MAP_WB);
+	engine->status_page.page_addr = memset(vaddr, 0, 4096);
 
 	DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n",
 			 engine->name, i915_ggtt_offset(vma));
 	return 0;
 
+err_unpin:
+	i915_vma_unpin(vma);
 err:
 	i915_gem_object_put(obj);
 	return ret;
Signature · 6 lines
2.9.3

_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
maintainer Reviewed-by on v2 1/1
#33 lore raw

Re: [PATCH v2] drm/i915: Introduce an internal allocator for disposable private objects

Quoted 14 lines · Chris Wilson

Quite a few of our objects used for internal hardware programming do not benefit from being swappable or from being zero initialised. As such they do not benefit from using a shmemfs backing storage and since they are internal and never directly exposed to the user, we do not need to worry about providing a filp. For these we can use an drm_i915_gem_object wrapper around a sg_table of plain struct page. They are not swap backed and not automatically pinned. If they are reaped by the shrinker, the pages are released and the contents discarded. For the internal use case, this is fine as for example, ringbuffers are pinned from being written by a request to be read by the hardware. Once they are idle, they can be discarded entirely. As such they are a good match for execlist ringbuffers and a small variety of other internal objects.

Quoted 4 lines

In the first iteration, this is limited to the scratch batch buffers we use (for command parsing and state initialisation).

v2: Allocate physically contiguous pages, where possible.

Does not hurt, but what does it gain us, exactly?

Quoted 3 lines
Cc Tvrtko Ursulin <[email protected]>

Regards, Joonas

Signature · 7 lines
Joonas Lahtinen
Open Source Technology Center
Intel Corporation
_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
#34 lore raw

Re: [PATCH v2] drm/i915: Introduce an internal allocator for disposable private objects

In reply to Chris Wilson
Quoted 14 lines · Chris Wilson

Quite a few of our objects used for internal hardware programming do not benefit from being swappable or from being zero initialised. As such they do not benefit from using a shmemfs backing storage and since they are internal and never directly exposed to the user, we do not need to worry about providing a filp. For these we can use an drm_i915_gem_object wrapper around a sg_table of plain struct page. They are not swap backed and not automatically pinned. If they are reaped by the shrinker, the pages are released and the contents discarded. For the internal use case, this is fine as for example, ringbuffers are pinned from being written by a request to be read by the hardware. Once they are idle, they can be discarded entirely. As such they are a good match for execlist ringbuffers and a small variety of other internal objects.

Quoted 4 lines

In the first iteration, this is limited to the scratch batch buffers we use (for command parsing and state initialisation).

v2: Allocate physically contiguous pages, where possible.

Since the allocator will be used constantly at runtime, my recollection is that higher order allocations can easily become next to impossible, so I am wondering why? Also, on your last reply you did not write why you think this is interesting to try?

Quoted 136 lines
Cc Tvrtko Ursulin <[email protected]>
---
  drivers/gpu/drm/i915/Makefile                |   1 +
  drivers/gpu/drm/i915/i915_drv.h              |   5 +
  drivers/gpu/drm/i915/i915_gem_batch_pool.c   |  28 ++---
  drivers/gpu/drm/i915/i915_gem_internal.c     | 162 +++++++++++++++++++++++++++
  drivers/gpu/drm/i915/i915_gem_render_state.c |   2 +-
  drivers/gpu/drm/i915/intel_engine_cs.c       |   2 +-
  drivers/gpu/drm/i915/intel_ringbuffer.c      |  14 ++-
  7 files changed, 190 insertions(+), 24 deletions(-)

create mode 100644 drivers/gpu/drm/i915/i915_gem_internal.c

drivers/gpu/drm/i915/Makefile +1 -0
index a998c2bce70a..b94a90f34d2d 100644
@@ -35,6 +35,7 @@ i915-y += i915_cmd_parser.o \
  	  i915_gem_execbuffer.o \
  	  i915_gem_fence.o \
  	  i915_gem_gtt.o \
+	  i915_gem_internal.o \
  	  i915_gem.o \
  	  i915_gem_render_state.o \
  	  i915_gem_request.o \
drivers/gpu/drm/i915/i915_drv.h +5 -0
index fee5cc92e2f2..bad97f1e5265 100644
@@ -3538,6 +3538,11 @@ i915_gem_object_create_stolen_for_preallocated(struct drm_device *dev,
  					       u32 gtt_offset,
  					       u32 size);
 
+/* i915_gem_internal.c */
+struct drm_i915_gem_object *
+i915_gem_object_create_internal(struct drm_device *dev,
+				unsigned int size);
+
  /* i915_gem_shrinker.c */
  unsigned long i915_gem_shrink(struct drm_i915_private *dev_priv,
  			      unsigned long target,
drivers/gpu/drm/i915/i915_gem_batch_pool.c +9 -19
index cb25cad3318c..3934c9103cf2 100644
@@ -97,9 +97,9 @@ i915_gem_batch_pool_get(struct i915_gem_batch_pool *pool,
  			size_t size)
  {
  	struct drm_i915_gem_object *obj = NULL;
-	struct drm_i915_gem_object *tmp, *next;
+	struct drm_i915_gem_object *tmp;
  	struct list_head *list;
-	int n;
+	int n, ret;
 
  	lockdep_assert_held(&pool->engine->i915->drm.struct_mutex);
 
@@ -112,19 +112,12 @@ i915_gem_batch_pool_get(struct i915_gem_batch_pool *pool,
  		n = ARRAY_SIZE(pool->cache_list) - 1;
  	list = &pool->cache_list[n];
 
-	list_for_each_entry_safe(tmp, next, list, batch_pool_link) {
+	list_for_each_entry(tmp, list, batch_pool_link) {
  		/* The batches are strictly LRU ordered */
  		if (!i915_gem_active_is_idle(&tmp->last_read[pool->engine->id],
  					     &tmp->base.dev->struct_mutex))
  			break;
 
-		/* While we're looping, do some clean up */
-		if (tmp->madv == __I915_MADV_PURGED) {
-			list_del(&tmp->batch_pool_link);
-			i915_gem_object_put(tmp);
-			continue;
-		}
-
  		if (tmp->base.size >= size) {
  			obj = tmp;
  			break;
@@ -132,19 +125,16 @@ i915_gem_batch_pool_get(struct i915_gem_batch_pool *pool,
  	}
 
  	if (obj == NULL) {
-		int ret;
-
-		obj = i915_gem_object_create(&pool->engine->i915->drm, size);
+		obj = i915_gem_object_create_internal(&pool->engine->i915->drm,
+						      size);
  		if (IS_ERR(obj))
  			return obj;
-
-		ret = i915_gem_object_get_pages(obj);
-		if (ret)
-			return ERR_PTR(ret);
-
-		obj->madv = I915_MADV_DONTNEED;
  	}
 
+	ret = i915_gem_object_get_pages(obj);
+	if (ret)
+		return ERR_PTR(ret);
+
  	list_move_tail(&obj->batch_pool_link, list);
  	i915_gem_object_pin_pages(obj);
  	return obj;
drivers/gpu/drm/i915/i915_gem_internal.c new file +26 -0
new file mode 100644
index 000000000000..255b9232b8ef
@@ -0,0 +1,162 @@
+/*
+ * Copyright © 2014-2016 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ */
+
+#include <drm/drmP.h>
+#include <drm/i915_drm.h>

drivers/gpu/drm/i915/i915_gem_internal.c line 36

Quoted 10 lines
drivers/gpu/drm/i915/i915_gem_internal.c new file +10 -0
+#include "i915_drv.h"
+
+#define QUIET (__GFP_NORETRY | __GFP_NOWARN)
+
+static void internal_free_pages(struct sg_table *st)
+{
+	struct scatterlist *sg;
+
+	for (sg = st->sgl; sg; sg = __sg_next(sg))
+		__free_pages(sg_page(sg), get_order(sg->length));

Fragile since wont work together with coalescing, which I am sad to see not implemented. For some reason it makes me feel real good when it is there.

Quoted 23 lines
drivers/gpu/drm/i915/i915_gem_internal.c new file +23 -0
+
+	sg_free_table(st);
+	kfree(st);
+}
+
+static int i915_gem_object_get_pages_internal(struct drm_i915_gem_object *obj)
+{
+	unsigned int npages = obj->base.size / PAGE_SIZE;
+	struct sg_table *st;
+	struct scatterlist *sg;
+	int max_order;
+	gfp_t gfp;
+
+	st = kmalloc(sizeof(*st), GFP_KERNEL);
+	if (!st)
+		return -ENOMEM;
+
+	if (sg_alloc_table(st, npages, GFP_KERNEL)) {
+		kfree(st);
+		return -ENOMEM;
+	}
+
+	sg = st->sgl;

drivers/gpu/drm/i915/i915_gem_internal.c line 69

Quoted 10 lines
drivers/gpu/drm/i915/i915_gem_internal.c new file +10 -0
+	st->nents = 0;
+
+	max_order = MAX_ORDER;
+#ifdef CONFIG_SWIOTLB
+	if (swiotlb_nr_tbl())
+		max_order = min(max_order, ilog2(IO_TLB_SEGSIZE));
+#endif
+
+	gfp = GFP_KERNEL | __GFP_HIGHMEM | __GFP_RECLAIMABLE;
+	if (IS_CRESTLINE(obj->base.dev) || IS_BROADWATER(obj->base.dev)) {

You don't want to avoid more usages of dev?

Quoted 6 lines
drivers/gpu/drm/i915/i915_gem_internal.c new file +6 -0
+		/* 965gm cannot relocate objects above 4GiB. */
+		gfp &= ~__GFP_HIGHMEM;
+		gfp |= __GFP_DMA32;
+	}
+
+	do {

drivers/gpu/drm/i915/i915_gem_internal.c line 85

Quoted 10 lines
drivers/gpu/drm/i915/i915_gem_internal.c new file +10 -0
+		int order = min(fls(npages) - 1, max_order);
+		struct page *page;
+
+		do {
+			page = alloc_pages(gfp | (order ? QUIET : 0), order);
+			if (page)
+				break;
+			if (!order--)
+				goto err;
+		} while (1);

Feels like it could hammer hard on a fragmented system, I have big concerns about this.

Quoted 120 lines
drivers/gpu/drm/i915/i915_gem_internal.c new file +77 -0
+
+		sg_set_page(sg, page, PAGE_SIZE << order, 0);
+		st->nents++;
+
+		npages -= 1 << order;
+		if (!npages) {
+			sg_mark_end(sg);
+			break;
+		}
+
+		sg = __sg_next(sg);
+	} while (1);
+	obj->pages = st;
+
+	if (i915_gem_gtt_prepare_object(obj)) {
+		obj->pages = NULL;
+		goto err;
+	}
+
+	/* Mark the pages as dontneed whilst they are still pinned. As soon
+	 * as they are unpinned they are allowed to be reaped by the shrinker,
+	 * and the caller is expected to repopulate - the contents of this
+	 * object are only valid whilst active and pinned.
+	 */
+	obj->madv = I915_MADV_DONTNEED;
+	return 0;
+
+err:
+	sg_mark_end(sg);
+	internal_free_pages(st);
+	return -ENOMEM;
+}
+
+static void i915_gem_object_put_pages_internal(struct drm_i915_gem_object *obj)
+{
+	internal_free_pages(obj->pages);
+
+	obj->dirty = 0;
+	obj->madv = I915_MADV_WILLNEED;
+}
+
+static const struct drm_i915_gem_object_ops i915_gem_object_internal_ops = {
+	.flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE,
+	.get_pages = i915_gem_object_get_pages_internal,
+	.put_pages = i915_gem_object_put_pages_internal,
+};
+
+/**
+ * Creates a new object that wraps some internal memory for private use.
+ * This object is not backed by swappable storage, and as such its contents
+ * are volatile and only valid whilst pinned. If the object is reaped by the
+ * shrinker, its pages and data will be discarded. Equally, it is not a full
+ * GEM object and so not valid for access from userspace. This makes it useful
+ * for hardware interfaces like ringbuffers (which are pinned from the time
+ * the request is written to the time the hardware stops accessing it), but
+ * not for contexts (which need to be preserved when not active for later
+ * reuse). Note that it is not cleared upon allocation.
+ */
+struct drm_i915_gem_object *
+i915_gem_object_create_internal(struct drm_device *dev,
+				unsigned int size)
+{
+	struct drm_i915_gem_object *obj;
+
+	obj = i915_gem_object_alloc(dev);
+	if (!obj)
+		return ERR_PTR(-ENOMEM);
+
+	drm_gem_private_object_init(dev, &obj->base, size);
+	i915_gem_object_init(obj, &i915_gem_object_internal_ops);
+
+	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
+	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
+	obj->cache_level = HAS_LLC(dev) ? I915_CACHE_LLC : I915_CACHE_NONE;
+
+	return obj;
+}
drivers/gpu/drm/i915/i915_gem_render_state.c +1 -1
index 09cf4874c45f..c009ee1c27fd 100644
@@ -187,7 +187,7 @@ int i915_gem_render_state_init(struct drm_i915_gem_request *req)
  	if (so.rodata->batch_items * 4 > 4096)
  		return -EINVAL;
 
-	obj = i915_gem_object_create(&req->i915->drm, 4096);
+	obj = i915_gem_object_create_internal(&req->i915->drm, 4096);
  	if (IS_ERR(obj))
  		return PTR_ERR(obj);
 
drivers/gpu/drm/i915/intel_engine_cs.c +1 -1
index 480584c09306..99737b842d73 100644
@@ -251,7 +251,7 @@ int intel_engine_create_scratch(struct intel_engine_cs *engine, int size)
 
  	obj = i915_gem_object_create_stolen(&engine->i915->drm, size);
  	if (!obj)
-		obj = i915_gem_object_create(&engine->i915->drm, size);
+		obj = i915_gem_object_create_internal(&engine->i915->drm, size);
  	if (IS_ERR(obj)) {
  		DRM_ERROR("Failed to allocate scratch page\n");
  		return PTR_ERR(obj);
drivers/gpu/drm/i915/intel_ringbuffer.c +2 -1
index f3dfb7ca625d..b5d9c03f84ce 100644
@@ -1783,9 +1783,10 @@ static int init_status_page(struct intel_engine_cs *engine)
  	struct drm_i915_gem_object *obj;
  	struct i915_vma *vma;
  	unsigned int flags;
+	void *vaddr;
  	int ret;
 
-	obj = i915_gem_object_create(&engine->i915->drm, 4096);
+	obj = i915_gem_object_create_internal(&engine->i915->drm, 4096);
  	if (IS_ERR(obj)) {
  		DRM_ERROR("Failed to allocate status page\n");
  		return PTR_ERR(obj);

drivers/gpu/drm/i915/intel_ringbuffer.c static int init_status_page(struct intel_engine_cs *engine) · line 1826

Quoted 8 lines
drivers/gpu/drm/i915/intel_ringbuffer.c +5 -0
@@ -1818,15 +1819,22 @@ static int init_status_page(struct intel_engine_cs *engine)
  	if (ret)
  		goto err;
 
+	vaddr = i915_gem_object_pin_map(obj, I915_MAP_WB);
+	if (IS_ERR(vaddr)) {
+		ret = PTR_ERR(vaddr);
+		goto err_unpin;
+	}

And you don't want to split the bugfix out?

Quoted 6 lines
drivers/gpu/drm/i915/intel_ringbuffer.c +2 -2
+
  	engine->status_page.vma = vma;
  	engine->status_page.ggtt_offset = i915_ggtt_offset(vma);
-	engine->status_page.page_addr =
-		i915_gem_object_pin_map(obj, I915_MAP_WB);
+	engine->status_page.page_addr = memset(vaddr, 0, 4096);

drivers/gpu/drm/i915/intel_ringbuffer.c static int init_status_page(struct intel_engine_cs *engine) · line 1840

Quoted 10 lines
drivers/gpu/drm/i915/intel_ringbuffer.c +2 -0
 
  	DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n",
  			 engine->name, i915_ggtt_offset(vma));
  	return 0;
 
+err_unpin:
+	i915_vma_unpin(vma);
  err:
  	i915_gem_object_put(obj);
  	return ret;

Regards,

Tvrtko

_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx

#35 lore raw

Re: [PATCH v2] drm/i915: Introduce an internal allocator for disposable private objects

Quoted 160 lines · Tvrtko Ursulin
Quoted 14 lines · Chris Wilson

Quite a few of our objects used for internal hardware programming do not benefit from being swappable or from being zero initialised. As such they do not benefit from using a shmemfs backing storage and since they are internal and never directly exposed to the user, we do not need to worry about providing a filp. For these we can use an drm_i915_gem_object wrapper around a sg_table of plain struct page. They are not swap backed and not automatically pinned. If they are reaped by the shrinker, the pages are released and the contents discarded. For the internal use case, this is fine as for example, ringbuffers are pinned from being written by a request to be read by the hardware. Once they are idle, they can be discarded entirely. As such they are a good match for execlist ringbuffers and a small variety of other internal objects.

Quoted 4 lines

In the first iteration, this is limited to the scratch batch buffers we use (for command parsing and state initialisation).

v2: Allocate physically contiguous pages, where possible.

Since the allocator will be used constantly at runtime, my recollection is that higher order allocations can easily become next to impossible, so I am wondering why? Also, on your last reply you did not write why you think this is interesting to try?

Quoted 135 lines
Cc Tvrtko Ursulin <[email protected]>
---
 drivers/gpu/drm/i915/Makefile                |   1 +
 drivers/gpu/drm/i915/i915_drv.h              |   5 +
 drivers/gpu/drm/i915/i915_gem_batch_pool.c   |  28 ++---
 drivers/gpu/drm/i915/i915_gem_internal.c     | 162 +++++++++++++++++++++++++++
 drivers/gpu/drm/i915/i915_gem_render_state.c |   2 +-
 drivers/gpu/drm/i915/intel_engine_cs.c       |   2 +-
 drivers/gpu/drm/i915/intel_ringbuffer.c      |  14 ++-
 7 files changed, 190 insertions(+), 24 deletions(-)
 create mode 100644 drivers/gpu/drm/i915/i915_gem_internal.c
drivers/gpu/drm/i915/Makefile +1 -0
index a998c2bce70a..b94a90f34d2d 100644
@@ -35,6 +35,7 @@ i915-y += i915_cmd_parser.o \
 	  i915_gem_execbuffer.o \
 	  i915_gem_fence.o \
 	  i915_gem_gtt.o \
+	  i915_gem_internal.o \
 	  i915_gem.o \
 	  i915_gem_render_state.o \
 	  i915_gem_request.o \
drivers/gpu/drm/i915/i915_drv.h +5 -0
index fee5cc92e2f2..bad97f1e5265 100644
@@ -3538,6 +3538,11 @@ i915_gem_object_create_stolen_for_preallocated(struct drm_device *dev,
 					       u32 gtt_offset,
 					       u32 size);
+/* i915_gem_internal.c */
+struct drm_i915_gem_object *
+i915_gem_object_create_internal(struct drm_device *dev,
+				unsigned int size);
+
 /* i915_gem_shrinker.c */
 unsigned long i915_gem_shrink(struct drm_i915_private *dev_priv,
 			      unsigned long target,
drivers/gpu/drm/i915/i915_gem_batch_pool.c +9 -19
index cb25cad3318c..3934c9103cf2 100644
@@ -97,9 +97,9 @@ i915_gem_batch_pool_get(struct i915_gem_batch_pool *pool,
 			size_t size)
 {
 	struct drm_i915_gem_object *obj = NULL;
-	struct drm_i915_gem_object *tmp, *next;
+	struct drm_i915_gem_object *tmp;
 	struct list_head *list;
-	int n;
+	int n, ret;
 	lockdep_assert_held(&pool->engine->i915->drm.struct_mutex);
@@ -112,19 +112,12 @@ i915_gem_batch_pool_get(struct i915_gem_batch_pool *pool,
 		n = ARRAY_SIZE(pool->cache_list) - 1;
 	list = &pool->cache_list[n];
-	list_for_each_entry_safe(tmp, next, list, batch_pool_link) {
+	list_for_each_entry(tmp, list, batch_pool_link) {
 		/* The batches are strictly LRU ordered */
 		if (!i915_gem_active_is_idle(&tmp->last_read[pool->engine->id],
 					     &tmp->base.dev->struct_mutex))
 			break;
-		/* While we're looping, do some clean up */
-		if (tmp->madv == __I915_MADV_PURGED) {
-			list_del(&tmp->batch_pool_link);
-			i915_gem_object_put(tmp);
-			continue;
-		}
-
 		if (tmp->base.size >= size) {
 			obj = tmp;
 			break;
@@ -132,19 +125,16 @@ i915_gem_batch_pool_get(struct i915_gem_batch_pool *pool,
 	}
 	if (obj == NULL) {
-		int ret;
-
-		obj = i915_gem_object_create(&pool->engine->i915->drm, size);
+		obj = i915_gem_object_create_internal(&pool->engine->i915->drm,
+						      size);
 		if (IS_ERR(obj))
 			return obj;
-
-		ret = i915_gem_object_get_pages(obj);
-		if (ret)
-			return ERR_PTR(ret);
-
-		obj->madv = I915_MADV_DONTNEED;
 	}
+	ret = i915_gem_object_get_pages(obj);
+	if (ret)
+		return ERR_PTR(ret);
+
 	list_move_tail(&obj->batch_pool_link, list);
 	i915_gem_object_pin_pages(obj);
 	return obj;
drivers/gpu/drm/i915/i915_gem_internal.c new file +32 -0
new file mode 100644
index 000000000000..255b9232b8ef
@@ -0,0 +1,162 @@
+/*
+ * Copyright © 2014-2016 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ */
+
+#include <drm/drmP.h>
+#include <drm/i915_drm.h>
+#include "i915_drv.h"
+
+#define QUIET (__GFP_NORETRY | __GFP_NOWARN)
+
+static void internal_free_pages(struct sg_table *st)
+{
Quoted 8 lines

drivers/gpu/drm/i915/i915_gem_internal.c line 36

Quoted 4 lines
drivers/gpu/drm/i915/i915_gem_internal.c new file +4 -0
+	struct scatterlist *sg;
+
+	for (sg = st->sgl; sg; sg = __sg_next(sg))
+		__free_pages(sg_page(sg), get_order(sg->length));

Fragile since wont work together with coalescing, which I am sad to see not implemented. For some reason it makes me feel real good when it is there.

We do the coalescing.

Quoted 10 lines

drivers/gpu/drm/i915/i915_gem_internal.c line 85

Quoted 7 lines
drivers/gpu/drm/i915/i915_gem_internal.c new file +7 -0
+		do {
+			page = alloc_pages(gfp | (order ? QUIET : 0), order);
+			if (page)
+				break;
+			if (!order--)
+				goto err;
+		} while (1);

Feels like it could hammer hard on a fragmented system, I have big concerns about this.

You did notice that these were marked as temporary allocations (because they are reclaimed under memory pressure), and on an already fragemented system we do the right thing anyway?
-Chris

Signature · 5 lines
Chris Wilson, Intel Open Source Technology Centre
_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
#36 lore raw

Re: [PATCH v2] drm/i915: Introduce an internal allocator for disposable private objects

Quoted 167 lines · Chris Wilson
Quoted 158 lines · Tvrtko Ursulin
Quoted 14 lines · Chris Wilson

Quite a few of our objects used for internal hardware programming do not benefit from being swappable or from being zero initialised. As such they do not benefit from using a shmemfs backing storage and since they are internal and never directly exposed to the user, we do not need to worry about providing a filp. For these we can use an drm_i915_gem_object wrapper around a sg_table of plain struct page. They are not swap backed and not automatically pinned. If they are reaped by the shrinker, the pages are released and the contents discarded. For the internal use case, this is fine as for example, ringbuffers are pinned from being written by a request to be read by the hardware. Once they are idle, they can be discarded entirely. As such they are a good match for execlist ringbuffers and a small variety of other internal objects.

Quoted 4 lines

In the first iteration, this is limited to the scratch batch buffers we use (for command parsing and state initialisation).

v2: Allocate physically contiguous pages, where possible.

Since the allocator will be used constantly at runtime, my recollection is that higher order allocations can easily become next to impossible, so I am wondering why? Also, on your last reply you did not write why you think this is interesting to try?

Quoted 134 lines
Cc Tvrtko Ursulin <[email protected]>
---
  drivers/gpu/drm/i915/Makefile                |   1 +
  drivers/gpu/drm/i915/i915_drv.h              |   5 +
  drivers/gpu/drm/i915/i915_gem_batch_pool.c   |  28 ++---
  drivers/gpu/drm/i915/i915_gem_internal.c     | 162 +++++++++++++++++++++++++++
  drivers/gpu/drm/i915/i915_gem_render_state.c |   2 +-
  drivers/gpu/drm/i915/intel_engine_cs.c       |   2 +-
  drivers/gpu/drm/i915/intel_ringbuffer.c      |  14 ++-
  7 files changed, 190 insertions(+), 24 deletions(-)

create mode 100644 drivers/gpu/drm/i915/i915_gem_internal.c

drivers/gpu/drm/i915/Makefile +1 -0
index a998c2bce70a..b94a90f34d2d 100644
@@ -35,6 +35,7 @@ i915-y += i915_cmd_parser.o \
  	  i915_gem_execbuffer.o \
  	  i915_gem_fence.o \
  	  i915_gem_gtt.o \
+	  i915_gem_internal.o \
  	  i915_gem.o \
  	  i915_gem_render_state.o \
  	  i915_gem_request.o \
drivers/gpu/drm/i915/i915_drv.h +5 -0
index fee5cc92e2f2..bad97f1e5265 100644
@@ -3538,6 +3538,11 @@ i915_gem_object_create_stolen_for_preallocated(struct drm_device *dev,
  					       u32 gtt_offset,
  					       u32 size);
+/* i915_gem_internal.c */
+struct drm_i915_gem_object *
+i915_gem_object_create_internal(struct drm_device *dev,
+				unsigned int size);
+
  /* i915_gem_shrinker.c */
  unsigned long i915_gem_shrink(struct drm_i915_private *dev_priv,
  			      unsigned long target,
drivers/gpu/drm/i915/i915_gem_batch_pool.c +9 -19
index cb25cad3318c..3934c9103cf2 100644
@@ -97,9 +97,9 @@ i915_gem_batch_pool_get(struct i915_gem_batch_pool *pool,
  			size_t size)
  {
  	struct drm_i915_gem_object *obj = NULL;
-	struct drm_i915_gem_object *tmp, *next;
+	struct drm_i915_gem_object *tmp;
  	struct list_head *list;
-	int n;
+	int n, ret;
  	lockdep_assert_held(&pool->engine->i915->drm.struct_mutex);
@@ -112,19 +112,12 @@ i915_gem_batch_pool_get(struct i915_gem_batch_pool *pool,
  		n = ARRAY_SIZE(pool->cache_list) - 1;
  	list = &pool->cache_list[n];
-	list_for_each_entry_safe(tmp, next, list, batch_pool_link) {
+	list_for_each_entry(tmp, list, batch_pool_link) {
  		/* The batches are strictly LRU ordered */
  		if (!i915_gem_active_is_idle(&tmp->last_read[pool->engine->id],
  					     &tmp->base.dev->struct_mutex))
  			break;
-		/* While we're looping, do some clean up */
-		if (tmp->madv == __I915_MADV_PURGED) {
-			list_del(&tmp->batch_pool_link);
-			i915_gem_object_put(tmp);
-			continue;
-		}
-
  		if (tmp->base.size >= size) {
  			obj = tmp;
  			break;
@@ -132,19 +125,16 @@ i915_gem_batch_pool_get(struct i915_gem_batch_pool *pool,
  	}
  	if (obj == NULL) {
-		int ret;
-
-		obj = i915_gem_object_create(&pool->engine->i915->drm, size);
+		obj = i915_gem_object_create_internal(&pool->engine->i915->drm,
+						      size);
  		if (IS_ERR(obj))
  			return obj;
-
-		ret = i915_gem_object_get_pages(obj);
-		if (ret)
-			return ERR_PTR(ret);
-
-		obj->madv = I915_MADV_DONTNEED;
  	}
+	ret = i915_gem_object_get_pages(obj);
+	if (ret)
+		return ERR_PTR(ret);
+
  	list_move_tail(&obj->batch_pool_link, list);
  	i915_gem_object_pin_pages(obj);
  	return obj;
drivers/gpu/drm/i915/i915_gem_internal.c new file +31 -0
new file mode 100644
index 000000000000..255b9232b8ef
@@ -0,0 +1,162 @@
+/*
+ * Copyright © 2014-2016 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ */
+
+#include <drm/drmP.h>
+#include <drm/i915_drm.h>
+#include "i915_drv.h"
+
+#define QUIET (__GFP_NORETRY | __GFP_NOWARN)
+
+static void internal_free_pages(struct sg_table *st)
Quoted 8 lines

drivers/gpu/drm/i915/i915_gem_internal.c line 36

Quoted 5 lines
drivers/gpu/drm/i915/i915_gem_internal.c new file +5 -0
+{
+	struct scatterlist *sg;
+
+	for (sg = st->sgl; sg; sg = __sg_next(sg))
+		__free_pages(sg_page(sg), get_order(sg->length));

Fragile since wont work together with coalescing, which I am sad to see not implemented. For some reason it makes me feel real good when it is there.

Quoted 1 line

We do the coalescing.

Where? It is not in this patch? One allocation == one st->nents++. If the next allocation is contiguous there is no coalescing.

Quoted 9 lines
Quoted 9 lines

drivers/gpu/drm/i915/i915_gem_internal.c line 85

Quoted 7 lines
drivers/gpu/drm/i915/i915_gem_internal.c new file +7 -0
+		do {
+			page = alloc_pages(gfp | (order ? QUIET : 0), order);
+			if (page)
+				break;
+			if (!order--)
+				goto err;
+		} while (1);

Feels like it could hammer hard on a fragmented system, I have big concerns about this.

Quoted 3 lines

You did notice that these were marked as temporary allocations (because they are reclaimed under memory pressure), and on an already fragemented system we do the right thing anyway?

You try smaller and then for the next allocation start from the initial order which did not work previously.

Regards,

Tvrtko

_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx

#37 lore raw

PATCH 12/42 drm/i915: Reuse the active golden render state batch

In reply to Chris Wilson

The golden render state is constant, but we recreate the batch setting it up for every new context. If we keep that batch in a volatile cache we can safely reuse it whenever we need to initialise a new context. We mark the pages as purgeable and use the shrinker to recover pages from the batch whenever we face memory pressues, recreating that batch afresh on the next new context.

---
 drivers/gpu/drm/i915/i915_gem_render_state.c | 184 +++++++++++++++++----------
 drivers/gpu/drm/i915/i915_gem_render_state.h |   4 +-
 drivers/gpu/drm/i915/intel_engine_cs.c       |   5 +
 drivers/gpu/drm/i915/intel_lrc.c             |   2 +-
 drivers/gpu/drm/i915/intel_ringbuffer.c      |   2 +-
 drivers/gpu/drm/i915/intel_ringbuffer.h      |   3 +
 6 files changed, 129 insertions(+), 71 deletions(-)
drivers/gpu/drm/i915/i915_gem_render_state.c +116 -68
index c009ee1c27fd..fc3421c59906 100644
@@ -28,17 +28,19 @@
 #include "i915_drv.h"
 #include "intel_renderstate.h"
 
-struct render_state {
+struct intel_render_state {
 	const struct intel_renderstate_rodata *rodata;
 	struct i915_vma *vma;
-	u32 aux_batch_size;
-	u32 aux_batch_offset;
+	u32 batch_offset;
+	u32 batch_size;
+	u32 aux_offset;
+	u32 aux_size;
 };
 
 static const struct intel_renderstate_rodata *
-render_state_get_rodata(const struct drm_i915_gem_request *req)
+render_state_get_rodata(const struct intel_engine_cs *engine)
 {
-	switch (INTEL_GEN(req->i915)) {
+	switch (INTEL_GEN(engine->i915)) {
 	case 6:
 		return &gen6_null_state;
 	case 7:
@@ -63,29 +65,27 @@ render_state_get_rodata(const struct drm_i915_gem_request *req)
  */
 #define OUT_BATCH(batch, i, val)				\
 	do {							\
-		if (WARN_ON((i) >= PAGE_SIZE / sizeof(u32))) {	\
-			ret = -ENOSPC;				\
-			goto err_out;				\
-		}						\
+		if ((i) >= PAGE_SIZE / sizeof(u32))		\
+			goto err;				\
 		(batch)[(i)++] = (val);				\
 	} while(0)
 
-static int render_state_setup(struct render_state *so)
+static int render_state_setup(struct intel_render_state *so,
+			      struct drm_i915_private *i915)
 {
-	struct drm_device *dev = so->vma->vm->dev;
 	const struct intel_renderstate_rodata *rodata = so->rodata;
-	const bool has_64bit_reloc = INTEL_GEN(dev) >= 8;
+	const bool has_64bit_reloc = INTEL_GEN(i915) >= 8;
+	struct drm_i915_gem_object *obj = so->vma->obj;
 	unsigned int i = 0, reloc_index = 0;
-	struct page *page;
+	unsigned int needs_clflush;
 	u32 *d;
 	int ret;
 
-	ret = i915_gem_object_set_to_cpu_domain(so->vma->obj, true);
+	ret = i915_gem_obj_prepare_shmem_write(obj, &needs_clflush);
 	if (ret)
 		return ret;
 
-	page = i915_gem_object_get_dirty_page(so->vma->obj, 0);
-	d = kmap(page);
+	d = kmap_atomic(i915_gem_object_get_dirty_page(obj, 0));
 
 	while (i < rodata->batch_items) {
 		u32 s = rodata->batch[i];
@@ -95,10 +95,8 @@ static int render_state_setup(struct render_state *so)
 			s = lower_32_bits(r);
 			if (has_64bit_reloc) {
 				if (i + 1 >= rodata->batch_items ||
-				    rodata->batch[i + 1] != 0) {
-					ret = -EINVAL;
-					goto err_out;
-				}
+				    rodata->batch[i + 1] != 0)
+					goto err;
 
 				d[i++] = s;
 				s = upper_32_bits(r);
@@ -110,12 +108,20 @@ static int render_state_setup(struct render_state *so)
 		d[i++] = s;
 	}
 
+	if (rodata->reloc[reloc_index] != -1) {
+		DRM_ERROR("only %d relocs resolved\n", reloc_index);
+		goto err;
+	}
+
+	so->batch_offset = so->vma->node.start;
+	so->batch_size = rodata->batch_items * sizeof(u32);
+
 	while (i % CACHELINE_DWORDS)
 		OUT_BATCH(d, i, MI_NOOP);
 
-	so->aux_batch_offset = i * sizeof(u32);
+	so->aux_offset = i * sizeof(u32);
 
-	if (HAS_POOLED_EU(dev)) {
+	if (HAS_POOLED_EU(i915)) {
 		/*
 		 * We always program 3x6 pool config but depending upon which
 		 * subslice is disabled HW drops down to appropriate config
@@ -143,89 +149,131 @@ static int render_state_setup(struct render_state *so)
 	}
 
 	OUT_BATCH(d, i, MI_BATCH_BUFFER_END);
-	so->aux_batch_size = (i * sizeof(u32)) - so->aux_batch_offset;
-
+	so->aux_size = i * sizeof(u32) - so->aux_offset;
+	so->aux_offset += so->batch_offset;
 	/*
 	 * Since we are sending length, we need to strictly conform to
 	 * all requirements. For Gen2 this must be a multiple of 8.
 	 */
-	so->aux_batch_size = ALIGN(so->aux_batch_size, 8);
-
-	kunmap(page);
-
-	ret = i915_gem_object_set_to_gtt_domain(so->vma->obj, false);
-	if (ret)
-		return ret;
-
-	if (rodata->reloc[reloc_index] != -1) {
-		DRM_ERROR("only %d relocs resolved\n", reloc_index);
-		return -EINVAL;
-	}
+	so->aux_size = ALIGN(so->aux_size, 8);
 
-	return 0;
+	if (needs_clflush)
+		drm_clflush_virt_range(d, i * sizeof(u32));
+	kunmap_atomic(d);
 
-err_out:
-	kunmap(page);
+	ret = i915_gem_object_set_to_gtt_domain(obj, false);
+out:
+	i915_gem_obj_finish_shmem_access(obj);
 	return ret;
+
+err:
+	kunmap_atomic(d);
+	ret = -EINVAL;
+	goto out;
 }
 
 #undef OUT_BATCH
 
-int i915_gem_render_state_init(struct drm_i915_gem_request *req)
+int i915_gem_render_state_init(struct intel_engine_cs *engine)
 {
-	struct render_state so;
+	struct intel_render_state *so;
+	const struct intel_renderstate_rodata *rodata;
 	struct drm_i915_gem_object *obj;
 	int ret;
 
-	if (WARN_ON(req->engine->id != RCS))
-		return -ENOENT;
+	if (engine->id != RCS)
+		return 0;
 
-	so.rodata = render_state_get_rodata(req);
-	if (!so.rodata)
+	rodata = render_state_get_rodata(engine);
+	if (!rodata)
 		return 0;
 
-	if (so.rodata->batch_items * 4 > 4096)
+	if (rodata->batch_items * 4 > 4096)
 		return -EINVAL;
 
-	obj = i915_gem_object_create_internal(&req->i915->drm, 4096);
-	if (IS_ERR(obj))
-		return PTR_ERR(obj);
+	so = kmalloc(sizeof(*so), GFP_KERNEL);
+	if (!so)
+		return -ENOMEM;
 
-	so.vma = i915_vma_create(obj, &req->i915->ggtt.base, NULL);
-	if (IS_ERR(so.vma)) {
-		ret = PTR_ERR(so.vma);
-		goto err_obj;
+	obj = i915_gem_object_create_internal(&engine->i915->drm, 4096);
+	if (IS_ERR(obj)) {
+		ret = PTR_ERR(obj);
+		goto err_free;
 	}
 
-	ret = i915_vma_pin(so.vma, 0, 0, PIN_GLOBAL);
-	if (ret)
+	so->vma = i915_vma_create(obj, &engine->i915->ggtt.base, NULL);
+	if (IS_ERR(so->vma)) {
+		ret = PTR_ERR(so->vma);
 		goto err_obj;
+	}
+
+	so->rodata = rodata;
+	engine->render_state = so;
+	return 0;
 
-	ret = render_state_setup(&so);
+err_obj:
+	i915_gem_object_put(obj);
+err_free:
+	kfree(so);
+	return ret;
+}
+
+int i915_gem_render_state_emit(struct drm_i915_gem_request *req)
+{
+	struct intel_render_state *so;
+	int ret;
+
+	so = req->engine->render_state;
+	if (!so)
+		return 0;
+
+	/* Recreate the page after shrinking */
+	if (!so->vma->obj->pages)
+		so->batch_offset = -1;
+
+	ret = i915_vma_pin(so->vma, 0, 0, PIN_GLOBAL | PIN_HIGH);
 	if (ret)
-		goto err_unpin;
+		return ret;
 
-	ret = req->engine->emit_bb_start(req, so.vma->node.start,
-					 so.rodata->batch_items * 4,
+	if (so->vma->node.start != so->batch_offset) {
+		ret = render_state_setup(so, req->i915);
+		if (ret)
+			goto err_unpin;
+	}
+
+	ret = req->engine->emit_bb_start(req,
+					 so->batch_offset, so->batch_size,
 					 I915_DISPATCH_SECURE);
 	if (ret)
 		goto err_unpin;
 
-	if (so.aux_batch_size > 8) {
+	if (so->aux_size > 8) {
 		ret = req->engine->emit_bb_start(req,
-						 (so.vma->node.start +
-						  so.aux_batch_offset),
-						 so.aux_batch_size,
+						 so->aux_offset, so->aux_size,
 						 I915_DISPATCH_SECURE);
 		if (ret)
 			goto err_unpin;
 	}
 
-	i915_vma_move_to_active(so.vma, req, 0);
+	i915_vma_move_to_active(so->vma, req, 0);
 err_unpin:
-	i915_vma_unpin(so.vma);
-	i915_vma_close(so.vma);
-err_obj:
-	__i915_gem_object_release_unless_active(obj);
+	i915_vma_unpin(so->vma);
 	return ret;
 }
+
+void i915_gem_render_state_fini(struct intel_engine_cs *engine)
+{
+	struct intel_render_state *so;
+	struct drm_i915_gem_object *obj;
+
+	so = fetch_and_zero(&engine->render_state);
+	if (!so)
+		return;
+
+	obj = so->vma->obj;
+
+	i915_vma_close(so->vma);
+	__i915_gem_object_release_unless_active(obj);
+
+	kfree(so);
+}
drivers/gpu/drm/i915/i915_gem_render_state.h +3 -1
index 18cce3f06e9c..87481845799d 100644
@@ -26,6 +26,8 @@
 
 struct drm_i915_gem_request;
 
-int i915_gem_render_state_init(struct drm_i915_gem_request *req);
+int i915_gem_render_state_init(struct intel_engine_cs *engine);
+int i915_gem_render_state_emit(struct drm_i915_gem_request *req);
+void i915_gem_render_state_fini(struct intel_engine_cs *engine);
 
 #endif /* _I915_GEM_RENDER_STATE_H_ */
drivers/gpu/drm/i915/intel_engine_cs.c +5 -0
index 99737b842d73..b48c51964cb3 100644
@@ -301,6 +301,10 @@ int intel_engine_init_common(struct intel_engine_cs *engine)
 	if (ret)
 		return ret;
 
+	ret = i915_gem_render_state_init(engine);
+	if (ret)
+		return ret;
+
 	return 0;
 }
 
@@ -315,6 +319,7 @@ void intel_engine_cleanup_common(struct intel_engine_cs *engine)
 {
 	intel_engine_cleanup_scratch(engine);
 
+	i915_gem_render_state_fini(engine);
 	intel_engine_fini_breadcrumbs(engine);
 	intel_engine_cleanup_cmd_parser(engine);
 	i915_gem_batch_pool_fini(&engine->batch_pool);
drivers/gpu/drm/i915/intel_lrc.c +1 -1
index 10fcea57e4dd..02c48d390148 100644
@@ -1637,7 +1637,7 @@ static int gen8_init_rcs_context(struct drm_i915_gem_request *req)
 	if (ret)
 		DRM_ERROR("MOCS failed to program: expect performance issues.\n");
 
-	return i915_gem_render_state_init(req);
+	return i915_gem_render_state_emit(req);
 }
 
 /**
drivers/gpu/drm/i915/intel_ringbuffer.c +1 -1
index b5d9c03f84ce..ec06d75a42aa 100644
@@ -648,7 +648,7 @@ static int intel_rcs_ctx_init(struct drm_i915_gem_request *req)
 	if (ret != 0)
 		return ret;
 
-	ret = i915_gem_render_state_init(req);
+	ret = i915_gem_render_state_emit(req);
 	if (ret)
 		return ret;
 
drivers/gpu/drm/i915/intel_ringbuffer.h +3 -0
index a0510d4940b2..4289fcda4b3f 100644
@@ -157,6 +157,7 @@ struct i915_ctx_workarounds {
 };
 
 struct drm_i915_gem_request;
+struct intel_render_state;
 
 struct intel_engine_cs {
 	struct drm_i915_private *i915;
@@ -184,6 +185,8 @@ struct intel_engine_cs {
 	unsigned int irq_shift;
 	struct intel_ring *buffer;
 
+	struct intel_render_state *render_state;
+
 	/* Rather than have every client wait upon all user interrupts,
 	 * with the herd waking after every interrupt and each doing the
 	 * heavyweight seqno dance, we delegate the task (of being the
Signature · 6 lines
2.9.3

_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
#38 lore raw

PATCH 13/42 drm/i915: Markup GEM API with lockdep asserts

In reply to Chris Wilson

Add lockdep_assert_held(struct_mutex) to the API preamble of the internal GEM interfaces.

---
 drivers/gpu/drm/i915/i915_gem.c              | 21 +++++++++++++++++++++
 drivers/gpu/drm/i915/i915_gem_evict.c        |  5 ++++-
 drivers/gpu/drm/i915/i915_gem_gtt.c          |  2 ++
 drivers/gpu/drm/i915/i915_gem_render_state.c |  2 ++
 drivers/gpu/drm/i915/i915_gem_request.c      |  6 ++++++
 5 files changed, 35 insertions(+), 1 deletion(-)
drivers/gpu/drm/i915/i915_gem.c +21 -0
index b560263bf446..ada837e393a7 100644
@@ -104,6 +104,8 @@ i915_gem_wait_for_error(struct i915_gpu_error *error)
 {
 	int ret;
 
+	might_sleep();
+
 	if (!i915_reset_in_progress(error))
 		return 0;
 
@@ -2295,6 +2297,8 @@ i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
 {
 	const struct drm_i915_gem_object_ops *ops = obj->ops;
 
+	lockdep_assert_held(&obj->base.dev->struct_mutex);
+
 	if (obj->pages == NULL)
 		return 0;
 
@@ -2466,6 +2470,8 @@ i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
 	const struct drm_i915_gem_object_ops *ops = obj->ops;
 	int ret;
 
+	lockdep_assert_held(&obj->base.dev->struct_mutex);
+
 	if (obj->pages)
 		return 0;
 
@@ -2746,6 +2752,8 @@ void i915_gem_reset(struct drm_i915_private *dev_priv)
 {
 	struct intel_engine_cs *engine;
 
+	lockdep_assert_held(&dev_priv->drm.struct_mutex);
+
 	i915_gem_retire_requests(dev_priv);
 
 	for_each_engine(engine, dev_priv)
@@ -2974,6 +2982,8 @@ int i915_vma_unbind(struct i915_vma *vma)
 	unsigned long active;
 	int ret;
 
+	lockdep_assert_held(&obj->base.dev->struct_mutex);
+
 	/* First wait upon any activity as retiring the request may
 	 * have side-effects such as unpinning or even unbinding this vma.
 	 */
@@ -3366,6 +3376,7 @@ i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
 	int ret;
 
 	lockdep_assert_held(&obj->base.dev->struct_mutex);
+
 	ret = i915_gem_object_wait(obj,
 				   I915_WAIT_INTERRUPTIBLE |
 				   I915_WAIT_LOCKED |
@@ -3444,6 +3455,8 @@ int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
 	struct i915_vma *vma;
 	int ret = 0;
 
+	lockdep_assert_held(&obj->base.dev->struct_mutex);
+
 	if (obj->cache_level == cache_level)
 		goto out;
 
@@ -3654,6 +3667,8 @@ i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
 	u32 old_read_domains, old_write_domain;
 	int ret;
 
+	lockdep_assert_held(&obj->base.dev->struct_mutex);
+
 	/* Mark the pin_display early so that we account for the
 	 * display coherency whilst setting up the cache domains.
 	 */
@@ -3720,6 +3735,8 @@ err_unpin_display:
 void
 i915_gem_object_unpin_from_display_plane(struct i915_vma *vma)
 {
+	lockdep_assert_held(&vma->vm->dev->struct_mutex);
+
 	if (WARN_ON(vma->obj->pin_display == 0))
 		return;
 
@@ -3749,6 +3766,7 @@ i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
 	int ret;
 
 	lockdep_assert_held(&obj->base.dev->struct_mutex);
+
 	ret = i915_gem_object_wait(obj,
 				   I915_WAIT_INTERRUPTIBLE |
 				   I915_WAIT_LOCKED |
@@ -3904,6 +3922,7 @@ int __i915_vma_do_pin(struct i915_vma *vma,
 	unsigned int bound = vma->flags;
 	int ret;
 
+	lockdep_assert_held(&vma->vm->dev->struct_mutex);
 	GEM_BUG_ON((flags & (PIN_GLOBAL | PIN_USER)) == 0);
 	GEM_BUG_ON((flags & PIN_GLOBAL) && !i915_vma_is_ggtt(vma));
 
@@ -3944,6 +3963,8 @@ i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj,
 	struct i915_vma *vma;
 	int ret;
 
+	lockdep_assert_held(&obj->base.dev->struct_mutex);
+
 	vma = i915_gem_obj_lookup_or_create_vma(obj, vm, view);
 	if (IS_ERR(vma))
 		return vma;
drivers/gpu/drm/i915/i915_gem_evict.c +4 -1
index 5b6f81c1dbca..61f716c8768c 100644
@@ -102,6 +102,7 @@ i915_gem_evict_something(struct i915_address_space *vm,
 	struct i915_vma *vma, *next;
 	int ret;
 
+	lockdep_assert_held(&vm->dev->struct_mutex);
 	trace_i915_gem_evict(vm, min_size, alignment, flags);
 
 	/*
@@ -212,6 +213,8 @@ i915_gem_evict_for_vma(struct i915_vma *target)
 {
 	struct drm_mm_node *node, *next;
 
+	lockdep_assert_held(&target->vm->dev->struct_mutex);
+
 	list_for_each_entry_safe(node, next,
 			&target->vm->mm.head_node.node_list,
 			node_list) {
@@ -265,7 +268,7 @@ int i915_gem_evict_vm(struct i915_address_space *vm, bool do_idle)
 	struct i915_vma *vma, *next;
 	int ret;
 
-	WARN_ON(!mutex_is_locked(&vm->dev->struct_mutex));
+	lockdep_assert_held(&vm->dev->struct_mutex);
 	trace_i915_gem_evict_vm(vm);
 
 	if (do_idle) {
drivers/gpu/drm/i915/i915_gem_gtt.c +2 -0
index 1c95da8424cb..78b692e5b998 100644
@@ -3427,6 +3427,7 @@ i915_vma_create(struct drm_i915_gem_object *obj,
 		struct i915_address_space *vm,
 		const struct i915_ggtt_view *view)
 {
+	lockdep_assert_held(&obj->base.dev->struct_mutex);
 	GEM_BUG_ON(view && !i915_is_ggtt(vm));
 	GEM_BUG_ON(i915_gem_obj_to_vma(obj, vm, view));
 
@@ -3454,6 +3455,7 @@ i915_gem_obj_lookup_or_create_vma(struct drm_i915_gem_object *obj,
 {
 	struct i915_vma *vma;
 
+	lockdep_assert_held(&obj->base.dev->struct_mutex);
 	GEM_BUG_ON(view && !i915_is_ggtt(vm));
 
 	vma = i915_gem_obj_to_vma(obj, vm, view);
drivers/gpu/drm/i915/i915_gem_render_state.c +2 -0
index fc3421c59906..adb00c5125ad 100644
@@ -223,6 +223,8 @@ int i915_gem_render_state_emit(struct drm_i915_gem_request *req)
 	struct intel_render_state *so;
 	int ret;
 
+	lockdep_assert_held(&req->i915->drm.struct_mutex);
+
 	so = req->engine->render_state;
 	if (!so)
 		return 0;
drivers/gpu/drm/i915/i915_gem_request.c +6 -0
index 9b2ac082d5da..5ad990add14d 100644
@@ -143,6 +143,9 @@ static void i915_gem_request_retire(struct drm_i915_gem_request *request)
 {
 	struct i915_gem_active *active, *next;
 
+	lockdep_assert_held(&request->i915->drm.struct_mutex);
+	GEM_BUG_ON(!i915_gem_request_completed(request));
+
 	trace_i915_gem_request_retire(request);
 	list_del_init(&request->link);
 
@@ -267,6 +270,8 @@ int i915_gem_set_seqno(struct drm_device *dev, u32 seqno)
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	int ret;
 
+	lockdep_assert_held(&dev_priv->drm.struct_mutex);
+
 	if (seqno == 0)
 		return -EINVAL;
 
@@ -603,6 +608,7 @@ void __i915_add_request(struct drm_i915_gem_request *request, bool flush_caches)
 	u32 reserved_tail;
 	int ret;
 
+	lockdep_assert_held(&request->i915->drm.struct_mutex);
 	trace_i915_gem_request_add(request);
 
 	/*
Signature · 6 lines
2.9.3

_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
#39 lore raw

PATCH 14/42 drm/i915: Use a radixtree for random access to the object's backing storage

In reply to Chris Wilson

A while ago we switched from a contiguous array of pages into an sglist, for that was both more convenient for mapping to hardware and avoided the requirement for a vmalloc array of pages on every object. However, certain GEM API calls (like pwrite, pread as well as performing relocations) do desire access to individual struct pages. A quick hack was to introduce a cache of the last access such that finding the following page was quick - this works so long as the caller desired sequential access. Walking backwards, or multiple callers, still hits a slow linear search for each page. One solution is to store each successful lookup in a radix tree.

---
 drivers/gpu/drm/i915/i915_drv.h         |  57 ++++--------
 drivers/gpu/drm/i915/i915_gem.c         | 149 ++++++++++++++++++++++++++++----
 drivers/gpu/drm/i915/i915_gem_stolen.c  |   4 +-
 drivers/gpu/drm/i915/i915_gem_userptr.c |   4 +-
 4 files changed, 154 insertions(+), 60 deletions(-)
drivers/gpu/drm/i915/i915_drv.h +18 -39
index bad97f1e5265..a96b446d8db4 100644
@@ -2278,9 +2278,12 @@ struct drm_i915_gem_object {
 
 	struct sg_table *pages;
 	int pages_pin_count;
-	struct get_page {
-		struct scatterlist *sg;
-		int last;
+	struct i915_gem_object_page_iter {
+		struct scatterlist *sg_pos;
+		unsigned long sg_idx;
+
+		struct radix_tree_root radix;
+		struct mutex lock;
 	} get_page;
 	void *mapping;
 
@@ -3168,45 +3171,21 @@ static inline int __sg_page_count(struct scatterlist *sg)
 	return sg->length >> PAGE_SHIFT;
 }
 
-struct page *
-i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj, int n);
-
-static inline dma_addr_t
-i915_gem_object_get_dma_address(struct drm_i915_gem_object *obj, int n)
-{
-	if (n < obj->get_page.last) {
-		obj->get_page.sg = obj->pages->sgl;
-		obj->get_page.last = 0;
-	}
-
-	while (obj->get_page.last + __sg_page_count(obj->get_page.sg) <= n) {
-		obj->get_page.last += __sg_page_count(obj->get_page.sg++);
-		if (unlikely(sg_is_chain(obj->get_page.sg)))
-			obj->get_page.sg = sg_chain_ptr(obj->get_page.sg);
-	}
-
-	return sg_dma_address(obj->get_page.sg) + ((n - obj->get_page.last) << PAGE_SHIFT);
-}
-
-static inline struct page *
-i915_gem_object_get_page(struct drm_i915_gem_object *obj, int n)
-{
-	if (WARN_ON(n >= obj->base.size >> PAGE_SHIFT))
-		return NULL;
+struct scatterlist *
+i915_gem_object_get_sg(struct drm_i915_gem_object *obj,
+		       unsigned long n, unsigned int *offset);
 
-	if (n < obj->get_page.last) {
-		obj->get_page.sg = obj->pages->sgl;
-		obj->get_page.last = 0;
-	}
+struct page *
+i915_gem_object_get_page(struct drm_i915_gem_object *obj,
+			 unsigned long n);
 
-	while (obj->get_page.last + __sg_page_count(obj->get_page.sg) <= n) {
-		obj->get_page.last += __sg_page_count(obj->get_page.sg++);
-		if (unlikely(sg_is_chain(obj->get_page.sg)))
-			obj->get_page.sg = sg_chain_ptr(obj->get_page.sg);
-	}
+struct page *
+i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj,
+			       unsigned long n);
 
-	return nth_page(sg_page(obj->get_page.sg), n - obj->get_page.last);
-}
+dma_addr_t
+i915_gem_object_get_dma_address(struct drm_i915_gem_object *obj,
+				unsigned long n);
 
 static inline void i915_gem_object_pin_pages(struct drm_i915_gem_object *obj)
 {
drivers/gpu/drm/i915/i915_gem.c +132 -17

182 lines not on this page · raw message

drivers/gpu/drm/i915/i915_gem_stolen.c +2 -2
index 59989e8ee5dc..24bad4e60ef0 100644
@@ -594,8 +594,8 @@ _i915_gem_object_create_stolen(struct drm_device *dev,
 	if (obj->pages == NULL)
 		goto cleanup;
 
-	obj->get_page.sg = obj->pages->sgl;
-	obj->get_page.last = 0;
+	obj->get_page.sg_pos = obj->pages->sgl;
+	obj->get_page.sg_idx = 0;
 
 	i915_gem_object_pin_pages(obj);
 	obj->stolen = stolen;
drivers/gpu/drm/i915/i915_gem_userptr.c +2 -2
index 1c891b92ac80..cb95789da76e 100644
@@ -526,8 +526,8 @@ __i915_gem_userptr_get_pages_worker(struct work_struct *_work)
 			if (ret == 0) {
 				list_add_tail(&obj->global_list,
 					      &to_i915(dev)->mm.unbound_list);
-				obj->get_page.sg = obj->pages->sgl;
-				obj->get_page.last = 0;
+				obj->get_page.sg_pos = obj->pages->sgl;
+				obj->get_page.sg_idx = 0;
 				pinned = 0;
 			}
 		}
Signature · 6 lines
2.9.3

_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
maintainer
#40 lore raw

Re: [PATCH 14/42] drm/i915: Use a radixtree for random access to the object's backing storage

Patches 14-25 to a separate series. Will continue their review there.

Regards, Joonas

Signature · 7 lines
Joonas Lahtinen
Open Source Technology Center
Intel Corporation
_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
#41 lore raw

Re: [PATCH 14/42] drm/i915: Use a radixtree for random access to the object's backing storage

Quoted 1 line · Joonas Lahtinen

Patches 14-25 to a separate series. Will continue their review there.

Nope. For reasons as I explained when I pulled them in, they are required to prevent a critical use-after-free when moving the active reference handling onto the vma retirement.
-Chris

Signature · 5 lines
Chris Wilson, Intel Open Source Technology Centre
_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
maintainer
#42 lore raw

Re: [PATCH 14/42] drm/i915: Use a radixtree for random access to the object's backing storage

Quoted 7 lines · Chris Wilson
Quoted 2 lines · Joonas Lahtinen

Patches 14-25 to a separate series. Will continue their review there.

Nope. For reasons as I explained when I pulled them in, they are required to prevent a critical use-after-free when moving the active reference handling onto the vma retirement.

Ack.

As discussed in IRC, asked John to give them a look too.

Will proceed with the remaining ones on Monday.

Regards, Joonas

Signature · 7 lines
Joonas Lahtinen
Open Source Technology Center
Intel Corporation
_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
Reviewed-by on v1 14/42
#43 lore raw

Re: [PATCH 14/42] drm/i915: Use a radixtree for random access to the object's backing storage

In reply to Chris Wilson
Quoted 11 lines · Chris Wilson

A while ago we switched from a contiguous array of pages into an sglist, for that was both more convenient for mapping to hardware and avoided the requirement for a vmalloc array of pages on every object. However, certain GEM API calls (like pwrite, pread as well as performing relocations) do desire access to individual struct pages. A quick hack was to introduce a cache of the last access such that finding the following page was quick - this works so long as the caller desired sequential access. Walking backwards, or multiple callers, still hits a slow linear search for each page. One solution is to store each successful lookup in a radix tree.

Quoted 7 lines
---
  drivers/gpu/drm/i915/i915_drv.h         |  57 ++++--------
  drivers/gpu/drm/i915/i915_gem.c         | 149 ++++++++++++++++++++++++++++----
  drivers/gpu/drm/i915/i915_gem_stolen.c  |   4 +-
  drivers/gpu/drm/i915/i915_gem_userptr.c |   4 +-
  4 files changed, 154 insertions(+), 60 deletions(-)

Am curious to know if you have any performance data to show how much of a benefit this is? Are there any real world apps that would notice or just IGT benchmarks?

_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx

#44 lore raw

Re: [PATCH 14/42] drm/i915: Use a radixtree for random access to the object's backing storage

In reply to Chris Wilson
Quoted 194 lines · Chris Wilson

A while ago we switched from a contiguous array of pages into an sglist, for that was both more convenient for mapping to hardware and avoided the requirement for a vmalloc array of pages on every object. However, certain GEM API calls (like pwrite, pread as well as performing relocations) do desire access to individual struct pages. A quick hack was to introduce a cache of the last access such that finding the following page was quick - this works so long as the caller desired sequential access. Walking backwards, or multiple callers, still hits a slow linear search for each page. One solution is to store each successful lookup in a radix tree.

---
  drivers/gpu/drm/i915/i915_drv.h         |  57 ++++--------
  drivers/gpu/drm/i915/i915_gem.c         | 149 ++++++++++++++++++++++++++++----
  drivers/gpu/drm/i915/i915_gem_stolen.c  |   4 +-
  drivers/gpu/drm/i915/i915_gem_userptr.c |   4 +-
  4 files changed, 154 insertions(+), 60 deletions(-)
drivers/gpu/drm/i915/i915_drv.h +18 -39
index bad97f1e5265..a96b446d8db4 100644
@@ -2278,9 +2278,12 @@ struct drm_i915_gem_object {
 
  	struct sg_table *pages;
  	int pages_pin_count;
-	struct get_page {
-		struct scatterlist *sg;
-		int last;
+	struct i915_gem_object_page_iter {
+		struct scatterlist *sg_pos;
+		unsigned long sg_idx;
+
+		struct radix_tree_root radix;
+		struct mutex lock;
  	} get_page;
  	void *mapping;
 
@@ -3168,45 +3171,21 @@ static inline int __sg_page_count(struct scatterlist *sg)
  	return sg->length >> PAGE_SHIFT;
  }
 
-struct page *
-i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj, int n);
-
-static inline dma_addr_t
-i915_gem_object_get_dma_address(struct drm_i915_gem_object *obj, int n)
-{
-	if (n < obj->get_page.last) {
-		obj->get_page.sg = obj->pages->sgl;
-		obj->get_page.last = 0;
-	}
-
-	while (obj->get_page.last + __sg_page_count(obj->get_page.sg) <= n) {
-		obj->get_page.last += __sg_page_count(obj->get_page.sg++);
-		if (unlikely(sg_is_chain(obj->get_page.sg)))
-			obj->get_page.sg = sg_chain_ptr(obj->get_page.sg);
-	}
-
-	return sg_dma_address(obj->get_page.sg) + ((n - obj->get_page.last) << PAGE_SHIFT);
-}
-
-static inline struct page *
-i915_gem_object_get_page(struct drm_i915_gem_object *obj, int n)
-{
-	if (WARN_ON(n >= obj->base.size >> PAGE_SHIFT))
-		return NULL;
+struct scatterlist *
+i915_gem_object_get_sg(struct drm_i915_gem_object *obj,
+		       unsigned long n, unsigned int *offset);
 
-	if (n < obj->get_page.last) {
-		obj->get_page.sg = obj->pages->sgl;
-		obj->get_page.last = 0;
-	}
+struct page *
+i915_gem_object_get_page(struct drm_i915_gem_object *obj,
+			 unsigned long n);
 
-	while (obj->get_page.last + __sg_page_count(obj->get_page.sg) <= n) {
-		obj->get_page.last += __sg_page_count(obj->get_page.sg++);
-		if (unlikely(sg_is_chain(obj->get_page.sg)))
-			obj->get_page.sg = sg_chain_ptr(obj->get_page.sg);
-	}
+struct page *
+i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj,
+			       unsigned long n);
 
-	return nth_page(sg_page(obj->get_page.sg), n - obj->get_page.last);
-}
+dma_addr_t
+i915_gem_object_get_dma_address(struct drm_i915_gem_object *obj,
+				unsigned long n);
 
  static inline void i915_gem_object_pin_pages(struct drm_i915_gem_object *obj)
  {
drivers/gpu/drm/i915/i915_gem.c +37 -17
index ada837e393a7..af7d51f16658 100644
@@ -2292,6 +2292,15 @@ i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
  	kfree(obj->pages);
  }
 
+static void __i915_gem_object_reset_page_iter(struct drm_i915_gem_object *obj)
+{
+	struct radix_tree_iter iter;
+	void **slot;
+
+	radix_tree_for_each_slot(slot, &obj->get_page.radix, &iter, 0)
+		radix_tree_delete(&obj->get_page.radix, iter.index);
+}
+
  int
  i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
  {
@@ -2324,6 +2333,8 @@ i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
  		obj->mapping = NULL;
  	}
 
+	__i915_gem_object_reset_page_iter(obj);
+
  	ops->put_pages(obj);
  	obj->pages = NULL;
 
@@ -2488,8 +2499,8 @@ i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
 
  	list_add_tail(&obj->global_list, &dev_priv->mm.unbound_list);
 
-	obj->get_page.sg = obj->pages->sgl;
-	obj->get_page.last = 0;
+	obj->get_page.sg_pos = obj->pages->sgl;
+	obj->get_page.sg_idx = 0;
 
  	return 0;
  }
@@ -4242,6 +4253,8 @@ void i915_gem_object_init(struct drm_i915_gem_object *obj,
 
  	obj->frontbuffer_ggtt_origin = ORIGIN_GTT;
  	obj->madv = I915_MADV_WILLNEED;
+	INIT_RADIX_TREE(&obj->get_page.radix, GFP_KERNEL);
+	mutex_init(&obj->get_page.lock);
 
  	i915_gem_info_add_obj(to_i915(obj->base.dev), obj->base.size);
  }
@@ -4904,21 +4917,6 @@ void i915_gem_track_fb(struct drm_i915_gem_object *old,
  	}
  }
 
-/* Like i915_gem_object_get_page(), but mark the returned page dirty */
-struct page *
-i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj, int n)
-{
-	struct page *page;
-
-	/* Only default objects have per-page dirty tracking */
-	if (WARN_ON(!i915_gem_object_has_struct_page(obj)))
-		return NULL;
-
-	page = i915_gem_object_get_page(obj, n);
-	set_page_dirty(page);
-	return page;
-}
-
  /* Allocate a new GEM object and fill it with the supplied data */
  struct drm_i915_gem_object *
  i915_gem_object_create_from_data(struct drm_device *dev,
@@ -4959,3 +4957,120 @@ fail:
  	i915_gem_object_put(obj);
  	return ERR_PTR(ret);
  }
+
+static struct scatterlist *
+__i915_gem_object_get_sg(struct drm_i915_gem_object *obj,
+			 unsigned long n, unsigned int *offset)
+{
+	struct scatterlist *sg = obj->pages->sgl;
+	int idx = 0;
+
+	while (idx + __sg_page_count(sg) <= n) {
+		idx += __sg_page_count(sg++);
+		if (unlikely(sg_is_chain(sg)))
+			sg = sg_chain_ptr(sg);
+	}
+
+	*offset = n - idx;
+	return sg;
+}
+
+struct scatterlist *
+i915_gem_object_get_sg(struct drm_i915_gem_object *obj,
+		       unsigned long n,
+		       unsigned int *offset)

drivers/gpu/drm/i915/i915_gem.c fail: · line 4991

Quoted 10 lines
drivers/gpu/drm/i915/i915_gem.c +10 -0
+{
+	struct i915_gem_object_page_iter *iter = &obj->get_page;
+	struct scatterlist *sg;
+
+	GEM_BUG_ON(n >= obj->base.size >> PAGE_SHIFT);
+	GEM_BUG_ON(obj->pages_pin_count == 0);
+
+	if (n < READ_ONCE(iter->sg_idx))
+		goto lookup;
+

Ok, so on lookup of "n" you build the radix tree for all sg entries from zero to n. Therefore for a lookup below the current index, there must be a radix tree entry, correct?

Quoted 20 lines
drivers/gpu/drm/i915/i915_gem.c +20 -0
+	mutex_lock(&iter->lock);
+	if (n >= iter->sg_idx &&
+	    n < iter->sg_idx + __sg_page_count(iter->sg_pos)) {
+		sg = iter->sg_pos;
+		*offset = n - iter->sg_idx;
+		mutex_unlock(&iter->lock);
+		return sg;
+	}
+
+	while (iter->sg_idx <= n) {
+		unsigned long exception;
+		unsigned int count, i;
+
+		radix_tree_insert(&iter->radix,
+				  iter->sg_idx,
+				  iter->sg_pos);
+
+		exception =
+			RADIX_TREE_EXCEPTIONAL_ENTRY |
+			iter->sg_idx << RADIX_TREE_EXCEPTIONAL_SHIFT;

drivers/gpu/drm/i915/i915_gem.c fail: · line 5021

Quoted 10 lines
drivers/gpu/drm/i915/i915_gem.c +10 -0
+		count = __sg_page_count(iter->sg_pos);
+		for (i = 1; i < count; i++)
+			radix_tree_insert(&iter->radix,
+					  iter->sg_idx + i,
+					  (void *)exception);
+
+		iter->sg_idx += count;
+		iter->sg_pos = __sg_next(iter->sg_pos);
+	}
+	mutex_unlock(&iter->lock);

Why not avoid falling through the lookup and return the previous sg from here?

drivers/gpu/drm/i915/i915_gem.c fail: · line 5030

Quoted 9 lines
drivers/gpu/drm/i915/i915_gem.c +9 -0
+
+lookup:
+	rcu_read_lock();
+	sg = radix_tree_lookup(&iter->radix, n);
+	rcu_read_unlock();
+
+	if (unlikely(!sg))
+		return __i915_gem_object_get_sg(obj, n, offset);
+

Considering the first observation from above, when does it then happen that there is no radix tree entry at this point?

Or in other words, maybe some comments should be added to this patch - there are none and it is not that trivial.

Quoted 66 lines
drivers/gpu/drm/i915/i915_gem.c +46 -0
+	*offset = 0;
+	if (unlikely(radix_tree_exception(sg))) {
+		unsigned long base =
+			(unsigned long)sg >> RADIX_TREE_EXCEPTIONAL_SHIFT;
+		sg = radix_tree_lookup(&iter->radix, base);
+		*offset = n - base;
+	}
+	return sg;
+}
+
+struct page *
+i915_gem_object_get_page(struct drm_i915_gem_object *obj, unsigned long n)
+{
+	struct scatterlist *sg;
+	unsigned int offset;
+
+	GEM_BUG_ON(!i915_gem_object_has_struct_page(obj));
+
+	sg = i915_gem_object_get_sg(obj, n, &offset);
+	return nth_page(sg_page(sg), offset);
+}
+
+/* Like i915_gem_object_get_page(), but mark the returned page dirty */
+struct page *
+i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj,
+			       unsigned long n)
+{
+	struct page *page;
+
+	page = i915_gem_object_get_page(obj, n);
+	if (!obj->dirty)
+		set_page_dirty(page);
+
+	return page;
+}
+
+dma_addr_t
+i915_gem_object_get_dma_address(struct drm_i915_gem_object *obj,
+				unsigned long n)
+{
+	struct scatterlist *sg;
+	unsigned int offset;
+
+	sg = i915_gem_object_get_sg(obj, n, &offset);
+	return sg_dma_address(sg) + (offset << PAGE_SHIFT);
+}
drivers/gpu/drm/i915/i915_gem_stolen.c +2 -2
index 59989e8ee5dc..24bad4e60ef0 100644
@@ -594,8 +594,8 @@ _i915_gem_object_create_stolen(struct drm_device *dev,
  	if (obj->pages == NULL)
  		goto cleanup;
 
-	obj->get_page.sg = obj->pages->sgl;
-	obj->get_page.last = 0;
+	obj->get_page.sg_pos = obj->pages->sgl;
+	obj->get_page.sg_idx = 0;
 
  	i915_gem_object_pin_pages(obj);
  	obj->stolen = stolen;
drivers/gpu/drm/i915/i915_gem_userptr.c +0 -0
index 1c891b92ac80..cb95789da76e 100644

drivers/gpu/drm/i915/i915_gem_userptr.c __i915_gem_userptr_get_pages_worker(struct work_struct *_work) · line 533

Quoted 10 lines
drivers/gpu/drm/i915/i915_gem_userptr.c +2 -2
@@ -526,8 +526,8 @@ __i915_gem_userptr_get_pages_worker(struct work_struct *_work)
  			if (ret == 0) {
  				list_add_tail(&obj->global_list,
  					      &to_i915(dev)->mm.unbound_list);
-				obj->get_page.sg = obj->pages->sgl;
-				obj->get_page.last = 0;
+				obj->get_page.sg_pos = obj->pages->sgl;
+				obj->get_page.sg_idx = 0;
  				pinned = 0;
  			}
  		}

Regards,

Tvrtko
_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx

#45 lore raw

Re: [PATCH 14/42] drm/i915: Use a radixtree for random access to the object's backing storage

Quoted 198 lines · Tvrtko Ursulin
Quoted 196 lines · Chris Wilson

A while ago we switched from a contiguous array of pages into an sglist, for that was both more convenient for mapping to hardware and avoided the requirement for a vmalloc array of pages on every object. However, certain GEM API calls (like pwrite, pread as well as performing relocations) do desire access to individual struct pages. A quick hack was to introduce a cache of the last access such that finding the following page was quick - this works so long as the caller desired sequential access. Walking backwards, or multiple callers, still hits a slow linear search for each page. One solution is to store each successful lookup in a radix tree.

---
  drivers/gpu/drm/i915/i915_drv.h         |  57 ++++--------
  drivers/gpu/drm/i915/i915_gem.c         | 149 
++++++++++++++++++++++++++++----
  drivers/gpu/drm/i915/i915_gem_stolen.c  |   4 +-
  drivers/gpu/drm/i915/i915_gem_userptr.c |   4 +-
  4 files changed, 154 insertions(+), 60 deletions(-)
drivers/gpu/drm/i915/i915_drv.h +6 -3
index bad97f1e5265..a96b446d8db4 100644
@@ -2278,9 +2278,12 @@ struct drm_i915_gem_object {
        struct sg_table *pages;
      int pages_pin_count;
-    struct get_page {
-        struct scatterlist *sg;
-        int last;
+    struct i915_gem_object_page_iter {
+        struct scatterlist *sg_pos;
+        unsigned long sg_idx;
+
+        struct radix_tree_root radix;
+        struct mutex lock;
      } get_page;
      void *mapping;
  @@ -3168,45 +3171,21 @@ static inline int __sg_page_count(struct 
 scatterlist *sg)
      return sg->length >> PAGE_SHIFT;

}
-struct page *

drivers/gpu/drm/i915/i915_drv.h +0 -11
-i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj, int n);
-
-static inline dma_addr_t
-i915_gem_object_get_dma_address(struct drm_i915_gem_object *obj, int n)
-{
-    if (n < obj->get_page.last) {
-        obj->get_page.sg = obj->pages->sgl;
-        obj->get_page.last = 0;
-    }
-
-    while (obj->get_page.last + __sg_page_count(obj->get_page.sg) <= 

n) {

drivers/gpu/drm/i915/i915_drv.h +0 -6
-        obj->get_page.last += __sg_page_count(obj->get_page.sg++);
-        if (unlikely(sg_is_chain(obj->get_page.sg)))
-            obj->get_page.sg = sg_chain_ptr(obj->get_page.sg);
-    }
-
-    return sg_dma_address(obj->get_page.sg) + ((n - 

obj->get_page.last) << PAGE_SHIFT);

drivers/gpu/drm/i915/i915_drv.h +6 -10
-}
-
-static inline struct page *
-i915_gem_object_get_page(struct drm_i915_gem_object *obj, int n)
-{
-    if (WARN_ON(n >= obj->base.size >> PAGE_SHIFT))
-        return NULL;
+struct scatterlist *
+i915_gem_object_get_sg(struct drm_i915_gem_object *obj,
+               unsigned long n, unsigned int *offset);
  -    if (n < obj->get_page.last) {
-        obj->get_page.sg = obj->pages->sgl;
-        obj->get_page.last = 0;
-    }
+struct page *
+i915_gem_object_get_page(struct drm_i915_gem_object *obj,
+             unsigned long n);
  -    while (obj->get_page.last + __sg_page_count(obj->get_page.sg) 

<= n) {

drivers/gpu/drm/i915/i915_drv.h +3 -4
-        obj->get_page.last += __sg_page_count(obj->get_page.sg++);
-        if (unlikely(sg_is_chain(obj->get_page.sg)))
-            obj->get_page.sg = sg_chain_ptr(obj->get_page.sg);
-    }
+struct page *
+i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj,
+                   unsigned long n);
  -    return nth_page(sg_page(obj->get_page.sg), n - 

obj->get_page.last);

drivers/gpu/drm/i915/i915_drv.h +3 -1
-}
+dma_addr_t
+i915_gem_object_get_dma_address(struct drm_i915_gem_object *obj,
+                unsigned long n);
    static inline void i915_gem_object_pin_pages(struct 

drm_i915_gem_object *obj)
{

drivers/gpu/drm/i915/i915_gem.c +9 -0
index ada837e393a7..af7d51f16658 100644
@@ -2292,6 +2292,15 @@ i915_gem_object_put_pages_gtt(struct 
 drm_i915_gem_object *obj)
      kfree(obj->pages);
  }
  +static void __i915_gem_object_reset_page_iter(struct 
 drm_i915_gem_object *obj)
+{
+    struct radix_tree_iter iter;
+    void **slot;
+
+    radix_tree_for_each_slot(slot, &obj->get_page.radix, &iter, 0)
+        radix_tree_delete(&obj->get_page.radix, iter.index);
+}
+
  int
  i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
  {
@@ -2324,6 +2333,8 @@ i915_gem_object_put_pages(struct 
 drm_i915_gem_object *obj)
          obj->mapping = NULL;
      }
  +    __i915_gem_object_reset_page_iter(obj);
+
      ops->put_pages(obj);
      obj->pages = NULL;
  @@ -2488,8 +2499,8 @@ i915_gem_object_get_pages(struct 

drm_i915_gem_object *obj)

        list_add_tail(&obj->global_list, &dev_priv->mm.unbound_list);

- obj->get_page.sg = obj->pages->sgl;

drivers/gpu/drm/i915/i915_gem.c +4 -1
-    obj->get_page.last = 0;
+    obj->get_page.sg_pos = obj->pages->sgl;
+    obj->get_page.sg_idx = 0;
        return 0;
  }
@@ -4242,6 +4253,8 @@ void i915_gem_object_init(struct 
 drm_i915_gem_object *obj,
        obj->frontbuffer_ggtt_origin = ORIGIN_GTT;
      obj->madv = I915_MADV_WILLNEED;
+    INIT_RADIX_TREE(&obj->get_page.radix, GFP_KERNEL);
+    mutex_init(&obj->get_page.lock);
        i915_gem_info_add_obj(to_i915(obj->base.dev), obj->base.size);
  }
@@ -4904,21 +4917,6 @@ void i915_gem_track_fb(struct 
 drm_i915_gem_object *old,
      }
  }
  -/* Like i915_gem_object_get_page(), but mark the returned page 

dirty */

drivers/gpu/drm/i915/i915_gem.c +26 -14
-struct page *
-i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj, int n)
-{
-    struct page *page;
-
-    /* Only default objects have per-page dirty tracking */
-    if (WARN_ON(!i915_gem_object_has_struct_page(obj)))
-        return NULL;
-
-    page = i915_gem_object_get_page(obj, n);
-    set_page_dirty(page);
-    return page;
-}
-
  /* Allocate a new GEM object and fill it with the supplied data */
  struct drm_i915_gem_object *
  i915_gem_object_create_from_data(struct drm_device *dev,
@@ -4959,3 +4957,120 @@ fail:
      i915_gem_object_put(obj);
      return ERR_PTR(ret);
  }
+
+static struct scatterlist *
+__i915_gem_object_get_sg(struct drm_i915_gem_object *obj,
+             unsigned long n, unsigned int *offset)
+{
+    struct scatterlist *sg = obj->pages->sgl;
+    int idx = 0;
+
+    while (idx + __sg_page_count(sg) <= n) {
+        idx += __sg_page_count(sg++);
+        if (unlikely(sg_is_chain(sg)))
+            sg = sg_chain_ptr(sg);
+    }
+
+    *offset = n - idx;
+    return sg;
+}
+
+struct scatterlist *
+i915_gem_object_get_sg(struct drm_i915_gem_object *obj,
+               unsigned long n,
+               unsigned int *offset)
+{
+    struct i915_gem_object_page_iter *iter = &obj->get_page;
+    struct scatterlist *sg;
+
Quoted 8 lines

drivers/gpu/drm/i915/i915_gem.c fail: · line 4991

Quoted 6 lines
drivers/gpu/drm/i915/i915_gem.c +6 -0
+    GEM_BUG_ON(n >= obj->base.size >> PAGE_SHIFT);
+    GEM_BUG_ON(obj->pages_pin_count == 0);
+
+    if (n < READ_ONCE(iter->sg_idx))
+        goto lookup;
+

Ok, so on lookup of "n" you build the radix tree for all sg entries from zero to n. Therefore for a lookup below the current index, there must be a radix tree entry, correct?

That is my understanding.

Quoted 25 lines
Quoted 25 lines
drivers/gpu/drm/i915/i915_gem.c +24 -0
+    mutex_lock(&iter->lock);
+    if (n >= iter->sg_idx &&
+        n < iter->sg_idx + __sg_page_count(iter->sg_pos)) {
+        sg = iter->sg_pos;
+        *offset = n - iter->sg_idx;
+        mutex_unlock(&iter->lock);
+        return sg;
+    }
+
+    while (iter->sg_idx <= n) {
+        unsigned long exception;
+        unsigned int count, i;
+
+        radix_tree_insert(&iter->radix,
+                  iter->sg_idx,
+                  iter->sg_pos);
+
+        exception =
+            RADIX_TREE_EXCEPTIONAL_ENTRY |
+            iter->sg_idx << RADIX_TREE_EXCEPTIONAL_SHIFT;
+        count = __sg_page_count(iter->sg_pos);
+        for (i = 1; i < count; i++)
+            radix_tree_insert(&iter->radix,
+                      iter->sg_idx + i,
Quoted 8 lines

drivers/gpu/drm/i915/i915_gem.c fail: · line 5021

Quoted 6 lines
drivers/gpu/drm/i915/i915_gem.c +6 -0
+                      (void *)exception);
+
+        iter->sg_idx += count;
+        iter->sg_pos = __sg_next(iter->sg_pos);
+    }
+    mutex_unlock(&iter->lock);

Why not avoid falling through the lookup and return the previous sg from here?

I thought I worked through this before and decided there was a good reason. However, looking again a few days later, I can't obviously see why. As you note below, this is non-trivial stuff and would definitely benefit from some explanatory comments.

Quoted 95 lines

drivers/gpu/drm/i915/i915_gem.c fail: · line 5030

Quoted 10 lines
drivers/gpu/drm/i915/i915_gem.c +9 -0
+
+lookup:
+    rcu_read_lock();
+    sg = radix_tree_lookup(&iter->radix, n);
+    rcu_read_unlock();
+
+    if (unlikely(!sg))
+        return __i915_gem_object_get_sg(obj, n, offset);
+

Considering the first observation from above, when does it then happen that there is no radix tree entry at this point?

Or in other words, maybe some comments should be added to this patch - there are none and it is not that trivial.

Quoted 76 lines
drivers/gpu/drm/i915/i915_gem.c +12 -0
+    *offset = 0;
+    if (unlikely(radix_tree_exception(sg))) {
+        unsigned long base =
+            (unsigned long)sg >> RADIX_TREE_EXCEPTIONAL_SHIFT;
+        sg = radix_tree_lookup(&iter->radix, base);
+        *offset = n - base;
+    }
+    return sg;
+}
+
+struct page *
+i915_gem_object_get_page(struct drm_i915_gem_object *obj, unsigned 

long n)

drivers/gpu/drm/i915/i915_gem.c +34 -0
+{
+    struct scatterlist *sg;
+    unsigned int offset;
+
+    GEM_BUG_ON(!i915_gem_object_has_struct_page(obj));
+
+    sg = i915_gem_object_get_sg(obj, n, &offset);
+    return nth_page(sg_page(sg), offset);
+}
+
+/* Like i915_gem_object_get_page(), but mark the returned page dirty */
+struct page *
+i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj,
+                   unsigned long n)
+{
+    struct page *page;
+
+    page = i915_gem_object_get_page(obj, n);
+    if (!obj->dirty)
+        set_page_dirty(page);
+
+    return page;
+}
+
+dma_addr_t
+i915_gem_object_get_dma_address(struct drm_i915_gem_object *obj,
+                unsigned long n)
+{
+    struct scatterlist *sg;
+    unsigned int offset;
+
+    sg = i915_gem_object_get_sg(obj, n, &offset);
+    return sg_dma_address(sg) + (offset << PAGE_SHIFT);
+}
drivers/gpu/drm/i915/i915_gem_stolen.c +2 -1
index 59989e8ee5dc..24bad4e60ef0 100644
@@ -594,8 +594,8 @@ _i915_gem_object_create_stolen(struct drm_device 
 *dev,
      if (obj->pages == NULL)
          goto cleanup;
  -    obj->get_page.sg = obj->pages->sgl;
-    obj->get_page.last = 0;
+    obj->get_page.sg_pos = obj->pages->sgl;
+    obj->get_page.sg_idx = 0;
        i915_gem_object_pin_pages(obj);
      obj->stolen = stolen;
drivers/gpu/drm/i915/i915_gem_userptr.c +2 -2
index 1c891b92ac80..cb95789da76e 100644
@@ -526,8 +526,8 @@ __i915_gem_userptr_get_pages_worker(struct 
 work_struct *_work)
              if (ret == 0) {
                  list_add_tail(&obj->global_list,
 &to_i915(dev)->mm.unbound_list);
-                obj->get_page.sg = obj->pages->sgl;
-                obj->get_page.last = 0;
+                obj->get_page.sg_pos = obj->pages->sgl;
+                obj->get_page.sg_idx = 0;
                  pinned = 0;
              }
Quoted 1 line
          }

Regards,

Quoted 5 lines

Tvrtko
_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx

_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx

#46 lore raw

PATCH 15/42 drm/i915: Use radixtree to jump start intel_partial_pages()

In reply to Chris Wilson

We can use the radixtree index of the obj->pages to find the start position of the desired partial range.

---
 drivers/gpu/drm/i915/i915_gem_gtt.c | 38 +++++++++++++++++++++++--------------
 1 file changed, 24 insertions(+), 14 deletions(-)
drivers/gpu/drm/i915/i915_gem_gtt.c +24 -14
index 78b692e5b998..7e0c98576e72 100644
@@ -3562,35 +3562,45 @@ intel_partial_pages(const struct i915_ggtt_view *view,
 		    struct drm_i915_gem_object *obj)
 {
 	struct sg_table *st;
-	struct scatterlist *sg;
-	struct sg_page_iter obj_sg_iter;
+	struct scatterlist *sg, *iter;
+	unsigned int count = view->params.partial.size;
+	unsigned int offset;
 	int ret = -ENOMEM;
 
 	st = kmalloc(sizeof(*st), GFP_KERNEL);
 	if (!st)
 		goto err_st_alloc;
 
-	ret = sg_alloc_table(st, view->params.partial.size, GFP_KERNEL);
+	ret = sg_alloc_table(st, count, GFP_KERNEL);
 	if (ret)
 		goto err_sg_alloc;
 
+	iter = i915_gem_object_get_sg(obj,
+				      view->params.partial.offset,
+				      &offset);
+
 	sg = st->sgl;
 	st->nents = 0;
-	for_each_sg_page(obj->pages->sgl, &obj_sg_iter, obj->pages->nents,
-		view->params.partial.offset)
-	{
-		if (st->nents >= view->params.partial.size)
-			break;
+	do {
+		unsigned int len =
+			min(iter->length, (count - offset) << PAGE_SHIFT);
 
-		sg_set_page(sg, NULL, PAGE_SIZE, 0);
-		sg_dma_address(sg) = sg_page_iter_dma_address(&obj_sg_iter);
-		sg_dma_len(sg) = PAGE_SIZE;
+		sg_set_page(sg, NULL, len, 0);
+		sg_dma_address(sg) =
+			sg_dma_address(iter) + (offset << PAGE_SHIFT);
+		sg_dma_len(sg) = len;
 
-		sg = sg_next(sg);
 		st->nents++;
-	}
+		count -= len >> PAGE_SHIFT;
+		if (count == 0) {
+			sg_mark_end(sg);
+			return st;
+		}
 
-	return st;
+		sg = __sg_next(sg);
+		iter = __sg_next(iter);
+		offset = 0;
+	} while (1);
 
 err_sg_alloc:
 	kfree(st);
Signature · 6 lines
2.9.3

_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
Reviewed-by on v1 15/42
#47 lore raw

Re: [PATCH 15/42] drm/i915: Use radixtree to jump start intel_partial_pages()

Quoted 62 lines · Chris Wilson

We can use the radixtree index of the obj->pages to find the start position of the desired partial range.

---
  drivers/gpu/drm/i915/i915_gem_gtt.c | 38 +++++++++++++++++++++++--------------
  1 file changed, 24 insertions(+), 14 deletions(-)
drivers/gpu/drm/i915/i915_gem_gtt.c +19 -13
index 78b692e5b998..7e0c98576e72 100644
@@ -3562,35 +3562,45 @@ intel_partial_pages(const struct i915_ggtt_view *view,
  		    struct drm_i915_gem_object *obj)
  {
  	struct sg_table *st;
-	struct scatterlist *sg;
-	struct sg_page_iter obj_sg_iter;
+	struct scatterlist *sg, *iter;
+	unsigned int count = view->params.partial.size;
+	unsigned int offset;
  	int ret = -ENOMEM;
 
  	st = kmalloc(sizeof(*st), GFP_KERNEL);
  	if (!st)
  		goto err_st_alloc;
 
-	ret = sg_alloc_table(st, view->params.partial.size, GFP_KERNEL);
+	ret = sg_alloc_table(st, count, GFP_KERNEL);
  	if (ret)
  		goto err_sg_alloc;
 
+	iter = i915_gem_object_get_sg(obj,
+				      view->params.partial.offset,
+				      &offset);
+
  	sg = st->sgl;
  	st->nents = 0;
-	for_each_sg_page(obj->pages->sgl, &obj_sg_iter, obj->pages->nents,
-		view->params.partial.offset)
-	{
-		if (st->nents >= view->params.partial.size)
-			break;
+	do {
+		unsigned int len =
+			min(iter->length, (count - offset) << PAGE_SHIFT);
 
-		sg_set_page(sg, NULL, PAGE_SIZE, 0);
-		sg_dma_address(sg) = sg_page_iter_dma_address(&obj_sg_iter);
-		sg_dma_len(sg) = PAGE_SIZE;
+		sg_set_page(sg, NULL, len, 0);
+		sg_dma_address(sg) =
+			sg_dma_address(iter) + (offset << PAGE_SHIFT);
+		sg_dma_len(sg) = len;
 
-		sg = sg_next(sg);
  		st->nents++;
-	}
+		count -= len >> PAGE_SHIFT;
+		if (count == 0) {
+			sg_mark_end(sg);
+			return st;

drivers/gpu/drm/i915/i915_gem_gtt.c intel_partial_pages(const struct i915_ggtt_view *view, · line 3606

Quoted 10 lines
drivers/gpu/drm/i915/i915_gem_gtt.c +5 -1
+		}
 
-	return st;
+		sg = __sg_next(sg);
+		iter = __sg_next(iter);
+		offset = 0;
+	} while (1);
 
  err_sg_alloc:
  	kfree(st);

_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx

#48 lore raw

PATCH 16/42 drm/i915: Refactor object page API

In reply to Chris Wilson

The plan is to make obtaining the backing storage for the object avoid struct_mutex (i.e. use its own locking). The first step is to update the API so that normal users only call pin/unpin whilst working on the backing storage.

---
 drivers/gpu/drm/i915/i915_cmd_parser.c       |   2 +-
 drivers/gpu/drm/i915/i915_debugfs.c          |  17 +--
 drivers/gpu/drm/i915/i915_drv.h              |  89 ++++++++----
 drivers/gpu/drm/i915/i915_gem.c              | 207 +++++++++++++--------------
 drivers/gpu/drm/i915/i915_gem_batch_pool.c   |   3 +-
 drivers/gpu/drm/i915/i915_gem_dmabuf.c       |  14 +-
 drivers/gpu/drm/i915/i915_gem_execbuffer.c   |   2 +-
 drivers/gpu/drm/i915/i915_gem_fence.c        |   4 +-
 drivers/gpu/drm/i915/i915_gem_gtt.c          |  10 +-
 drivers/gpu/drm/i915/i915_gem_internal.c     |  19 +--
 drivers/gpu/drm/i915/i915_gem_render_state.c |   2 +-
 drivers/gpu/drm/i915/i915_gem_shrinker.c     |  10 +-
 drivers/gpu/drm/i915/i915_gem_stolen.c       |  24 ++--
 drivers/gpu/drm/i915/i915_gem_tiling.c       |   8 +-
 drivers/gpu/drm/i915/i915_gem_userptr.c      |  30 ++--
 drivers/gpu/drm/i915/i915_gpu_error.c        |   4 +-
 drivers/gpu/drm/i915/intel_lrc.c             |   6 +-
 17 files changed, 234 insertions(+), 217 deletions(-)
drivers/gpu/drm/i915/i915_cmd_parser.c +1 -1
index 70980f82a15b..8d20020cb9f9 100644
@@ -1290,7 +1290,7 @@ int intel_engine_cmd_parser(struct intel_engine_cs *engine,
 	}
 
 	if (ret == 0 && needs_clflush_after)
-		drm_clflush_virt_range(shadow_batch_obj->mapping, batch_len);
+		drm_clflush_virt_range(shadow_batch_obj->mm.mapping, batch_len);
 	i915_gem_object_unpin_map(shadow_batch_obj);
 
 	return ret;
drivers/gpu/drm/i915/i915_debugfs.c +9 -8
index e4b5ba771bea..b807ddf65e04 100644
@@ -112,7 +112,7 @@ static char get_global_flag(struct drm_i915_gem_object *obj)
 
 static char get_pin_mapped_flag(struct drm_i915_gem_object *obj)
 {
-	return obj->mapping ? 'M' : ' ';
+	return obj->mm.mapping ? 'M' : ' ';
 }
 
 static u64 i915_gem_obj_total_ggtt_size(struct drm_i915_gem_object *obj)
@@ -158,8 +158,8 @@ describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj)
 		   i915_gem_active_get_seqno(&obj->last_write,
 					     &obj->base.dev->struct_mutex),
 		   i915_cache_level_str(dev_priv, obj->cache_level),
-		   obj->dirty ? " dirty" : "",
-		   obj->madv == I915_MADV_DONTNEED ? " purgeable" : "");
+		   obj->mm.dirty ? " dirty" : "",
+		   obj->mm.madv == I915_MADV_DONTNEED ? " purgeable" : "");
 	if (obj->base.name)
 		seq_printf(m, " (name: %d)", obj->base.name);
 	list_for_each_entry(vma, &obj->vma_list, obj_link) {
@@ -411,12 +411,12 @@ static int i915_gem_object_info(struct seq_file *m, void *data)
 		size += obj->base.size;
 		++count;
 
-		if (obj->madv == I915_MADV_DONTNEED) {
+		if (obj->mm.madv == I915_MADV_DONTNEED) {
 			purgeable_size += obj->base.size;
 			++purgeable_count;
 		}
 
-		if (obj->mapping) {
+		if (obj->mm.mapping) {
 			mapped_count++;
 			mapped_size += obj->base.size;
 		}
@@ -433,12 +433,12 @@ static int i915_gem_object_info(struct seq_file *m, void *data)
 			++dpy_count;
 		}
 
-		if (obj->madv == I915_MADV_DONTNEED) {
+		if (obj->mm.madv == I915_MADV_DONTNEED) {
 			purgeable_size += obj->base.size;
 			++purgeable_count;
 		}
 
-		if (obj->mapping) {
+		if (obj->mm.mapping) {
 			mapped_count++;
 			mapped_size += obj->base.size;
 		}
@@ -2018,7 +2018,7 @@ static void i915_dump_lrc_obj(struct seq_file *m,
 		seq_printf(m, "\tBound in GGTT at 0x%08x\n",
 			   i915_ggtt_offset(vma));
 
-	if (i915_gem_object_get_pages(vma->obj)) {
+	if (i915_gem_object_pin_pages(vma->obj)) {
 		seq_puts(m, "\tFailed to get pages for context object\n\n");
 		return;
 	}
@@ -2037,6 +2037,7 @@ static void i915_dump_lrc_obj(struct seq_file *m,
 		kunmap_atomic(reg_state);
 	}
 
+	i915_gem_object_unpin_pages(vma->obj);
 	seq_putc(m, '\n');
 }
 
drivers/gpu/drm/i915/i915_drv.h +60 -29

124 lines not on this page · raw message

drivers/gpu/drm/i915/i915_gem.c +96 -111

578 lines not on this page · raw message

drivers/gpu/drm/i915/i915_gem_batch_pool.c +1 -2
index 3934c9103cf2..6b656822bb3a 100644
@@ -131,11 +131,10 @@ i915_gem_batch_pool_get(struct i915_gem_batch_pool *pool,
 			return obj;
 	}
 
-	ret = i915_gem_object_get_pages(obj);
+	ret = i915_gem_object_pin_pages(obj);
 	if (ret)
 		return ERR_PTR(ret);
 
 	list_move_tail(&obj->batch_pool_link, list);
-	i915_gem_object_pin_pages(obj);
 	return obj;
 }
drivers/gpu/drm/i915/i915_gem_dmabuf.c +6 -8
index 97c9d68b45df..10441dc72e73 100644
@@ -48,12 +48,10 @@ static struct sg_table *i915_gem_map_dma_buf(struct dma_buf_attachment *attachme
 	if (ret)
 		goto err;
 
-	ret = i915_gem_object_get_pages(obj);
+	ret = i915_gem_object_pin_pages(obj);
 	if (ret)
 		goto err_unlock;
 
-	i915_gem_object_pin_pages(obj);
-
 	/* Copy sg so that we make an independent mapping */
 	st = kmalloc(sizeof(struct sg_table), GFP_KERNEL);
 	if (st == NULL) {
@@ -61,13 +59,13 @@ static struct sg_table *i915_gem_map_dma_buf(struct dma_buf_attachment *attachme
 		goto err_unpin;
 	}
 
-	ret = sg_alloc_table(st, obj->pages->nents, GFP_KERNEL);
+	ret = sg_alloc_table(st, obj->mm.pages->nents, GFP_KERNEL);
 	if (ret)
 		goto err_free;
 
-	src = obj->pages->sgl;
+	src = obj->mm.pages->sgl;
 	dst = st->sgl;
-	for (i = 0; i < obj->pages->nents; i++) {
+	for (i = 0; i < obj->mm.pages->nents; i++) {
 		sg_set_page(dst, sg_page(src), src->length, 0);
 		dst = sg_next(dst);
 		src = sg_next(src);
@@ -299,14 +297,14 @@ static int i915_gem_object_get_pages_dmabuf(struct drm_i915_gem_object *obj)
 	if (IS_ERR(sg))
 		return PTR_ERR(sg);
 
-	obj->pages = sg;
+	obj->mm.pages = sg;
 	return 0;
 }
 
 static void i915_gem_object_put_pages_dmabuf(struct drm_i915_gem_object *obj)
 {
 	dma_buf_unmap_attachment(obj->base.import_attach,
-				 obj->pages, DMA_BIDIRECTIONAL);
+				 obj->mm.pages, DMA_BIDIRECTIONAL);
 }
 
 static const struct drm_i915_gem_object_ops i915_gem_object_dmabuf_ops = {
drivers/gpu/drm/i915/i915_gem_execbuffer.c +1 -1
index 0deecd4e3b6c..062e098b0909 100644
@@ -1290,7 +1290,7 @@ void i915_vma_move_to_active(struct i915_vma *vma,
 
 	GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
 
-	obj->dirty = 1; /* be paranoid  */
+	obj->mm.dirty = true; /* be paranoid  */
 
 	/* Add a reference if we're newly entering the active list.
 	 * The order in which we add operations to the retirement queue is
drivers/gpu/drm/i915/i915_gem_fence.c +2 -2
index 8df1fa7234e8..398856160656 100644
@@ -648,7 +648,7 @@ i915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object *obj)
 		return;
 
 	i = 0;
-	for_each_sgt_page(page, sgt_iter, obj->pages) {
+	for_each_sgt_page(page, sgt_iter, obj->mm.pages) {
 		char new_bit_17 = page_to_phys(page) >> 17;
 		if ((new_bit_17 & 0x1) !=
 		    (test_bit(i, obj->bit_17) != 0)) {
@@ -687,7 +687,7 @@ i915_gem_object_save_bit_17_swizzle(struct drm_i915_gem_object *obj)
 
 	i = 0;
 
-	for_each_sgt_page(page, sgt_iter, obj->pages) {
+	for_each_sgt_page(page, sgt_iter, obj->mm.pages) {
 		if (page_to_phys(page) & (1 << 17))
 			__set_bit(i, obj->bit_17);
 		else
drivers/gpu/drm/i915/i915_gem_gtt.c +5 -5
index 7e0c98576e72..3c711e0b8a3f 100644
@@ -175,7 +175,7 @@ static int ppgtt_bind_vma(struct i915_vma *vma,
 {
 	u32 pte_flags = 0;
 
-	vma->pages = vma->obj->pages;
+	vma->pages = vma->obj->mm.pages;
 
 	/* Currently applicable only to VLV */
 	if (vma->obj->gt_ro)
@@ -2295,7 +2295,7 @@ void i915_gem_suspend_gtt_mappings(struct drm_device *dev)
 int i915_gem_gtt_prepare_object(struct drm_i915_gem_object *obj)
 {
 	if (!dma_map_sg(&obj->base.dev->pdev->dev,
-			obj->pages->sgl, obj->pages->nents,
+			obj->mm.pages->sgl, obj->mm.pages->nents,
 			PCI_DMA_BIDIRECTIONAL))
 		return -ENOSPC;
 
@@ -2685,7 +2685,7 @@ void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj)
 		}
 	}
 
-	dma_unmap_sg(kdev, obj->pages->sgl, obj->pages->nents,
+	dma_unmap_sg(kdev, obj->mm.pages->sgl, obj->mm.pages->nents,
 		     PCI_DMA_BIDIRECTIONAL);
 }
 
@@ -3526,7 +3526,7 @@ intel_rotate_fb_obj_pages(const struct intel_rotation_info *rot_info,
 
 	/* Populate source page list from the object. */
 	i = 0;
-	for_each_sgt_dma(dma_addr, sgt_iter, obj->pages)
+	for_each_sgt_dma(dma_addr, sgt_iter, obj->mm.pages)
 		page_addr_list[i++] = dma_addr;
 
 	GEM_BUG_ON(i != n_pages);
@@ -3617,7 +3617,7 @@ i915_get_ggtt_vma_pages(struct i915_vma *vma)
 		return 0;
 
 	if (vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL)
-		vma->pages = vma->obj->pages;
+		vma->pages = vma->obj->mm.pages;
 	else if (vma->ggtt_view.type == I915_GGTT_VIEW_ROTATED)
 		vma->pages =
 			intel_rotate_fb_obj_pages(&vma->ggtt_view.params.rotated, vma->obj);
drivers/gpu/drm/i915/i915_gem_internal.c +10 -9
index 534a61c1aba2..24eb347f9e0a 100644
@@ -28,10 +28,11 @@
 
 static void internal_free_pages(struct sg_table *st)
 {
-	struct sg_page_iter sg_iter;
+	struct sgt_iter iter;
+	struct page *page;
 
-	for_each_sg_page(st->sgl, &sg_iter, st->nents, 0)
-		put_page(sg_page_iter_page(&sg_iter));
+	for_each_sgt_page(page, iter, st)
+		put_page(page);
 
 	sg_free_table(st);
 	kfree(st);
@@ -94,10 +95,10 @@ static int i915_gem_object_get_pages_internal(struct drm_i915_gem_object *obj)
 	if (!swiotlb_nr_tbl())
 #endif
 		sg_mark_end(sg);
-	obj->pages = st;
+	obj->mm.pages = st;
 
 	if (i915_gem_gtt_prepare_object(obj)) {
-		obj->pages = NULL;
+		obj->mm.pages = NULL;
 		goto err;
 	}
 
@@ -106,7 +107,7 @@ static int i915_gem_object_get_pages_internal(struct drm_i915_gem_object *obj)
 	 * and the caller is expected to repopulate - the contents of this
 	 * object are only valid whilst active and pinned.
 	 */
-	obj->madv = I915_MADV_DONTNEED;
+	obj->mm.madv = I915_MADV_DONTNEED;
 	return 0;
 
 err:
@@ -117,10 +118,10 @@ err:
 
 static void i915_gem_object_put_pages_internal(struct drm_i915_gem_object *obj)
 {
-	internal_free_pages(obj->pages);
+	internal_free_pages(obj->mm.pages);
 
-	obj->dirty = 0;
-	obj->madv = I915_MADV_WILLNEED;
+	obj->mm.dirty = false;
+	obj->mm.madv = I915_MADV_WILLNEED;
 }
 
 static const struct drm_i915_gem_object_ops i915_gem_object_internal_ops = {
drivers/gpu/drm/i915/i915_gem_render_state.c +1 -1
index adb00c5125ad..64d9712ec789 100644
@@ -230,7 +230,7 @@ int i915_gem_render_state_emit(struct drm_i915_gem_request *req)
 		return 0;
 
 	/* Recreate the page after shrinking */
-	if (!so->vma->obj->pages)
+	if (!so->vma->obj->mm.pages)
 		so->batch_offset = -1;
 
 	ret = i915_vma_pin(so->vma, 0, 0, PIN_GLOBAL | PIN_HIGH);
drivers/gpu/drm/i915/i915_gem_shrinker.c +5 -5
index 1c237d02f30b..e0e6d68fe470 100644
@@ -78,7 +78,7 @@ static bool can_release_pages(struct drm_i915_gem_object *obj)
 	 * to the GPU, simply unbinding from the GPU is not going to succeed
 	 * in releasing our pin count on the pages themselves.
 	 */
-	if (obj->pages_pin_count > obj->bind_count)
+	if (obj->mm.pages_pin_count > obj->bind_count)
 		return false;
 
 	if (any_vma_pinned(obj))
@@ -88,7 +88,7 @@ static bool can_release_pages(struct drm_i915_gem_object *obj)
 	 * discard the contents (because the user has marked them as being
 	 * purgeable) or if we can move their contents out to swap.
 	 */
-	return swap_available() || obj->madv == I915_MADV_DONTNEED;
+	return swap_available() || obj->mm.madv == I915_MADV_DONTNEED;
 }
 
 /**
@@ -175,11 +175,11 @@ i915_gem_shrink(struct drm_i915_private *dev_priv,
 			list_move_tail(&obj->global_list, &still_in_list);
 
 			if (flags & I915_SHRINK_PURGEABLE &&
-			    obj->madv != I915_MADV_DONTNEED)
+			    obj->mm.madv != I915_MADV_DONTNEED)
 				continue;
 
 			if (flags & I915_SHRINK_VMAPS &&
-			    !is_vmalloc_addr(obj->mapping))
+			    !is_vmalloc_addr(obj->mm.mapping))
 				continue;
 
 			if ((flags & I915_SHRINK_ACTIVE) == 0 &&
@@ -193,7 +193,7 @@ i915_gem_shrink(struct drm_i915_private *dev_priv,
 
 			/* For the unbound phase, this should be a no-op! */
 			i915_gem_object_unbind(obj);
-			if (i915_gem_object_put_pages(obj) == 0)
+			if (__i915_gem_object_put_pages(obj) == 0)
 				count += obj->base.size >> PAGE_SHIFT;
 
 			i915_gem_object_put(obj);
drivers/gpu/drm/i915/i915_gem_stolen.c +13 -11
index 24bad4e60ef0..6d139d6b4609 100644
@@ -554,16 +554,17 @@ static int i915_gem_object_get_pages_stolen(struct drm_i915_gem_object *obj)
 static void i915_gem_object_put_pages_stolen(struct drm_i915_gem_object *obj)
 {
 	/* Should only be called during free */
-	sg_free_table(obj->pages);
-	kfree(obj->pages);
+	sg_free_table(obj->mm.pages);
+	kfree(obj->mm.pages);
 }
 
-
 static void
 i915_gem_object_release_stolen(struct drm_i915_gem_object *obj)
 {
 	struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
 
+	__i915_gem_object_unpin_pages(obj);
+
 	if (obj->stolen) {
 		i915_gem_stolen_remove_node(dev_priv, obj->stolen);
 		kfree(obj->stolen);
@@ -589,15 +590,16 @@ _i915_gem_object_create_stolen(struct drm_device *dev,
 	drm_gem_private_object_init(dev, &obj->base, stolen->size);
 	i915_gem_object_init(obj, &i915_gem_object_stolen_ops);
 
-	obj->pages = i915_pages_create_for_stolen(dev,
-						  stolen->start, stolen->size);
-	if (obj->pages == NULL)
+	obj->mm.pages = i915_pages_create_for_stolen(dev,
+						     stolen->start,
+						     stolen->size);
+	if (!obj->mm.pages)
 		goto cleanup;
 
-	obj->get_page.sg_pos = obj->pages->sgl;
-	obj->get_page.sg_idx = 0;
+	obj->mm.get_page.sg_pos = obj->mm.pages->sgl;
+	obj->mm.get_page.sg_idx = 0;
 
-	i915_gem_object_pin_pages(obj);
+	__i915_gem_object_pin_pages(obj);
 	obj->stolen = stolen;
 
 	obj->base.read_domains = I915_GEM_DOMAIN_CPU | I915_GEM_DOMAIN_GTT;
@@ -717,14 +719,14 @@ i915_gem_object_create_stolen_for_preallocated(struct drm_device *dev,
 		goto err;
 	}
 
-	vma->pages = obj->pages;
+	vma->pages = obj->mm.pages;
 	vma->flags |= I915_VMA_GLOBAL_BIND;
 	__i915_vma_set_map_and_fenceable(vma);
 	list_move_tail(&vma->vm_link, &ggtt->base.inactive_list);
 	obj->bind_count++;
 
 	list_add_tail(&obj->global_list, &dev_priv->mm.bound_list);
-	i915_gem_object_pin_pages(obj);
+	__i915_gem_object_pin_pages(obj);
 
 	return obj;
 
drivers/gpu/drm/i915/i915_gem_tiling.c +4 -4
index a14b1e3d4c78..7286de7bd25e 100644
@@ -260,13 +260,13 @@ i915_gem_set_tiling(struct drm_device *dev, void *data,
 		if (!err) {
 			struct i915_vma *vma;
 
-			if (obj->pages &&
-			    obj->madv == I915_MADV_WILLNEED &&
+			if (obj->mm.pages &&
+			    obj->mm.madv == I915_MADV_WILLNEED &&
 			    dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES) {
 				if (args->tiling_mode == I915_TILING_NONE)
-					i915_gem_object_unpin_pages(obj);
+					__i915_gem_object_unpin_pages(obj);
 				if (!i915_gem_object_is_tiled(obj))
-					i915_gem_object_pin_pages(obj);
+					__i915_gem_object_pin_pages(obj);
 			}
 
 			list_for_each_entry(vma, &obj->vma_list, obj_link) {
drivers/gpu/drm/i915/i915_gem_userptr.c +15 -15
index cb95789da76e..0fdd1d6723d1 100644
@@ -73,10 +73,10 @@ static void cancel_userptr(struct work_struct *work)
 	/* Cancel any active worker and force us to re-evaluate gup */
 	obj->userptr.work = NULL;
 
-	if (obj->pages != NULL) {
+	if (obj->mm.pages) {
 		/* We are inside a kthread context and can't be interrupted */
 		WARN_ON(i915_gem_object_unbind(obj));
-		WARN_ON(i915_gem_object_put_pages(obj));
+		WARN_ON(__i915_gem_object_put_pages(obj));
 	}
 
 	i915_gem_object_put(obj);
@@ -432,15 +432,15 @@ __i915_gem_userptr_set_pages(struct drm_i915_gem_object *obj,
 {
 	int ret;
 
-	ret = st_set_pages(&obj->pages, pvec, num_pages);
+	ret = st_set_pages(&obj->mm.pages, pvec, num_pages);
 	if (ret)
 		return ret;
 
 	ret = i915_gem_gtt_prepare_object(obj);
 	if (ret) {
-		sg_free_table(obj->pages);
-		kfree(obj->pages);
-		obj->pages = NULL;
+		sg_free_table(obj->mm.pages);
+		kfree(obj->mm.pages);
+		obj->mm.pages = NULL;
 	}
 
 	return ret;
@@ -526,8 +526,8 @@ __i915_gem_userptr_get_pages_worker(struct work_struct *_work)
 			if (ret == 0) {
 				list_add_tail(&obj->global_list,
 					      &to_i915(dev)->mm.unbound_list);
-				obj->get_page.sg_pos = obj->pages->sgl;
-				obj->get_page.sg_idx = 0;
+				obj->mm.get_page.sg_pos = obj->mm.pages->sgl;
+				obj->mm.get_page.sg_idx = 0;
 				pinned = 0;
 			}
 		}
@@ -668,22 +668,22 @@ i915_gem_userptr_put_pages(struct drm_i915_gem_object *obj)
 	BUG_ON(obj->userptr.work != NULL);
 	__i915_gem_userptr_set_active(obj, false);
 
-	if (obj->madv != I915_MADV_WILLNEED)
-		obj->dirty = 0;
+	if (obj->mm.madv != I915_MADV_WILLNEED)
+		obj->mm.dirty = false;
 
 	i915_gem_gtt_finish_object(obj);
 
-	for_each_sgt_page(page, sgt_iter, obj->pages) {
-		if (obj->dirty)
+	for_each_sgt_page(page, sgt_iter, obj->mm.pages) {
+		if (obj->mm.dirty)
 			set_page_dirty(page);
 
 		mark_page_accessed(page);
 		put_page(page);
 	}
-	obj->dirty = 0;
+	obj->mm.dirty = false;
 
-	sg_free_table(obj->pages);
-	kfree(obj->pages);
+	sg_free_table(obj->mm.pages);
+	kfree(obj->mm.pages);
 }
 
 static void
drivers/gpu/drm/i915/i915_gpu_error.c +2 -2
index 6c22be28ba01..628d5cdf9200 100644
@@ -881,8 +881,8 @@ static void capture_bo(struct drm_i915_error_buffer *err,
 	err->write_domain = obj->base.write_domain;
 	err->fence_reg = vma->fence ? vma->fence->id : -1;
 	err->tiling = i915_gem_object_get_tiling(obj);
-	err->dirty = obj->dirty;
-	err->purgeable = obj->madv != I915_MADV_WILLNEED;
+	err->dirty = obj->mm.dirty;
+	err->purgeable = obj->mm.madv != I915_MADV_WILLNEED;
 	err->userptr = obj->userptr.mm != NULL;
 	err->cache_level = obj->cache_level;
 }
drivers/gpu/drm/i915/intel_lrc.c +3 -3
index 02c48d390148..6acecfc41f5e 100644
@@ -744,7 +744,7 @@ static int intel_lr_context_pin(struct i915_gem_context *ctx,
 	ce->lrc_reg_state[CTX_RING_BUFFER_START+1] =
 		i915_ggtt_offset(ce->ring->vma);
 
-	ce->state->obj->dirty = true;
+	ce->state->obj->mm.dirty = true;
 
 	/* Invalidate GuC TLB. */
 	if (i915.enable_guc_submission) {
@@ -2042,7 +2042,7 @@ populate_lr_context(struct i915_gem_context *ctx,
 		DRM_DEBUG_DRIVER("Could not map object pages! (%d)\n", ret);
 		return ret;
 	}
-	ctx_obj->dirty = true;
+	ctx_obj->mm.dirty = true;
 
 	/* The second page of the context object contains some fields which must
 	 * be set up prior to the first execution. */
@@ -2179,7 +2179,7 @@ void intel_lr_context_resume(struct drm_i915_private *dev_priv)
 			reg[CTX_RING_HEAD+1] = 0;
 			reg[CTX_RING_TAIL+1] = 0;
 
-			ce->state->obj->dirty = true;
+			ce->state->obj->mm.dirty = true;
 			i915_gem_object_unpin_map(ce->state->obj);
 
 			ce->ring->head = ce->ring->tail = 0;
Signature · 6 lines
2.9.3

_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
#49 lore raw

Re: [PATCH 16/42] drm/i915: Refactor object page API

Quoted 94 lines · Chris Wilson

The plan is to make obtaining the backing storage for the object avoid struct_mutex (i.e. use its own locking). The first step is to update the API so that normal users only call pin/unpin whilst working on the backing storage.

---
  drivers/gpu/drm/i915/i915_cmd_parser.c       |   2 +-
  drivers/gpu/drm/i915/i915_debugfs.c          |  17 +--
  drivers/gpu/drm/i915/i915_drv.h              |  89 ++++++++----
  drivers/gpu/drm/i915/i915_gem.c              | 207 +++++++++++++--------------
  drivers/gpu/drm/i915/i915_gem_batch_pool.c   |   3 +-
  drivers/gpu/drm/i915/i915_gem_dmabuf.c       |  14 +-
  drivers/gpu/drm/i915/i915_gem_execbuffer.c   |   2 +-
  drivers/gpu/drm/i915/i915_gem_fence.c        |   4 +-
  drivers/gpu/drm/i915/i915_gem_gtt.c          |  10 +-
  drivers/gpu/drm/i915/i915_gem_internal.c     |  19 +--
  drivers/gpu/drm/i915/i915_gem_render_state.c |   2 +-
  drivers/gpu/drm/i915/i915_gem_shrinker.c     |  10 +-
  drivers/gpu/drm/i915/i915_gem_stolen.c       |  24 ++--
  drivers/gpu/drm/i915/i915_gem_tiling.c       |   8 +-
  drivers/gpu/drm/i915/i915_gem_userptr.c      |  30 ++--
  drivers/gpu/drm/i915/i915_gpu_error.c        |   4 +-
  drivers/gpu/drm/i915/intel_lrc.c             |   6 +-
  17 files changed, 234 insertions(+), 217 deletions(-)
drivers/gpu/drm/i915/i915_cmd_parser.c +1 -1
index 70980f82a15b..8d20020cb9f9 100644
@@ -1290,7 +1290,7 @@ int intel_engine_cmd_parser(struct intel_engine_cs *engine,
  	}
 
  	if (ret == 0 && needs_clflush_after)
-		drm_clflush_virt_range(shadow_batch_obj->mapping, batch_len);
+		drm_clflush_virt_range(shadow_batch_obj->mm.mapping, batch_len);
  	i915_gem_object_unpin_map(shadow_batch_obj);
 
  	return ret;
drivers/gpu/drm/i915/i915_debugfs.c +7 -7
index e4b5ba771bea..b807ddf65e04 100644
@@ -112,7 +112,7 @@ static char get_global_flag(struct drm_i915_gem_object *obj)
 
  static char get_pin_mapped_flag(struct drm_i915_gem_object *obj)
  {
-	return obj->mapping ? 'M' : ' ';
+	return obj->mm.mapping ? 'M' : ' ';
  }
 
  static u64 i915_gem_obj_total_ggtt_size(struct drm_i915_gem_object *obj)
@@ -158,8 +158,8 @@ describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj)
  		   i915_gem_active_get_seqno(&obj->last_write,
  					     &obj->base.dev->struct_mutex),
  		   i915_cache_level_str(dev_priv, obj->cache_level),
-		   obj->dirty ? " dirty" : "",
-		   obj->madv == I915_MADV_DONTNEED ? " purgeable" : "");
+		   obj->mm.dirty ? " dirty" : "",
+		   obj->mm.madv == I915_MADV_DONTNEED ? " purgeable" : "");
  	if (obj->base.name)
  		seq_printf(m, " (name: %d)", obj->base.name);
  	list_for_each_entry(vma, &obj->vma_list, obj_link) {
@@ -411,12 +411,12 @@ static int i915_gem_object_info(struct seq_file *m, void *data)
  		size += obj->base.size;
  		++count;
 
-		if (obj->madv == I915_MADV_DONTNEED) {
+		if (obj->mm.madv == I915_MADV_DONTNEED) {
  			purgeable_size += obj->base.size;
  			++purgeable_count;
  		}
 
-		if (obj->mapping) {
+		if (obj->mm.mapping) {
  			mapped_count++;
  			mapped_size += obj->base.size;
  		}
@@ -433,12 +433,12 @@ static int i915_gem_object_info(struct seq_file *m, void *data)
  			++dpy_count;
  		}
 
-		if (obj->madv == I915_MADV_DONTNEED) {
+		if (obj->mm.madv == I915_MADV_DONTNEED) {
  			purgeable_size += obj->base.size;
  			++purgeable_count;
  		}
 
-		if (obj->mapping) {
+		if (obj->mm.mapping) {
  			mapped_count++;
  			mapped_size += obj->base.size;
  		}

drivers/gpu/drm/i915/i915_debugfs.c static void i915_dump_lrc_obj(struct seq_file *m, · line 2022

Quoted 6 lines
drivers/gpu/drm/i915/i915_debugfs.c +1 -1
@@ -2018,7 +2018,7 @@ static void i915_dump_lrc_obj(struct seq_file *m,
  		seq_printf(m, "\tBound in GGTT at 0x%08x\n",
  			   i915_ggtt_offset(vma));
 
-	if (i915_gem_object_get_pages(vma->obj)) {
+	if (i915_gem_object_pin_pages(vma->obj)) {
  		seq_puts(m, "\tFailed to get pages for context object\n\n");

Error message needs updating to match the function call change.

Quoted 126 lines
drivers/gpu/drm/i915/i915_debugfs.c +1 -0
  		return;
  	}
@@ -2037,6 +2037,7 @@ static void i915_dump_lrc_obj(struct seq_file *m,
  		kunmap_atomic(reg_state);
  	}
 
+	i915_gem_object_unpin_pages(vma->obj);
  	seq_putc(m, '\n');
  }
 
drivers/gpu/drm/i915/i915_drv.h +55 -27
index a96b446d8db4..3c22d49005fe 100644
@@ -2238,17 +2238,6 @@ struct drm_i915_gem_object {
  #define I915_BO_ACTIVE_REF (I915_BO_ACTIVE_SHIFT + I915_NUM_ENGINES)
 
  	/**
-	 * This is set if the object has been written to since last bound
-	 * to the GTT
-	 */
-	unsigned int dirty:1;
-
-	/**
-	 * Advice: are the backing pages purgeable?
-	 */
-	unsigned int madv:2;
-
-	/**
  	 * Whether the current gtt mapping needs to be mappable (and isn't just
  	 * mappable by accident). Track pin and fault separate for a more
  	 * accurate mappable working set.
@@ -2276,16 +2265,31 @@ struct drm_i915_gem_object {
  	unsigned int bind_count;
  	unsigned int pin_display;
 
-	struct sg_table *pages;
-	int pages_pin_count;
-	struct i915_gem_object_page_iter {
-		struct scatterlist *sg_pos;
-		unsigned long sg_idx;
+	struct {
+		unsigned int pages_pin_count;
+
+		struct sg_table *pages;
+		void *mapping;
+
+		struct i915_gem_object_page_iter {
+			struct scatterlist *sg_pos;
+			unsigned long sg_idx;
 
-		struct radix_tree_root radix;
-		struct mutex lock;
-	} get_page;
-	void *mapping;
+			struct radix_tree_root radix;
+			struct mutex lock;
+		} get_page;
+
+		/**
+		 * Advice: are the backing pages purgeable?
+		 */
+		unsigned int madv:2;
+
+		/**
+		 * This is set if the object has been written to since the
+		 * pages were last acquired.
+		 */
+		unsigned int dirty:1;
+	} mm;
 
  	/** Breadcrumb of last rendering to the buffer.
  	 * There can only be one writer, but we allow for multiple readers.
@@ -3160,13 +3164,10 @@ void i915_vma_close(struct i915_vma *vma);
  void i915_vma_destroy(struct i915_vma *vma);
 
  int i915_gem_object_unbind(struct drm_i915_gem_object *obj);
-int i915_gem_object_put_pages(struct drm_i915_gem_object *obj);
  void i915_gem_release_all_mmaps(struct drm_i915_private *dev_priv);
  void i915_gem_release_mmap(struct drm_i915_gem_object *obj);
 
-int __must_check i915_gem_object_get_pages(struct drm_i915_gem_object *obj);
-
-static inline int __sg_page_count(struct scatterlist *sg)
+static inline int __sg_page_count(const struct scatterlist *sg)
  {
  	return sg->length >> PAGE_SHIFT;
  }
@@ -3187,18 +3188,48 @@ dma_addr_t
  i915_gem_object_get_dma_address(struct drm_i915_gem_object *obj,
  				unsigned long n);
 
-static inline void i915_gem_object_pin_pages(struct drm_i915_gem_object *obj)
+int __i915_gem_object_get_pages(struct drm_i915_gem_object *obj);
+
+static inline int __must_check
+i915_gem_object_pin_pages(struct drm_i915_gem_object *obj)
  {
-	BUG_ON(obj->pages == NULL);
-	obj->pages_pin_count++;
+	lockdep_assert_held(&obj->base.dev->struct_mutex);
+	if (obj->mm.pages_pin_count++)
+		return 0;
+
+	return __i915_gem_object_get_pages(obj);
+}
+
+static inline void
+__i915_gem_object_pin_pages(struct drm_i915_gem_object *obj)
+{
+	lockdep_assert_held(&obj->base.dev->struct_mutex);
+	GEM_BUG_ON(!obj->mm.pages);
+	obj->mm.pages_pin_count++;
+}
+
+static inline bool
+i915_gem_object_has_pinned_pages(struct drm_i915_gem_object *obj)
+{
+	return obj->mm.pages_pin_count;
+}
+
+static inline void
+__i915_gem_object_unpin_pages(struct drm_i915_gem_object *obj)
+{
+	lockdep_assert_held(&obj->base.dev->struct_mutex);
+	GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));

drivers/gpu/drm/i915/i915_drv.h dma_addr_t · line 3229

Quoted 10 lines
drivers/gpu/drm/i915/i915_drv.h +3 -2
+	GEM_BUG_ON(!obj->mm.pages);
+	obj->mm.pages_pin_count--;
  }
 
  static inline void i915_gem_object_unpin_pages(struct drm_i915_gem_object *obj)
  {
-	BUG_ON(obj->pages_pin_count == 0);
-	obj->pages_pin_count--;
+	__i915_gem_object_unpin_pages(obj);
  }

Am I missing something or is the above identical to simply '#define i915_gem_object_unpin_pages __i915_gem_object_unpin_pages'? In which case, why bother with having two separate functions?

Quoted 1,102 lines
drivers/gpu/drm/i915/i915_drv.h +2 -0
+int __i915_gem_object_put_pages(struct drm_i915_gem_object *obj);
+
  enum i915_map_type {
  	I915_MAP_WB = 0,
  	I915_MAP_WC,
drivers/gpu/drm/i915/i915_gem.c +96 -111
index af7d51f16658..df774ddf62ae 100644
@@ -216,7 +216,7 @@ i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj)
  	sg_dma_address(sg) = obj->phys_handle->busaddr;
  	sg_dma_len(sg) = obj->base.size;
 
-	obj->pages = st;
+	obj->mm.pages = st;
  	return 0;
  }
 
@@ -225,7 +225,7 @@ i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj)
  {
  	int ret;
 
-	BUG_ON(obj->madv == __I915_MADV_PURGED);
+	GEM_BUG_ON(obj->mm.madv == __I915_MADV_PURGED);
 
  	ret = i915_gem_object_set_to_cpu_domain(obj, true);
  	if (WARN_ON(ret)) {
@@ -235,10 +235,10 @@ i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj)
  		obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
  	}
 
-	if (obj->madv == I915_MADV_DONTNEED)
-		obj->dirty = 0;
+	if (obj->mm.madv == I915_MADV_DONTNEED)
+		obj->mm.dirty = false;
 
-	if (obj->dirty) {
+	if (obj->mm.dirty) {
  		struct address_space *mapping = obj->base.filp->f_mapping;
  		char *vaddr = obj->phys_handle->vaddr;
  		int i;
@@ -257,22 +257,23 @@ i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj)
  			kunmap_atomic(dst);
 
  			set_page_dirty(page);
-			if (obj->madv == I915_MADV_WILLNEED)
+			if (obj->mm.madv == I915_MADV_WILLNEED)
  				mark_page_accessed(page);
  			put_page(page);
  			vaddr += PAGE_SIZE;
  		}
-		obj->dirty = 0;
+		obj->mm.dirty = false;
  	}
 
-	sg_free_table(obj->pages);
-	kfree(obj->pages);
+	sg_free_table(obj->mm.pages);
+	kfree(obj->mm.pages);
  }
 
  static void
  i915_gem_object_release_phys(struct drm_i915_gem_object *obj)
  {
  	drm_pci_free(obj->base.dev, obj->phys_handle);
+	i915_gem_object_unpin_pages(obj);
  }
 
  static const struct drm_i915_gem_object_ops i915_gem_phys_ops = {
@@ -506,7 +507,7 @@ i915_gem_object_attach_phys(struct drm_i915_gem_object *obj,
  		return 0;
  	}
 
-	if (obj->madv != I915_MADV_WILLNEED)
+	if (obj->mm.madv != I915_MADV_WILLNEED)
  		return -EFAULT;
 
  	if (obj->base.filp == NULL)
@@ -516,7 +517,7 @@ i915_gem_object_attach_phys(struct drm_i915_gem_object *obj,
  	if (ret)
  		return ret;
 
-	ret = i915_gem_object_put_pages(obj);
+	ret = __i915_gem_object_put_pages(obj);
  	if (ret)
  		return ret;
 
@@ -528,7 +529,7 @@ i915_gem_object_attach_phys(struct drm_i915_gem_object *obj,
  	obj->phys_handle = phys;
  	obj->ops = &i915_gem_phys_ops;
 
-	return i915_gem_object_get_pages(obj);
+	return i915_gem_object_pin_pages(obj);
  }
 
  static int
@@ -724,12 +725,10 @@ int i915_gem_obj_prepare_shmem_read(struct drm_i915_gem_object *obj,
  	if (ret)
  		return ret;
 
-	ret = i915_gem_object_get_pages(obj);
+	ret = i915_gem_object_pin_pages(obj);
  	if (ret)
  		return ret;
 
-	i915_gem_object_pin_pages(obj);
-
  	i915_gem_object_flush_gtt_write_domain(obj);
 
  	/* If we're not in the cpu read domain, set ourself into the gtt
@@ -777,12 +776,10 @@ int i915_gem_obj_prepare_shmem_write(struct drm_i915_gem_object *obj,
  	if (ret)
  		return ret;
 
-	ret = i915_gem_object_get_pages(obj);
+	ret = i915_gem_object_pin_pages(obj);
  	if (ret)
  		return ret;
 
-	i915_gem_object_pin_pages(obj);
-
  	i915_gem_object_flush_gtt_write_domain(obj);
 
  	/* If we're not in the cpu write domain, set ourself into the
@@ -812,7 +809,7 @@ int i915_gem_obj_prepare_shmem_write(struct drm_i915_gem_object *obj,
  		obj->cache_dirty = true;
 
  	intel_fb_obj_invalidate(obj, ORIGIN_CPU);
-	obj->dirty = 1;
+	obj->mm.dirty = true;
  	/* return with the pages pinned */
  	return 0;
 
@@ -949,13 +946,11 @@ i915_gem_gtt_pread(struct drm_device *dev,
  		if (ret)
  			goto out;
 
-		ret = i915_gem_object_get_pages(obj);
+		ret = i915_gem_object_pin_pages(obj);
  		if (ret) {
  			remove_mappable_node(&node);
  			goto out;
  		}
-
-		i915_gem_object_pin_pages(obj);
  	}
 
  	ret = i915_gem_object_set_to_gtt_domain(obj, false);
@@ -1062,7 +1057,7 @@ i915_gem_shmem_pread(struct drm_device *dev,
  	offset = args->offset;
  	remain = args->size;
 
-	for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
+	for_each_sg_page(obj->mm.pages->sgl, &sg_iter, obj->mm.pages->nents,
  			 offset >> PAGE_SHIFT) {
  		struct page *page = sg_page_iter_page(&sg_iter);
 
@@ -1254,13 +1249,11 @@ i915_gem_gtt_pwrite_fast(struct drm_i915_private *i915,
  		if (ret)
  			goto out;
 
-		ret = i915_gem_object_get_pages(obj);
+		ret = i915_gem_object_pin_pages(obj);
  		if (ret) {
  			remove_mappable_node(&node);
  			goto out;
  		}
-
-		i915_gem_object_pin_pages(obj);
  	}
 
  	ret = i915_gem_object_set_to_gtt_domain(obj, true);
@@ -1268,7 +1261,7 @@ i915_gem_gtt_pwrite_fast(struct drm_i915_private *i915,
  		goto out_unpin;
 
  	intel_fb_obj_invalidate(obj, ORIGIN_CPU);
-	obj->dirty = true;
+	obj->mm.dirty = true;
 
  	user_data = u64_to_user_ptr(args->data_ptr);
  	offset = args->offset;
@@ -1439,7 +1432,7 @@ i915_gem_shmem_pwrite(struct drm_device *dev,
  	offset = args->offset;
  	remain = args->size;
 
-	for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
+	for_each_sg_page(obj->mm.pages->sgl, &sg_iter, obj->mm.pages->nents,
  			 offset >> PAGE_SHIFT) {
  		struct page *page = sg_page_iter_page(&sg_iter);
  		int partial_cacheline_write;
@@ -2228,7 +2221,7 @@ i915_gem_object_truncate(struct drm_i915_gem_object *obj)
  	 * backing pages, *now*.
  	 */
  	shmem_truncate_range(file_inode(obj->base.filp), 0, (loff_t)-1);
-	obj->madv = __I915_MADV_PURGED;
+	obj->mm.madv = __I915_MADV_PURGED;
  }
 
  /* Try to discard unwanted pages */
@@ -2237,7 +2230,7 @@ i915_gem_object_invalidate(struct drm_i915_gem_object *obj)
  {
  	struct address_space *mapping;
 
-	switch (obj->madv) {
+	switch (obj->mm.madv) {
  	case I915_MADV_DONTNEED:
  		i915_gem_object_truncate(obj);
  	case __I915_MADV_PURGED:
@@ -2258,7 +2251,7 @@ i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
  	struct page *page;
  	int ret;
 
-	BUG_ON(obj->madv == __I915_MADV_PURGED);
+	GEM_BUG_ON(obj->mm.madv == __I915_MADV_PURGED);
 
  	ret = i915_gem_object_set_to_cpu_domain(obj, true);
  	if (WARN_ON(ret)) {
@@ -2274,22 +2267,22 @@ i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
  	if (i915_gem_object_needs_bit17_swizzle(obj))
  		i915_gem_object_save_bit_17_swizzle(obj);
 
-	if (obj->madv == I915_MADV_DONTNEED)
-		obj->dirty = 0;
+	if (obj->mm.madv == I915_MADV_DONTNEED)
+		obj->mm.dirty = false;
 
-	for_each_sgt_page(page, sgt_iter, obj->pages) {
-		if (obj->dirty)
+	for_each_sgt_page(page, sgt_iter, obj->mm.pages) {
+		if (obj->mm.dirty)
  			set_page_dirty(page);
 
-		if (obj->madv == I915_MADV_WILLNEED)
+		if (obj->mm.madv == I915_MADV_WILLNEED)
  			mark_page_accessed(page);
 
  		put_page(page);
  	}
-	obj->dirty = 0;
+	obj->mm.dirty = false;
 
-	sg_free_table(obj->pages);
-	kfree(obj->pages);
+	sg_free_table(obj->mm.pages);
+	kfree(obj->mm.pages);
  }
 
  static void __i915_gem_object_reset_page_iter(struct drm_i915_gem_object *obj)
@@ -2297,21 +2290,20 @@ static void __i915_gem_object_reset_page_iter(struct drm_i915_gem_object *obj)
  	struct radix_tree_iter iter;
  	void **slot;
 
-	radix_tree_for_each_slot(slot, &obj->get_page.radix, &iter, 0)
-		radix_tree_delete(&obj->get_page.radix, iter.index);
+	radix_tree_for_each_slot(slot, &obj->mm.get_page.radix, &iter, 0)
+		radix_tree_delete(&obj->mm.get_page.radix, iter.index);
  }
 
-int
-i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
+int __i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
  {
  	const struct drm_i915_gem_object_ops *ops = obj->ops;
 
  	lockdep_assert_held(&obj->base.dev->struct_mutex);
 
-	if (obj->pages == NULL)
+	if (!obj->mm.pages)
  		return 0;
 
-	if (obj->pages_pin_count)
+	if (i915_gem_object_has_pinned_pages(obj))
  		return -EBUSY;
 
  	GEM_BUG_ON(obj->bind_count);
@@ -2321,22 +2313,22 @@ i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
  	 * lists early. */
  	list_del(&obj->global_list);
 
-	if (obj->mapping) {
+	if (obj->mm.mapping) {
  		void *ptr;
 
-		ptr = ptr_mask_bits(obj->mapping);
+		ptr = ptr_mask_bits(obj->mm.mapping);
  		if (is_vmalloc_addr(ptr))
  			vunmap(ptr);
  		else
  			kunmap(kmap_to_page(ptr));
 
-		obj->mapping = NULL;
+		obj->mm.mapping = NULL;
  	}
 
  	__i915_gem_object_reset_page_iter(obj);
 
  	ops->put_pages(obj);
-	obj->pages = NULL;
+	obj->mm.pages = NULL;
 
  	i915_gem_object_invalidate(obj);
 
@@ -2431,7 +2423,7 @@ i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
  	if (!swiotlb_nr_tbl())
  #endif
  		sg_mark_end(sg);
-	obj->pages = st;
+	obj->mm.pages = st;
 
  	ret = i915_gem_gtt_prepare_object(obj);
  	if (ret)
@@ -2442,7 +2434,7 @@ i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
 
  	if (i915_gem_object_is_tiled(obj) &&
  	    dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES)
-		i915_gem_object_pin_pages(obj);
+		__i915_gem_object_pin_pages(obj);
 
  	return 0;
 
@@ -2474,8 +2466,7 @@ err_pages:
   * either as a result of memory pressure (reaping pages under the shrinker)
   * or as the object is itself released.
   */
-int
-i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
+int __i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
  {
  	struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
  	const struct drm_i915_gem_object_ops *ops = obj->ops;
@@ -2483,24 +2474,25 @@ i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
 
  	lockdep_assert_held(&obj->base.dev->struct_mutex);
 
-	if (obj->pages)
+	if (obj->mm.pages)
  		return 0;
 
-	if (obj->madv != I915_MADV_WILLNEED) {
+	if (obj->mm.madv != I915_MADV_WILLNEED) {
  		DRM_DEBUG("Attempting to obtain a purgeable object\n");
+		__i915_gem_object_unpin_pages(obj);
  		return -EFAULT;
  	}
 
-	BUG_ON(obj->pages_pin_count);
-
  	ret = ops->get_pages(obj);
-	if (ret)
+	if (ret) {
+		__i915_gem_object_unpin_pages(obj);
  		return ret;
+	}
 
  	list_add_tail(&obj->global_list, &dev_priv->mm.unbound_list);
 
-	obj->get_page.sg_pos = obj->pages->sgl;
-	obj->get_page.sg_idx = 0;
+	obj->mm.get_page.sg_pos = obj->mm.pages->sgl;
+	obj->mm.get_page.sg_idx = 0;
 
  	return 0;
  }
@@ -2510,7 +2502,7 @@ static void *i915_gem_object_map(const struct drm_i915_gem_object *obj,
  				 enum i915_map_type type)
  {
  	unsigned long n_pages = obj->base.size >> PAGE_SHIFT;
-	struct sg_table *sgt = obj->pages;
+	struct sg_table *sgt = obj->mm.pages;
  	struct sgt_iter sgt_iter;
  	struct page *page;
  	struct page *stack_pages[32];
@@ -2564,14 +2556,13 @@ void *i915_gem_object_pin_map(struct drm_i915_gem_object *obj,
  	lockdep_assert_held(&obj->base.dev->struct_mutex);
  	GEM_BUG_ON(!i915_gem_object_has_struct_page(obj));
 
-	ret = i915_gem_object_get_pages(obj);
+	ret = i915_gem_object_pin_pages(obj);
  	if (ret)
  		return ERR_PTR(ret);
 
-	i915_gem_object_pin_pages(obj);
-	pinned = obj->pages_pin_count > 1;
+	pinned = obj->mm.pages_pin_count > 1;
 
-	ptr = ptr_unpack_bits(obj->mapping, has_type);
+	ptr = ptr_unpack_bits(obj->mm.mapping, has_type);
  	if (ptr && has_type != type) {
  		if (pinned) {
  			ret = -EBUSY;
@@ -2583,7 +2574,7 @@ void *i915_gem_object_pin_map(struct drm_i915_gem_object *obj,
  		else
  			kunmap(kmap_to_page(ptr));
 
-		ptr = obj->mapping = NULL;
+		ptr = obj->mm.mapping = NULL;
  	}
 
  	if (!ptr) {
@@ -2593,7 +2584,7 @@ void *i915_gem_object_pin_map(struct drm_i915_gem_object *obj,
  			goto err;
  		}
 
-		obj->mapping = ptr_pack_bits(ptr, type);
+		obj->mm.mapping = ptr_pack_bits(ptr, type);
  	}
 
  	return ptr;
@@ -3030,7 +3021,7 @@ int i915_vma_unbind(struct i915_vma *vma)
  		goto destroy;
 
  	GEM_BUG_ON(obj->bind_count == 0);
-	GEM_BUG_ON(!obj->pages);
+	GEM_BUG_ON(!obj->mm.pages);
 
  	if (i915_vma_is_map_and_fenceable(vma)) {
  		/* release the fence reg _after_ flushing */
@@ -3054,7 +3045,7 @@ int i915_vma_unbind(struct i915_vma *vma)
  	drm_mm_remove_node(&vma->node);
  	list_move_tail(&vma->vm_link, &vma->vm->unbound_list);
 
-	if (vma->pages != obj->pages) {
+	if (vma->pages != obj->mm.pages) {
  		GEM_BUG_ON(!vma->pages);
  		sg_free_table(vma->pages);
  		kfree(vma->pages);
@@ -3186,12 +3177,10 @@ i915_vma_insert(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
  		return -E2BIG;
  	}
 
-	ret = i915_gem_object_get_pages(obj);
+	ret = i915_gem_object_pin_pages(obj);
  	if (ret)
  		return ret;
 
-	i915_gem_object_pin_pages(obj);
-
  	if (flags & PIN_OFFSET_FIXED) {
  		u64 offset = flags & PIN_OFFSET_MASK;
  		if (offset & (alignment - 1) || offset > end - size) {
@@ -3270,7 +3259,7 @@ i915_gem_clflush_object(struct drm_i915_gem_object *obj,
  	 * to GPU, and we can ignore the cache flush because it'll happen
  	 * again at bind time.
  	 */
-	if (obj->pages == NULL)
+	if (!obj->mm.pages)
  		return false;
 
  	/*
@@ -3294,7 +3283,7 @@ i915_gem_clflush_object(struct drm_i915_gem_object *obj,
  	}
 
  	trace_i915_gem_object_clflush(obj);
-	drm_clflush_sg(obj->pages);
+	drm_clflush_sg(obj->mm.pages);
  	obj->cache_dirty = false;
 
  	return true;
@@ -3408,7 +3397,7 @@ i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
  	 * continue to assume that the obj remained out of the CPU cached
  	 * domain.
  	 */
-	ret = i915_gem_object_get_pages(obj);
+	ret = i915_gem_object_pin_pages(obj);
  	if (ret)
  		return ret;
 
@@ -3432,7 +3421,7 @@ i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
  	if (write) {
  		obj->base.read_domains = I915_GEM_DOMAIN_GTT;
  		obj->base.write_domain = I915_GEM_DOMAIN_GTT;
-		obj->dirty = 1;
+		obj->mm.dirty = true;
  	}
 
  	trace_i915_gem_object_change_domain(obj,
@@ -3441,6 +3430,7 @@ i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
 
  	/* And bump the LRU for this access */
  	i915_gem_object_bump_inactive_ggtt(obj);
+	i915_gem_object_unpin_pages(obj);
 
  	return 0;
  }
@@ -4210,23 +4200,23 @@ i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
  		goto unlock;
  	}
 
-	if (obj->pages &&
+	if (obj->mm.pages &&
  	    i915_gem_object_is_tiled(obj) &&
  	    dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES) {
-		if (obj->madv == I915_MADV_WILLNEED)
-			i915_gem_object_unpin_pages(obj);
+		if (obj->mm.madv == I915_MADV_WILLNEED)
+			__i915_gem_object_unpin_pages(obj);
  		if (args->madv == I915_MADV_WILLNEED)
-			i915_gem_object_pin_pages(obj);
+			__i915_gem_object_pin_pages(obj);
  	}
 
-	if (obj->madv != __I915_MADV_PURGED)
-		obj->madv = args->madv;
+	if (obj->mm.madv != __I915_MADV_PURGED)
+		obj->mm.madv = args->madv;
 
  	/* if the object is no longer attached, discard its backing storage */
-	if (obj->madv == I915_MADV_DONTNEED && obj->pages == NULL)
+	if (obj->mm.madv == I915_MADV_DONTNEED && !obj->mm.pages)
  		i915_gem_object_truncate(obj);
 
-	args->retained = obj->madv != __I915_MADV_PURGED;
+	args->retained = obj->mm.madv != __I915_MADV_PURGED;
 
  	i915_gem_object_put(obj);
  unlock:
@@ -4252,9 +4242,10 @@ void i915_gem_object_init(struct drm_i915_gem_object *obj,
  	obj->ops = ops;
 
  	obj->frontbuffer_ggtt_origin = ORIGIN_GTT;
-	obj->madv = I915_MADV_WILLNEED;
-	INIT_RADIX_TREE(&obj->get_page.radix, GFP_KERNEL);
-	mutex_init(&obj->get_page.lock);
+
+	obj->mm.madv = I915_MADV_WILLNEED;
+	INIT_RADIX_TREE(&obj->mm.get_page.radix, GFP_KERNEL);
+	mutex_init(&obj->mm.get_page.lock);
 
  	i915_gem_info_add_obj(to_i915(obj->base.dev), obj->base.size);
  }
@@ -4331,7 +4322,7 @@ static bool discard_backing_storage(struct drm_i915_gem_object *obj)
  	 * back the contents from the GPU.
  	 */
 
-	if (obj->madv != I915_MADV_WILLNEED)
+	if (obj->mm.madv != I915_MADV_WILLNEED)
  		return false;
 
  	if (obj->base.filp == NULL)
@@ -4373,32 +4364,27 @@ void i915_gem_free_object(struct drm_gem_object *gem_obj)
  	}
  	GEM_BUG_ON(obj->bind_count);
 
-	/* Stolen objects don't hold a ref, but do hold pin count. Fix that up
-	 * before progressing. */
-	if (obj->stolen)
-		i915_gem_object_unpin_pages(obj);
-
  	WARN_ON(atomic_read(&obj->frontbuffer_bits));
 
-	if (obj->pages && obj->madv == I915_MADV_WILLNEED &&
+	if (obj->mm.pages && obj->mm.madv == I915_MADV_WILLNEED &&
  	    dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES &&
  	    i915_gem_object_is_tiled(obj))
-		i915_gem_object_unpin_pages(obj);
+		__i915_gem_object_unpin_pages(obj);
 
-	if (WARN_ON(obj->pages_pin_count))
-		obj->pages_pin_count = 0;
+	if (obj->ops->release)
+		obj->ops->release(obj);
+
+	if (WARN_ON(i915_gem_object_has_pinned_pages(obj)))
+		obj->mm.pages_pin_count = 0;
  	if (discard_backing_storage(obj))
-		obj->madv = I915_MADV_DONTNEED;
-	i915_gem_object_put_pages(obj);
+		obj->mm.madv = I915_MADV_DONTNEED;
+	__i915_gem_object_put_pages(obj);
 
-	BUG_ON(obj->pages);
+	GEM_BUG_ON(obj->mm.pages);
 
  	if (obj->base.import_attach)
  		drm_prime_gem_destroy(&obj->base, NULL);
 
-	if (obj->ops->release)
-		obj->ops->release(obj);
-
  	drm_gem_object_release(&obj->base);
  	i915_gem_info_remove_obj(dev_priv, obj->base.size);
 
@@ -4935,14 +4921,13 @@ i915_gem_object_create_from_data(struct drm_device *dev,
  	if (ret)
  		goto fail;
 
-	ret = i915_gem_object_get_pages(obj);
+	ret = i915_gem_object_pin_pages(obj);
  	if (ret)
  		goto fail;
 
-	i915_gem_object_pin_pages(obj);
-	sg = obj->pages;
+	sg = obj->mm.pages;
  	bytes = sg_copy_from_buffer(sg->sgl, sg->nents, (void *)data, size);
-	obj->dirty = 1;		/* Backing store is now out of date */
+	obj->mm.dirty = true; /* Backing store is now out of date */
  	i915_gem_object_unpin_pages(obj);
 
  	if (WARN_ON(bytes != size)) {
@@ -4962,7 +4947,7 @@ static struct scatterlist *
  __i915_gem_object_get_sg(struct drm_i915_gem_object *obj,
  			 unsigned long n, unsigned int *offset)
  {
-	struct scatterlist *sg = obj->pages->sgl;
+	struct scatterlist *sg = obj->mm.pages->sgl;
  	int idx = 0;
 
  	while (idx + __sg_page_count(sg) <= n) {
@@ -4980,11 +4965,11 @@ i915_gem_object_get_sg(struct drm_i915_gem_object *obj,
  		       unsigned long n,
  		       unsigned int *offset)
  {
-	struct i915_gem_object_page_iter *iter = &obj->get_page;
+	struct i915_gem_object_page_iter *iter = &obj->mm.get_page;
  	struct scatterlist *sg;
 
  	GEM_BUG_ON(n >= obj->base.size >> PAGE_SHIFT);
-	GEM_BUG_ON(obj->pages_pin_count == 0);
+	GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));
 
  	if (n < READ_ONCE(iter->sg_idx))
  		goto lookup;
@@ -5058,7 +5043,7 @@ i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj,
  	struct page *page;
 
  	page = i915_gem_object_get_page(obj, n);
-	if (!obj->dirty)
+	if (!obj->mm.dirty)
  		set_page_dirty(page);
 
  	return page;
drivers/gpu/drm/i915/i915_gem_batch_pool.c +1 -2
index 3934c9103cf2..6b656822bb3a 100644
@@ -131,11 +131,10 @@ i915_gem_batch_pool_get(struct i915_gem_batch_pool *pool,
  			return obj;
  	}
 
-	ret = i915_gem_object_get_pages(obj);
+	ret = i915_gem_object_pin_pages(obj);
  	if (ret)
  		return ERR_PTR(ret);
 
  	list_move_tail(&obj->batch_pool_link, list);
-	i915_gem_object_pin_pages(obj);
  	return obj;
  }
drivers/gpu/drm/i915/i915_gem_dmabuf.c +6 -8
index 97c9d68b45df..10441dc72e73 100644
@@ -48,12 +48,10 @@ static struct sg_table *i915_gem_map_dma_buf(struct dma_buf_attachment *attachme
  	if (ret)
  		goto err;
 
-	ret = i915_gem_object_get_pages(obj);
+	ret = i915_gem_object_pin_pages(obj);
  	if (ret)
  		goto err_unlock;
 
-	i915_gem_object_pin_pages(obj);
-
  	/* Copy sg so that we make an independent mapping */
  	st = kmalloc(sizeof(struct sg_table), GFP_KERNEL);
  	if (st == NULL) {
@@ -61,13 +59,13 @@ static struct sg_table *i915_gem_map_dma_buf(struct dma_buf_attachment *attachme
  		goto err_unpin;
  	}
 
-	ret = sg_alloc_table(st, obj->pages->nents, GFP_KERNEL);
+	ret = sg_alloc_table(st, obj->mm.pages->nents, GFP_KERNEL);
  	if (ret)
  		goto err_free;
 
-	src = obj->pages->sgl;
+	src = obj->mm.pages->sgl;
  	dst = st->sgl;
-	for (i = 0; i < obj->pages->nents; i++) {
+	for (i = 0; i < obj->mm.pages->nents; i++) {
  		sg_set_page(dst, sg_page(src), src->length, 0);
  		dst = sg_next(dst);
  		src = sg_next(src);
@@ -299,14 +297,14 @@ static int i915_gem_object_get_pages_dmabuf(struct drm_i915_gem_object *obj)
  	if (IS_ERR(sg))
  		return PTR_ERR(sg);
 
-	obj->pages = sg;
+	obj->mm.pages = sg;
  	return 0;
  }
 
  static void i915_gem_object_put_pages_dmabuf(struct drm_i915_gem_object *obj)
  {
  	dma_buf_unmap_attachment(obj->base.import_attach,
-				 obj->pages, DMA_BIDIRECTIONAL);
+				 obj->mm.pages, DMA_BIDIRECTIONAL);
  }
 
  static const struct drm_i915_gem_object_ops i915_gem_object_dmabuf_ops = {
drivers/gpu/drm/i915/i915_gem_execbuffer.c +1 -1
index 0deecd4e3b6c..062e098b0909 100644
@@ -1290,7 +1290,7 @@ void i915_vma_move_to_active(struct i915_vma *vma,
 
  	GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
 
-	obj->dirty = 1; /* be paranoid  */
+	obj->mm.dirty = true; /* be paranoid  */
 
  	/* Add a reference if we're newly entering the active list.
  	 * The order in which we add operations to the retirement queue is
drivers/gpu/drm/i915/i915_gem_fence.c +2 -2
index 8df1fa7234e8..398856160656 100644
@@ -648,7 +648,7 @@ i915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object *obj)
  		return;
 
  	i = 0;
-	for_each_sgt_page(page, sgt_iter, obj->pages) {
+	for_each_sgt_page(page, sgt_iter, obj->mm.pages) {
  		char new_bit_17 = page_to_phys(page) >> 17;
  		if ((new_bit_17 & 0x1) !=
  		    (test_bit(i, obj->bit_17) != 0)) {
@@ -687,7 +687,7 @@ i915_gem_object_save_bit_17_swizzle(struct drm_i915_gem_object *obj)
 
  	i = 0;
 
-	for_each_sgt_page(page, sgt_iter, obj->pages) {
+	for_each_sgt_page(page, sgt_iter, obj->mm.pages) {
  		if (page_to_phys(page) & (1 << 17))
  			__set_bit(i, obj->bit_17);
  		else
drivers/gpu/drm/i915/i915_gem_gtt.c +5 -5
index 7e0c98576e72..3c711e0b8a3f 100644
@@ -175,7 +175,7 @@ static int ppgtt_bind_vma(struct i915_vma *vma,
  {
  	u32 pte_flags = 0;
 
-	vma->pages = vma->obj->pages;
+	vma->pages = vma->obj->mm.pages;
 
  	/* Currently applicable only to VLV */
  	if (vma->obj->gt_ro)
@@ -2295,7 +2295,7 @@ void i915_gem_suspend_gtt_mappings(struct drm_device *dev)
  int i915_gem_gtt_prepare_object(struct drm_i915_gem_object *obj)
  {
  	if (!dma_map_sg(&obj->base.dev->pdev->dev,
-			obj->pages->sgl, obj->pages->nents,
+			obj->mm.pages->sgl, obj->mm.pages->nents,
  			PCI_DMA_BIDIRECTIONAL))
  		return -ENOSPC;
 
@@ -2685,7 +2685,7 @@ void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj)
  		}
  	}
 
-	dma_unmap_sg(kdev, obj->pages->sgl, obj->pages->nents,
+	dma_unmap_sg(kdev, obj->mm.pages->sgl, obj->mm.pages->nents,
  		     PCI_DMA_BIDIRECTIONAL);
  }
 
@@ -3526,7 +3526,7 @@ intel_rotate_fb_obj_pages(const struct intel_rotation_info *rot_info,
 
  	/* Populate source page list from the object. */
  	i = 0;
-	for_each_sgt_dma(dma_addr, sgt_iter, obj->pages)
+	for_each_sgt_dma(dma_addr, sgt_iter, obj->mm.pages)
  		page_addr_list[i++] = dma_addr;
 
  	GEM_BUG_ON(i != n_pages);
@@ -3617,7 +3617,7 @@ i915_get_ggtt_vma_pages(struct i915_vma *vma)
  		return 0;
 
  	if (vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL)
-		vma->pages = vma->obj->pages;
+		vma->pages = vma->obj->mm.pages;
  	else if (vma->ggtt_view.type == I915_GGTT_VIEW_ROTATED)
  		vma->pages =
  			intel_rotate_fb_obj_pages(&vma->ggtt_view.params.rotated, vma->obj);
drivers/gpu/drm/i915/i915_gem_internal.c +10 -9
index 534a61c1aba2..24eb347f9e0a 100644
@@ -28,10 +28,11 @@
 
  static void internal_free_pages(struct sg_table *st)
  {
-	struct sg_page_iter sg_iter;
+	struct sgt_iter iter;
+	struct page *page;
 
-	for_each_sg_page(st->sgl, &sg_iter, st->nents, 0)
-		put_page(sg_page_iter_page(&sg_iter));
+	for_each_sgt_page(page, iter, st)
+		put_page(page);
 
  	sg_free_table(st);
  	kfree(st);
@@ -94,10 +95,10 @@ static int i915_gem_object_get_pages_internal(struct drm_i915_gem_object *obj)
  	if (!swiotlb_nr_tbl())
  #endif
  		sg_mark_end(sg);
-	obj->pages = st;
+	obj->mm.pages = st;
 
  	if (i915_gem_gtt_prepare_object(obj)) {
-		obj->pages = NULL;
+		obj->mm.pages = NULL;
  		goto err;
  	}
 
@@ -106,7 +107,7 @@ static int i915_gem_object_get_pages_internal(struct drm_i915_gem_object *obj)
  	 * and the caller is expected to repopulate - the contents of this
  	 * object are only valid whilst active and pinned.
  	 */
-	obj->madv = I915_MADV_DONTNEED;
+	obj->mm.madv = I915_MADV_DONTNEED;
  	return 0;
 
  err:
@@ -117,10 +118,10 @@ err:
 
  static void i915_gem_object_put_pages_internal(struct drm_i915_gem_object *obj)
  {
-	internal_free_pages(obj->pages);
+	internal_free_pages(obj->mm.pages);
 
-	obj->dirty = 0;
-	obj->madv = I915_MADV_WILLNEED;
+	obj->mm.dirty = false;
+	obj->mm.madv = I915_MADV_WILLNEED;
  }
 
  static const struct drm_i915_gem_object_ops i915_gem_object_internal_ops = {
drivers/gpu/drm/i915/i915_gem_render_state.c +1 -1
index adb00c5125ad..64d9712ec789 100644
@@ -230,7 +230,7 @@ int i915_gem_render_state_emit(struct drm_i915_gem_request *req)
  		return 0;
 
  	/* Recreate the page after shrinking */
-	if (!so->vma->obj->pages)
+	if (!so->vma->obj->mm.pages)
  		so->batch_offset = -1;
 
  	ret = i915_vma_pin(so->vma, 0, 0, PIN_GLOBAL | PIN_HIGH);
drivers/gpu/drm/i915/i915_gem_shrinker.c +5 -5
index 1c237d02f30b..e0e6d68fe470 100644
@@ -78,7 +78,7 @@ static bool can_release_pages(struct drm_i915_gem_object *obj)
  	 * to the GPU, simply unbinding from the GPU is not going to succeed
  	 * in releasing our pin count on the pages themselves.
  	 */
-	if (obj->pages_pin_count > obj->bind_count)
+	if (obj->mm.pages_pin_count > obj->bind_count)
  		return false;
 
  	if (any_vma_pinned(obj))
@@ -88,7 +88,7 @@ static bool can_release_pages(struct drm_i915_gem_object *obj)
  	 * discard the contents (because the user has marked them as being
  	 * purgeable) or if we can move their contents out to swap.
  	 */
-	return swap_available() || obj->madv == I915_MADV_DONTNEED;
+	return swap_available() || obj->mm.madv == I915_MADV_DONTNEED;
  }
 
  /**
@@ -175,11 +175,11 @@ i915_gem_shrink(struct drm_i915_private *dev_priv,
  			list_move_tail(&obj->global_list, &still_in_list);
 
  			if (flags & I915_SHRINK_PURGEABLE &&
-			    obj->madv != I915_MADV_DONTNEED)
+			    obj->mm.madv != I915_MADV_DONTNEED)
  				continue;
 
  			if (flags & I915_SHRINK_VMAPS &&
-			    !is_vmalloc_addr(obj->mapping))
+			    !is_vmalloc_addr(obj->mm.mapping))
  				continue;
 
  			if ((flags & I915_SHRINK_ACTIVE) == 0 &&
@@ -193,7 +193,7 @@ i915_gem_shrink(struct drm_i915_private *dev_priv,
 
  			/* For the unbound phase, this should be a no-op! */
  			i915_gem_object_unbind(obj);
-			if (i915_gem_object_put_pages(obj) == 0)
+			if (__i915_gem_object_put_pages(obj) == 0)
  				count += obj->base.size >> PAGE_SHIFT;
 
  			i915_gem_object_put(obj);
drivers/gpu/drm/i915/i915_gem_stolen.c +13 -11
index 24bad4e60ef0..6d139d6b4609 100644
@@ -554,16 +554,17 @@ static int i915_gem_object_get_pages_stolen(struct drm_i915_gem_object *obj)
  static void i915_gem_object_put_pages_stolen(struct drm_i915_gem_object *obj)
  {
  	/* Should only be called during free */
-	sg_free_table(obj->pages);
-	kfree(obj->pages);
+	sg_free_table(obj->mm.pages);
+	kfree(obj->mm.pages);
  }
 
-
  static void
  i915_gem_object_release_stolen(struct drm_i915_gem_object *obj)
  {
  	struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
 
+	__i915_gem_object_unpin_pages(obj);
+
  	if (obj->stolen) {
  		i915_gem_stolen_remove_node(dev_priv, obj->stolen);
  		kfree(obj->stolen);
@@ -589,15 +590,16 @@ _i915_gem_object_create_stolen(struct drm_device *dev,
  	drm_gem_private_object_init(dev, &obj->base, stolen->size);
  	i915_gem_object_init(obj, &i915_gem_object_stolen_ops);
 
-	obj->pages = i915_pages_create_for_stolen(dev,
-						  stolen->start, stolen->size);
-	if (obj->pages == NULL)
+	obj->mm.pages = i915_pages_create_for_stolen(dev,
+						     stolen->start,
+						     stolen->size);
+	if (!obj->mm.pages)
  		goto cleanup;
 
-	obj->get_page.sg_pos = obj->pages->sgl;
-	obj->get_page.sg_idx = 0;
+	obj->mm.get_page.sg_pos = obj->mm.pages->sgl;
+	obj->mm.get_page.sg_idx = 0;
 
-	i915_gem_object_pin_pages(obj);
+	__i915_gem_object_pin_pages(obj);
  	obj->stolen = stolen;
 
  	obj->base.read_domains = I915_GEM_DOMAIN_CPU | I915_GEM_DOMAIN_GTT;
@@ -717,14 +719,14 @@ i915_gem_object_create_stolen_for_preallocated(struct drm_device *dev,
  		goto err;
  	}
 
-	vma->pages = obj->pages;
+	vma->pages = obj->mm.pages;
  	vma->flags |= I915_VMA_GLOBAL_BIND;
  	__i915_vma_set_map_and_fenceable(vma);
  	list_move_tail(&vma->vm_link, &ggtt->base.inactive_list);
  	obj->bind_count++;
 
  	list_add_tail(&obj->global_list, &dev_priv->mm.bound_list);
-	i915_gem_object_pin_pages(obj);
+	__i915_gem_object_pin_pages(obj);
 
  	return obj;
 
drivers/gpu/drm/i915/i915_gem_tiling.c +4 -4
index a14b1e3d4c78..7286de7bd25e 100644
@@ -260,13 +260,13 @@ i915_gem_set_tiling(struct drm_device *dev, void *data,
  		if (!err) {
  			struct i915_vma *vma;
 
-			if (obj->pages &&
-			    obj->madv == I915_MADV_WILLNEED &&
+			if (obj->mm.pages &&
+			    obj->mm.madv == I915_MADV_WILLNEED &&
  			    dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES) {
  				if (args->tiling_mode == I915_TILING_NONE)
-					i915_gem_object_unpin_pages(obj);
+					__i915_gem_object_unpin_pages(obj);
  				if (!i915_gem_object_is_tiled(obj))
-					i915_gem_object_pin_pages(obj);
+					__i915_gem_object_pin_pages(obj);
  			}
 
  			list_for_each_entry(vma, &obj->vma_list, obj_link) {
drivers/gpu/drm/i915/i915_gem_userptr.c +15 -15
index cb95789da76e..0fdd1d6723d1 100644
@@ -73,10 +73,10 @@ static void cancel_userptr(struct work_struct *work)
  	/* Cancel any active worker and force us to re-evaluate gup */
  	obj->userptr.work = NULL;
 
-	if (obj->pages != NULL) {
+	if (obj->mm.pages) {
  		/* We are inside a kthread context and can't be interrupted */
  		WARN_ON(i915_gem_object_unbind(obj));
-		WARN_ON(i915_gem_object_put_pages(obj));
+		WARN_ON(__i915_gem_object_put_pages(obj));
  	}
 
  	i915_gem_object_put(obj);
@@ -432,15 +432,15 @@ __i915_gem_userptr_set_pages(struct drm_i915_gem_object *obj,
  {
  	int ret;
 
-	ret = st_set_pages(&obj->pages, pvec, num_pages);
+	ret = st_set_pages(&obj->mm.pages, pvec, num_pages);
  	if (ret)
  		return ret;
 
  	ret = i915_gem_gtt_prepare_object(obj);
  	if (ret) {
-		sg_free_table(obj->pages);
-		kfree(obj->pages);
-		obj->pages = NULL;
+		sg_free_table(obj->mm.pages);
+		kfree(obj->mm.pages);
+		obj->mm.pages = NULL;
  	}
 
  	return ret;
@@ -526,8 +526,8 @@ __i915_gem_userptr_get_pages_worker(struct work_struct *_work)
  			if (ret == 0) {
  				list_add_tail(&obj->global_list,
  					      &to_i915(dev)->mm.unbound_list);
-				obj->get_page.sg_pos = obj->pages->sgl;
-				obj->get_page.sg_idx = 0;
+				obj->mm.get_page.sg_pos = obj->mm.pages->sgl;
+				obj->mm.get_page.sg_idx = 0;
  				pinned = 0;
  			}
  		}
@@ -668,22 +668,22 @@ i915_gem_userptr_put_pages(struct drm_i915_gem_object *obj)
  	BUG_ON(obj->userptr.work != NULL);
  	__i915_gem_userptr_set_active(obj, false);
 
-	if (obj->madv != I915_MADV_WILLNEED)
-		obj->dirty = 0;
+	if (obj->mm.madv != I915_MADV_WILLNEED)
+		obj->mm.dirty = false;
 
  	i915_gem_gtt_finish_object(obj);
 
-	for_each_sgt_page(page, sgt_iter, obj->pages) {
-		if (obj->dirty)
+	for_each_sgt_page(page, sgt_iter, obj->mm.pages) {
+		if (obj->mm.dirty)
  			set_page_dirty(page);
 
  		mark_page_accessed(page);
  		put_page(page);
  	}
-	obj->dirty = 0;
+	obj->mm.dirty = false;
 
-	sg_free_table(obj->pages);
-	kfree(obj->pages);
+	sg_free_table(obj->mm.pages);
+	kfree(obj->mm.pages);
  }
 
  static void
drivers/gpu/drm/i915/i915_gpu_error.c +2 -2
index 6c22be28ba01..628d5cdf9200 100644
@@ -881,8 +881,8 @@ static void capture_bo(struct drm_i915_error_buffer *err,
  	err->write_domain = obj->base.write_domain;
  	err->fence_reg = vma->fence ? vma->fence->id : -1;
  	err->tiling = i915_gem_object_get_tiling(obj);
-	err->dirty = obj->dirty;
-	err->purgeable = obj->madv != I915_MADV_WILLNEED;
+	err->dirty = obj->mm.dirty;
+	err->purgeable = obj->mm.madv != I915_MADV_WILLNEED;
  	err->userptr = obj->userptr.mm != NULL;
  	err->cache_level = obj->cache_level;
  }
drivers/gpu/drm/i915/intel_lrc.c +2 -2
index 02c48d390148..6acecfc41f5e 100644
@@ -744,7 +744,7 @@ static int intel_lr_context_pin(struct i915_gem_context *ctx,
  	ce->lrc_reg_state[CTX_RING_BUFFER_START+1] =
  		i915_ggtt_offset(ce->ring->vma);
 
-	ce->state->obj->dirty = true;
+	ce->state->obj->mm.dirty = true;
 
  	/* Invalidate GuC TLB. */
  	if (i915.enable_guc_submission) {
@@ -2042,7 +2042,7 @@ populate_lr_context(struct i915_gem_context *ctx,
  		DRM_DEBUG_DRIVER("Could not map object pages! (%d)\n", ret);
  		return ret;
  	}
-	ctx_obj->dirty = true;
+	ctx_obj->mm.dirty = true;
 
  	/* The second page of the context object contains some fields which must
  	 * be set up prior to the first execution. */

drivers/gpu/drm/i915/intel_lrc.c void intel_lr_context_resume(struct drm_i915_private *dev_priv) · line 2185

Quoted 8 lines
drivers/gpu/drm/i915/intel_lrc.c +1 -1
@@ -2179,7 +2179,7 @@ void intel_lr_context_resume(struct drm_i915_private *dev_priv)
  			reg[CTX_RING_HEAD+1] = 0;
  			reg[CTX_RING_TAIL+1] = 0;
 
-			ce->state->obj->dirty = true;
+			ce->state->obj->mm.dirty = true;
  			i915_gem_object_unpin_map(ce->state->obj);
 
  			ce->ring->head = ce->ring->tail = 0;

_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx

Reviewed-by on v1 16/42
#50 lore raw

Re: [PATCH 16/42] drm/i915: Refactor object page API

In reply to Chris Wilson
Quoted 1,338 lines · Chris Wilson

The plan is to make obtaining the backing storage for the object avoid struct_mutex (i.e. use its own locking). The first step is to update the API so that normal users only call pin/unpin whilst working on the backing storage.

---
  drivers/gpu/drm/i915/i915_cmd_parser.c       |   2 +-
  drivers/gpu/drm/i915/i915_debugfs.c          |  17 +--
  drivers/gpu/drm/i915/i915_drv.h              |  89 ++++++++----
  drivers/gpu/drm/i915/i915_gem.c              | 207 +++++++++++++--------------
  drivers/gpu/drm/i915/i915_gem_batch_pool.c   |   3 +-
  drivers/gpu/drm/i915/i915_gem_dmabuf.c       |  14 +-
  drivers/gpu/drm/i915/i915_gem_execbuffer.c   |   2 +-
  drivers/gpu/drm/i915/i915_gem_fence.c        |   4 +-
  drivers/gpu/drm/i915/i915_gem_gtt.c          |  10 +-
  drivers/gpu/drm/i915/i915_gem_internal.c     |  19 +--
  drivers/gpu/drm/i915/i915_gem_render_state.c |   2 +-
  drivers/gpu/drm/i915/i915_gem_shrinker.c     |  10 +-
  drivers/gpu/drm/i915/i915_gem_stolen.c       |  24 ++--
  drivers/gpu/drm/i915/i915_gem_tiling.c       |   8 +-
  drivers/gpu/drm/i915/i915_gem_userptr.c      |  30 ++--
  drivers/gpu/drm/i915/i915_gpu_error.c        |   4 +-
  drivers/gpu/drm/i915/intel_lrc.c             |   6 +-
  17 files changed, 234 insertions(+), 217 deletions(-)
drivers/gpu/drm/i915/i915_cmd_parser.c +1 -1
index 70980f82a15b..8d20020cb9f9 100644
@@ -1290,7 +1290,7 @@ int intel_engine_cmd_parser(struct intel_engine_cs *engine,
  	}
 
  	if (ret == 0 && needs_clflush_after)
-		drm_clflush_virt_range(shadow_batch_obj->mapping, batch_len);
+		drm_clflush_virt_range(shadow_batch_obj->mm.mapping, batch_len);
  	i915_gem_object_unpin_map(shadow_batch_obj);
 
  	return ret;
drivers/gpu/drm/i915/i915_debugfs.c +9 -8
index e4b5ba771bea..b807ddf65e04 100644
@@ -112,7 +112,7 @@ static char get_global_flag(struct drm_i915_gem_object *obj)
 
  static char get_pin_mapped_flag(struct drm_i915_gem_object *obj)
  {
-	return obj->mapping ? 'M' : ' ';
+	return obj->mm.mapping ? 'M' : ' ';
  }
 
  static u64 i915_gem_obj_total_ggtt_size(struct drm_i915_gem_object *obj)
@@ -158,8 +158,8 @@ describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj)
  		   i915_gem_active_get_seqno(&obj->last_write,
  					     &obj->base.dev->struct_mutex),
  		   i915_cache_level_str(dev_priv, obj->cache_level),
-		   obj->dirty ? " dirty" : "",
-		   obj->madv == I915_MADV_DONTNEED ? " purgeable" : "");
+		   obj->mm.dirty ? " dirty" : "",
+		   obj->mm.madv == I915_MADV_DONTNEED ? " purgeable" : "");
  	if (obj->base.name)
  		seq_printf(m, " (name: %d)", obj->base.name);
  	list_for_each_entry(vma, &obj->vma_list, obj_link) {
@@ -411,12 +411,12 @@ static int i915_gem_object_info(struct seq_file *m, void *data)
  		size += obj->base.size;
  		++count;
 
-		if (obj->madv == I915_MADV_DONTNEED) {
+		if (obj->mm.madv == I915_MADV_DONTNEED) {
  			purgeable_size += obj->base.size;
  			++purgeable_count;
  		}
 
-		if (obj->mapping) {
+		if (obj->mm.mapping) {
  			mapped_count++;
  			mapped_size += obj->base.size;
  		}
@@ -433,12 +433,12 @@ static int i915_gem_object_info(struct seq_file *m, void *data)
  			++dpy_count;
  		}
 
-		if (obj->madv == I915_MADV_DONTNEED) {
+		if (obj->mm.madv == I915_MADV_DONTNEED) {
  			purgeable_size += obj->base.size;
  			++purgeable_count;
  		}
 
-		if (obj->mapping) {
+		if (obj->mm.mapping) {
  			mapped_count++;
  			mapped_size += obj->base.size;
  		}
@@ -2018,7 +2018,7 @@ static void i915_dump_lrc_obj(struct seq_file *m,
  		seq_printf(m, "\tBound in GGTT at 0x%08x\n",
  			   i915_ggtt_offset(vma));
 
-	if (i915_gem_object_get_pages(vma->obj)) {
+	if (i915_gem_object_pin_pages(vma->obj)) {
  		seq_puts(m, "\tFailed to get pages for context object\n\n");
  		return;
  	}
@@ -2037,6 +2037,7 @@ static void i915_dump_lrc_obj(struct seq_file *m,
  		kunmap_atomic(reg_state);
  	}
 
+	i915_gem_object_unpin_pages(vma->obj);
  	seq_putc(m, '\n');
  }
 
drivers/gpu/drm/i915/i915_drv.h +60 -29
index a96b446d8db4..3c22d49005fe 100644
@@ -2238,17 +2238,6 @@ struct drm_i915_gem_object {
  #define I915_BO_ACTIVE_REF (I915_BO_ACTIVE_SHIFT + I915_NUM_ENGINES)
 
  	/**
-	 * This is set if the object has been written to since last bound
-	 * to the GTT
-	 */
-	unsigned int dirty:1;
-
-	/**
-	 * Advice: are the backing pages purgeable?
-	 */
-	unsigned int madv:2;
-
-	/**
  	 * Whether the current gtt mapping needs to be mappable (and isn't just
  	 * mappable by accident). Track pin and fault separate for a more
  	 * accurate mappable working set.
@@ -2276,16 +2265,31 @@ struct drm_i915_gem_object {
  	unsigned int bind_count;
  	unsigned int pin_display;
 
-	struct sg_table *pages;
-	int pages_pin_count;
-	struct i915_gem_object_page_iter {
-		struct scatterlist *sg_pos;
-		unsigned long sg_idx;
+	struct {
+		unsigned int pages_pin_count;
+
+		struct sg_table *pages;
+		void *mapping;
+
+		struct i915_gem_object_page_iter {
+			struct scatterlist *sg_pos;
+			unsigned long sg_idx;
 
-		struct radix_tree_root radix;
-		struct mutex lock;
-	} get_page;
-	void *mapping;
+			struct radix_tree_root radix;
+			struct mutex lock;
+		} get_page;
+
+		/**
+		 * Advice: are the backing pages purgeable?
+		 */
+		unsigned int madv:2;
+
+		/**
+		 * This is set if the object has been written to since the
+		 * pages were last acquired.
+		 */
+		unsigned int dirty:1;
+	} mm;
 
  	/** Breadcrumb of last rendering to the buffer.
  	 * There can only be one writer, but we allow for multiple readers.
@@ -3160,13 +3164,10 @@ void i915_vma_close(struct i915_vma *vma);
  void i915_vma_destroy(struct i915_vma *vma);
 
  int i915_gem_object_unbind(struct drm_i915_gem_object *obj);
-int i915_gem_object_put_pages(struct drm_i915_gem_object *obj);
  void i915_gem_release_all_mmaps(struct drm_i915_private *dev_priv);
  void i915_gem_release_mmap(struct drm_i915_gem_object *obj);
 
-int __must_check i915_gem_object_get_pages(struct drm_i915_gem_object *obj);
-
-static inline int __sg_page_count(struct scatterlist *sg)
+static inline int __sg_page_count(const struct scatterlist *sg)
  {
  	return sg->length >> PAGE_SHIFT;
  }
@@ -3187,18 +3188,48 @@ dma_addr_t
  i915_gem_object_get_dma_address(struct drm_i915_gem_object *obj,
  				unsigned long n);
 
-static inline void i915_gem_object_pin_pages(struct drm_i915_gem_object *obj)
+int __i915_gem_object_get_pages(struct drm_i915_gem_object *obj);
+
+static inline int __must_check
+i915_gem_object_pin_pages(struct drm_i915_gem_object *obj)
  {
-	BUG_ON(obj->pages == NULL);
-	obj->pages_pin_count++;
+	lockdep_assert_held(&obj->base.dev->struct_mutex);
+	if (obj->mm.pages_pin_count++)
+		return 0;
+
+	return __i915_gem_object_get_pages(obj);
+}
+
+static inline void
+__i915_gem_object_pin_pages(struct drm_i915_gem_object *obj)
+{
+	lockdep_assert_held(&obj->base.dev->struct_mutex);
+	GEM_BUG_ON(!obj->mm.pages);
+	obj->mm.pages_pin_count++;
+}
+
+static inline bool
+i915_gem_object_has_pinned_pages(struct drm_i915_gem_object *obj)
+{
+	return obj->mm.pages_pin_count;
+}
+
+static inline void
+__i915_gem_object_unpin_pages(struct drm_i915_gem_object *obj)
+{
+	lockdep_assert_held(&obj->base.dev->struct_mutex);
+	GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));
+	GEM_BUG_ON(!obj->mm.pages);
+	obj->mm.pages_pin_count--;
  }
 
  static inline void i915_gem_object_unpin_pages(struct drm_i915_gem_object *obj)
  {
-	BUG_ON(obj->pages_pin_count == 0);
-	obj->pages_pin_count--;
+	__i915_gem_object_unpin_pages(obj);
  }
 
+int __i915_gem_object_put_pages(struct drm_i915_gem_object *obj);
+
  enum i915_map_type {
  	I915_MAP_WB = 0,
  	I915_MAP_WC,
drivers/gpu/drm/i915/i915_gem.c +96 -111
index af7d51f16658..df774ddf62ae 100644
@@ -216,7 +216,7 @@ i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj)
  	sg_dma_address(sg) = obj->phys_handle->busaddr;
  	sg_dma_len(sg) = obj->base.size;
 
-	obj->pages = st;
+	obj->mm.pages = st;
  	return 0;
  }
 
@@ -225,7 +225,7 @@ i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj)
  {
  	int ret;
 
-	BUG_ON(obj->madv == __I915_MADV_PURGED);
+	GEM_BUG_ON(obj->mm.madv == __I915_MADV_PURGED);
 
  	ret = i915_gem_object_set_to_cpu_domain(obj, true);
  	if (WARN_ON(ret)) {
@@ -235,10 +235,10 @@ i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj)
  		obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
  	}
 
-	if (obj->madv == I915_MADV_DONTNEED)
-		obj->dirty = 0;
+	if (obj->mm.madv == I915_MADV_DONTNEED)
+		obj->mm.dirty = false;
 
-	if (obj->dirty) {
+	if (obj->mm.dirty) {
  		struct address_space *mapping = obj->base.filp->f_mapping;
  		char *vaddr = obj->phys_handle->vaddr;
  		int i;
@@ -257,22 +257,23 @@ i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj)
  			kunmap_atomic(dst);
 
  			set_page_dirty(page);
-			if (obj->madv == I915_MADV_WILLNEED)
+			if (obj->mm.madv == I915_MADV_WILLNEED)
  				mark_page_accessed(page);
  			put_page(page);
  			vaddr += PAGE_SIZE;
  		}
-		obj->dirty = 0;
+		obj->mm.dirty = false;
  	}
 
-	sg_free_table(obj->pages);
-	kfree(obj->pages);
+	sg_free_table(obj->mm.pages);
+	kfree(obj->mm.pages);
  }
 
  static void
  i915_gem_object_release_phys(struct drm_i915_gem_object *obj)
  {
  	drm_pci_free(obj->base.dev, obj->phys_handle);
+	i915_gem_object_unpin_pages(obj);
  }
 
  static const struct drm_i915_gem_object_ops i915_gem_phys_ops = {
@@ -506,7 +507,7 @@ i915_gem_object_attach_phys(struct drm_i915_gem_object *obj,
  		return 0;
  	}
 
-	if (obj->madv != I915_MADV_WILLNEED)
+	if (obj->mm.madv != I915_MADV_WILLNEED)
  		return -EFAULT;
 
  	if (obj->base.filp == NULL)
@@ -516,7 +517,7 @@ i915_gem_object_attach_phys(struct drm_i915_gem_object *obj,
  	if (ret)
  		return ret;
 
-	ret = i915_gem_object_put_pages(obj);
+	ret = __i915_gem_object_put_pages(obj);
  	if (ret)
  		return ret;
 
@@ -528,7 +529,7 @@ i915_gem_object_attach_phys(struct drm_i915_gem_object *obj,
  	obj->phys_handle = phys;
  	obj->ops = &i915_gem_phys_ops;
 
-	return i915_gem_object_get_pages(obj);
+	return i915_gem_object_pin_pages(obj);
  }
 
  static int
@@ -724,12 +725,10 @@ int i915_gem_obj_prepare_shmem_read(struct drm_i915_gem_object *obj,
  	if (ret)
  		return ret;
 
-	ret = i915_gem_object_get_pages(obj);
+	ret = i915_gem_object_pin_pages(obj);
  	if (ret)
  		return ret;
 
-	i915_gem_object_pin_pages(obj);
-
  	i915_gem_object_flush_gtt_write_domain(obj);
 
  	/* If we're not in the cpu read domain, set ourself into the gtt
@@ -777,12 +776,10 @@ int i915_gem_obj_prepare_shmem_write(struct drm_i915_gem_object *obj,
  	if (ret)
  		return ret;
 
-	ret = i915_gem_object_get_pages(obj);
+	ret = i915_gem_object_pin_pages(obj);
  	if (ret)
  		return ret;
 
-	i915_gem_object_pin_pages(obj);
-
  	i915_gem_object_flush_gtt_write_domain(obj);
 
  	/* If we're not in the cpu write domain, set ourself into the
@@ -812,7 +809,7 @@ int i915_gem_obj_prepare_shmem_write(struct drm_i915_gem_object *obj,
  		obj->cache_dirty = true;
 
  	intel_fb_obj_invalidate(obj, ORIGIN_CPU);
-	obj->dirty = 1;
+	obj->mm.dirty = true;
  	/* return with the pages pinned */
  	return 0;
 
@@ -949,13 +946,11 @@ i915_gem_gtt_pread(struct drm_device *dev,
  		if (ret)
  			goto out;
 
-		ret = i915_gem_object_get_pages(obj);
+		ret = i915_gem_object_pin_pages(obj);
  		if (ret) {
  			remove_mappable_node(&node);
  			goto out;
  		}
-
-		i915_gem_object_pin_pages(obj);
  	}
 
  	ret = i915_gem_object_set_to_gtt_domain(obj, false);
@@ -1062,7 +1057,7 @@ i915_gem_shmem_pread(struct drm_device *dev,
  	offset = args->offset;
  	remain = args->size;
 
-	for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
+	for_each_sg_page(obj->mm.pages->sgl, &sg_iter, obj->mm.pages->nents,
  			 offset >> PAGE_SHIFT) {
  		struct page *page = sg_page_iter_page(&sg_iter);
 
@@ -1254,13 +1249,11 @@ i915_gem_gtt_pwrite_fast(struct drm_i915_private *i915,
  		if (ret)
  			goto out;
 
-		ret = i915_gem_object_get_pages(obj);
+		ret = i915_gem_object_pin_pages(obj);
  		if (ret) {
  			remove_mappable_node(&node);
  			goto out;
  		}
-
-		i915_gem_object_pin_pages(obj);
  	}
 
  	ret = i915_gem_object_set_to_gtt_domain(obj, true);
@@ -1268,7 +1261,7 @@ i915_gem_gtt_pwrite_fast(struct drm_i915_private *i915,
  		goto out_unpin;
 
  	intel_fb_obj_invalidate(obj, ORIGIN_CPU);
-	obj->dirty = true;
+	obj->mm.dirty = true;
 
  	user_data = u64_to_user_ptr(args->data_ptr);
  	offset = args->offset;
@@ -1439,7 +1432,7 @@ i915_gem_shmem_pwrite(struct drm_device *dev,
  	offset = args->offset;
  	remain = args->size;
 
-	for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
+	for_each_sg_page(obj->mm.pages->sgl, &sg_iter, obj->mm.pages->nents,
  			 offset >> PAGE_SHIFT) {
  		struct page *page = sg_page_iter_page(&sg_iter);
  		int partial_cacheline_write;
@@ -2228,7 +2221,7 @@ i915_gem_object_truncate(struct drm_i915_gem_object *obj)
  	 * backing pages, *now*.
  	 */
  	shmem_truncate_range(file_inode(obj->base.filp), 0, (loff_t)-1);
-	obj->madv = __I915_MADV_PURGED;
+	obj->mm.madv = __I915_MADV_PURGED;
  }
 
  /* Try to discard unwanted pages */
@@ -2237,7 +2230,7 @@ i915_gem_object_invalidate(struct drm_i915_gem_object *obj)
  {
  	struct address_space *mapping;
 
-	switch (obj->madv) {
+	switch (obj->mm.madv) {
  	case I915_MADV_DONTNEED:
  		i915_gem_object_truncate(obj);
  	case __I915_MADV_PURGED:
@@ -2258,7 +2251,7 @@ i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
  	struct page *page;
  	int ret;
 
-	BUG_ON(obj->madv == __I915_MADV_PURGED);
+	GEM_BUG_ON(obj->mm.madv == __I915_MADV_PURGED);
 
  	ret = i915_gem_object_set_to_cpu_domain(obj, true);
  	if (WARN_ON(ret)) {
@@ -2274,22 +2267,22 @@ i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
  	if (i915_gem_object_needs_bit17_swizzle(obj))
  		i915_gem_object_save_bit_17_swizzle(obj);
 
-	if (obj->madv == I915_MADV_DONTNEED)
-		obj->dirty = 0;
+	if (obj->mm.madv == I915_MADV_DONTNEED)
+		obj->mm.dirty = false;
 
-	for_each_sgt_page(page, sgt_iter, obj->pages) {
-		if (obj->dirty)
+	for_each_sgt_page(page, sgt_iter, obj->mm.pages) {
+		if (obj->mm.dirty)
  			set_page_dirty(page);
 
-		if (obj->madv == I915_MADV_WILLNEED)
+		if (obj->mm.madv == I915_MADV_WILLNEED)
  			mark_page_accessed(page);
 
  		put_page(page);
  	}
-	obj->dirty = 0;
+	obj->mm.dirty = false;
 
-	sg_free_table(obj->pages);
-	kfree(obj->pages);
+	sg_free_table(obj->mm.pages);
+	kfree(obj->mm.pages);
  }
 
  static void __i915_gem_object_reset_page_iter(struct drm_i915_gem_object *obj)
@@ -2297,21 +2290,20 @@ static void __i915_gem_object_reset_page_iter(struct drm_i915_gem_object *obj)
  	struct radix_tree_iter iter;
  	void **slot;
 
-	radix_tree_for_each_slot(slot, &obj->get_page.radix, &iter, 0)
-		radix_tree_delete(&obj->get_page.radix, iter.index);
+	radix_tree_for_each_slot(slot, &obj->mm.get_page.radix, &iter, 0)
+		radix_tree_delete(&obj->mm.get_page.radix, iter.index);
  }
 
-int
-i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
+int __i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
  {
  	const struct drm_i915_gem_object_ops *ops = obj->ops;
 
  	lockdep_assert_held(&obj->base.dev->struct_mutex);
 
-	if (obj->pages == NULL)
+	if (!obj->mm.pages)
  		return 0;
 
-	if (obj->pages_pin_count)
+	if (i915_gem_object_has_pinned_pages(obj))
  		return -EBUSY;
 
  	GEM_BUG_ON(obj->bind_count);
@@ -2321,22 +2313,22 @@ i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
  	 * lists early. */
  	list_del(&obj->global_list);
 
-	if (obj->mapping) {
+	if (obj->mm.mapping) {
  		void *ptr;
 
-		ptr = ptr_mask_bits(obj->mapping);
+		ptr = ptr_mask_bits(obj->mm.mapping);
  		if (is_vmalloc_addr(ptr))
  			vunmap(ptr);
  		else
  			kunmap(kmap_to_page(ptr));
 
-		obj->mapping = NULL;
+		obj->mm.mapping = NULL;
  	}
 
  	__i915_gem_object_reset_page_iter(obj);
 
  	ops->put_pages(obj);
-	obj->pages = NULL;
+	obj->mm.pages = NULL;
 
  	i915_gem_object_invalidate(obj);
 
@@ -2431,7 +2423,7 @@ i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
  	if (!swiotlb_nr_tbl())
  #endif
  		sg_mark_end(sg);
-	obj->pages = st;
+	obj->mm.pages = st;
 
  	ret = i915_gem_gtt_prepare_object(obj);
  	if (ret)
@@ -2442,7 +2434,7 @@ i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
 
  	if (i915_gem_object_is_tiled(obj) &&
  	    dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES)
-		i915_gem_object_pin_pages(obj);
+		__i915_gem_object_pin_pages(obj);
 
  	return 0;
 
@@ -2474,8 +2466,7 @@ err_pages:
   * either as a result of memory pressure (reaping pages under the shrinker)
   * or as the object is itself released.
   */
-int
-i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
+int __i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
  {
  	struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
  	const struct drm_i915_gem_object_ops *ops = obj->ops;
@@ -2483,24 +2474,25 @@ i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
 
  	lockdep_assert_held(&obj->base.dev->struct_mutex);
 
-	if (obj->pages)
+	if (obj->mm.pages)
  		return 0;
 
-	if (obj->madv != I915_MADV_WILLNEED) {
+	if (obj->mm.madv != I915_MADV_WILLNEED) {
  		DRM_DEBUG("Attempting to obtain a purgeable object\n");
+		__i915_gem_object_unpin_pages(obj);
  		return -EFAULT;
  	}
 
-	BUG_ON(obj->pages_pin_count);
-
  	ret = ops->get_pages(obj);
-	if (ret)
+	if (ret) {
+		__i915_gem_object_unpin_pages(obj);
  		return ret;
+	}
 
  	list_add_tail(&obj->global_list, &dev_priv->mm.unbound_list);
 
-	obj->get_page.sg_pos = obj->pages->sgl;
-	obj->get_page.sg_idx = 0;
+	obj->mm.get_page.sg_pos = obj->mm.pages->sgl;
+	obj->mm.get_page.sg_idx = 0;
 
  	return 0;
  }
@@ -2510,7 +2502,7 @@ static void *i915_gem_object_map(const struct drm_i915_gem_object *obj,
  				 enum i915_map_type type)
  {
  	unsigned long n_pages = obj->base.size >> PAGE_SHIFT;
-	struct sg_table *sgt = obj->pages;
+	struct sg_table *sgt = obj->mm.pages;
  	struct sgt_iter sgt_iter;
  	struct page *page;
  	struct page *stack_pages[32];
@@ -2564,14 +2556,13 @@ void *i915_gem_object_pin_map(struct drm_i915_gem_object *obj,
  	lockdep_assert_held(&obj->base.dev->struct_mutex);
  	GEM_BUG_ON(!i915_gem_object_has_struct_page(obj));
 
-	ret = i915_gem_object_get_pages(obj);
+	ret = i915_gem_object_pin_pages(obj);
  	if (ret)
  		return ERR_PTR(ret);
 
-	i915_gem_object_pin_pages(obj);
-	pinned = obj->pages_pin_count > 1;
+	pinned = obj->mm.pages_pin_count > 1;
 
-	ptr = ptr_unpack_bits(obj->mapping, has_type);
+	ptr = ptr_unpack_bits(obj->mm.mapping, has_type);
  	if (ptr && has_type != type) {
  		if (pinned) {
  			ret = -EBUSY;
@@ -2583,7 +2574,7 @@ void *i915_gem_object_pin_map(struct drm_i915_gem_object *obj,
  		else
  			kunmap(kmap_to_page(ptr));
 
-		ptr = obj->mapping = NULL;
+		ptr = obj->mm.mapping = NULL;
  	}
 
  	if (!ptr) {
@@ -2593,7 +2584,7 @@ void *i915_gem_object_pin_map(struct drm_i915_gem_object *obj,
  			goto err;
  		}
 
-		obj->mapping = ptr_pack_bits(ptr, type);
+		obj->mm.mapping = ptr_pack_bits(ptr, type);
  	}
 
  	return ptr;
@@ -3030,7 +3021,7 @@ int i915_vma_unbind(struct i915_vma *vma)
  		goto destroy;
 
  	GEM_BUG_ON(obj->bind_count == 0);
-	GEM_BUG_ON(!obj->pages);
+	GEM_BUG_ON(!obj->mm.pages);
 
  	if (i915_vma_is_map_and_fenceable(vma)) {
  		/* release the fence reg _after_ flushing */
@@ -3054,7 +3045,7 @@ int i915_vma_unbind(struct i915_vma *vma)
  	drm_mm_remove_node(&vma->node);
  	list_move_tail(&vma->vm_link, &vma->vm->unbound_list);
 
-	if (vma->pages != obj->pages) {
+	if (vma->pages != obj->mm.pages) {
  		GEM_BUG_ON(!vma->pages);
  		sg_free_table(vma->pages);
  		kfree(vma->pages);
@@ -3186,12 +3177,10 @@ i915_vma_insert(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
  		return -E2BIG;
  	}
 
-	ret = i915_gem_object_get_pages(obj);
+	ret = i915_gem_object_pin_pages(obj);
  	if (ret)
  		return ret;
 
-	i915_gem_object_pin_pages(obj);
-
  	if (flags & PIN_OFFSET_FIXED) {
  		u64 offset = flags & PIN_OFFSET_MASK;
  		if (offset & (alignment - 1) || offset > end - size) {
@@ -3270,7 +3259,7 @@ i915_gem_clflush_object(struct drm_i915_gem_object *obj,
  	 * to GPU, and we can ignore the cache flush because it'll happen
  	 * again at bind time.
  	 */
-	if (obj->pages == NULL)
+	if (!obj->mm.pages)
  		return false;
 
  	/*
@@ -3294,7 +3283,7 @@ i915_gem_clflush_object(struct drm_i915_gem_object *obj,
  	}
 
  	trace_i915_gem_object_clflush(obj);
-	drm_clflush_sg(obj->pages);
+	drm_clflush_sg(obj->mm.pages);
  	obj->cache_dirty = false;
 
  	return true;
@@ -3408,7 +3397,7 @@ i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
  	 * continue to assume that the obj remained out of the CPU cached
  	 * domain.
  	 */
-	ret = i915_gem_object_get_pages(obj);
+	ret = i915_gem_object_pin_pages(obj);
  	if (ret)
  		return ret;
 
@@ -3432,7 +3421,7 @@ i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
  	if (write) {
  		obj->base.read_domains = I915_GEM_DOMAIN_GTT;
  		obj->base.write_domain = I915_GEM_DOMAIN_GTT;
-		obj->dirty = 1;
+		obj->mm.dirty = true;
  	}
 
  	trace_i915_gem_object_change_domain(obj,
@@ -3441,6 +3430,7 @@ i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
 
  	/* And bump the LRU for this access */
  	i915_gem_object_bump_inactive_ggtt(obj);
+	i915_gem_object_unpin_pages(obj);
 
  	return 0;
  }
@@ -4210,23 +4200,23 @@ i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
  		goto unlock;
  	}
 
-	if (obj->pages &&
+	if (obj->mm.pages &&
  	    i915_gem_object_is_tiled(obj) &&
  	    dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES) {
-		if (obj->madv == I915_MADV_WILLNEED)
-			i915_gem_object_unpin_pages(obj);
+		if (obj->mm.madv == I915_MADV_WILLNEED)
+			__i915_gem_object_unpin_pages(obj);
  		if (args->madv == I915_MADV_WILLNEED)
-			i915_gem_object_pin_pages(obj);
+			__i915_gem_object_pin_pages(obj);
  	}
 
-	if (obj->madv != __I915_MADV_PURGED)
-		obj->madv = args->madv;
+	if (obj->mm.madv != __I915_MADV_PURGED)
+		obj->mm.madv = args->madv;
 
  	/* if the object is no longer attached, discard its backing storage */
-	if (obj->madv == I915_MADV_DONTNEED && obj->pages == NULL)
+	if (obj->mm.madv == I915_MADV_DONTNEED && !obj->mm.pages)
  		i915_gem_object_truncate(obj);
 
-	args->retained = obj->madv != __I915_MADV_PURGED;
+	args->retained = obj->mm.madv != __I915_MADV_PURGED;
 
  	i915_gem_object_put(obj);
  unlock:
@@ -4252,9 +4242,10 @@ void i915_gem_object_init(struct drm_i915_gem_object *obj,
  	obj->ops = ops;
 
  	obj->frontbuffer_ggtt_origin = ORIGIN_GTT;
-	obj->madv = I915_MADV_WILLNEED;
-	INIT_RADIX_TREE(&obj->get_page.radix, GFP_KERNEL);
-	mutex_init(&obj->get_page.lock);
+
+	obj->mm.madv = I915_MADV_WILLNEED;
+	INIT_RADIX_TREE(&obj->mm.get_page.radix, GFP_KERNEL);
+	mutex_init(&obj->mm.get_page.lock);
 
  	i915_gem_info_add_obj(to_i915(obj->base.dev), obj->base.size);
  }
@@ -4331,7 +4322,7 @@ static bool discard_backing_storage(struct drm_i915_gem_object *obj)
  	 * back the contents from the GPU.
  	 */
 
-	if (obj->madv != I915_MADV_WILLNEED)
+	if (obj->mm.madv != I915_MADV_WILLNEED)
  		return false;
 
  	if (obj->base.filp == NULL)
@@ -4373,32 +4364,27 @@ void i915_gem_free_object(struct drm_gem_object *gem_obj)
  	}
  	GEM_BUG_ON(obj->bind_count);
 
-	/* Stolen objects don't hold a ref, but do hold pin count. Fix that up
-	 * before progressing. */
-	if (obj->stolen)
-		i915_gem_object_unpin_pages(obj);
-
  	WARN_ON(atomic_read(&obj->frontbuffer_bits));
 
-	if (obj->pages && obj->madv == I915_MADV_WILLNEED &&
+	if (obj->mm.pages && obj->mm.madv == I915_MADV_WILLNEED &&
  	    dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES &&
  	    i915_gem_object_is_tiled(obj))
-		i915_gem_object_unpin_pages(obj);
+		__i915_gem_object_unpin_pages(obj);
 
-	if (WARN_ON(obj->pages_pin_count))
-		obj->pages_pin_count = 0;
+	if (obj->ops->release)
+		obj->ops->release(obj);
+
+	if (WARN_ON(i915_gem_object_has_pinned_pages(obj)))
+		obj->mm.pages_pin_count = 0;
  	if (discard_backing_storage(obj))
-		obj->madv = I915_MADV_DONTNEED;
-	i915_gem_object_put_pages(obj);
+		obj->mm.madv = I915_MADV_DONTNEED;
+	__i915_gem_object_put_pages(obj);
 
-	BUG_ON(obj->pages);
+	GEM_BUG_ON(obj->mm.pages);
 
  	if (obj->base.import_attach)
  		drm_prime_gem_destroy(&obj->base, NULL);
 
-	if (obj->ops->release)
-		obj->ops->release(obj);
-
  	drm_gem_object_release(&obj->base);
  	i915_gem_info_remove_obj(dev_priv, obj->base.size);
 
@@ -4935,14 +4921,13 @@ i915_gem_object_create_from_data(struct drm_device *dev,
  	if (ret)
  		goto fail;
 
-	ret = i915_gem_object_get_pages(obj);
+	ret = i915_gem_object_pin_pages(obj);
  	if (ret)
  		goto fail;
 
-	i915_gem_object_pin_pages(obj);
-	sg = obj->pages;
+	sg = obj->mm.pages;
  	bytes = sg_copy_from_buffer(sg->sgl, sg->nents, (void *)data, size);
-	obj->dirty = 1;		/* Backing store is now out of date */
+	obj->mm.dirty = true; /* Backing store is now out of date */
  	i915_gem_object_unpin_pages(obj);
 
  	if (WARN_ON(bytes != size)) {
@@ -4962,7 +4947,7 @@ static struct scatterlist *
  __i915_gem_object_get_sg(struct drm_i915_gem_object *obj,
  			 unsigned long n, unsigned int *offset)
  {
-	struct scatterlist *sg = obj->pages->sgl;
+	struct scatterlist *sg = obj->mm.pages->sgl;
  	int idx = 0;
 
  	while (idx + __sg_page_count(sg) <= n) {
@@ -4980,11 +4965,11 @@ i915_gem_object_get_sg(struct drm_i915_gem_object *obj,
  		       unsigned long n,
  		       unsigned int *offset)
  {
-	struct i915_gem_object_page_iter *iter = &obj->get_page;
+	struct i915_gem_object_page_iter *iter = &obj->mm.get_page;
  	struct scatterlist *sg;
 
  	GEM_BUG_ON(n >= obj->base.size >> PAGE_SHIFT);
-	GEM_BUG_ON(obj->pages_pin_count == 0);
+	GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));
 
  	if (n < READ_ONCE(iter->sg_idx))
  		goto lookup;
@@ -5058,7 +5043,7 @@ i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj,
  	struct page *page;
 
  	page = i915_gem_object_get_page(obj, n);
-	if (!obj->dirty)
+	if (!obj->mm.dirty)
  		set_page_dirty(page);
 
  	return page;
drivers/gpu/drm/i915/i915_gem_batch_pool.c +1 -2
index 3934c9103cf2..6b656822bb3a 100644
@@ -131,11 +131,10 @@ i915_gem_batch_pool_get(struct i915_gem_batch_pool *pool,
  			return obj;
  	}
 
-	ret = i915_gem_object_get_pages(obj);
+	ret = i915_gem_object_pin_pages(obj);
  	if (ret)
  		return ERR_PTR(ret);
 
  	list_move_tail(&obj->batch_pool_link, list);
-	i915_gem_object_pin_pages(obj);
  	return obj;
  }
drivers/gpu/drm/i915/i915_gem_dmabuf.c +6 -8
index 97c9d68b45df..10441dc72e73 100644
@@ -48,12 +48,10 @@ static struct sg_table *i915_gem_map_dma_buf(struct dma_buf_attachment *attachme
  	if (ret)
  		goto err;
 
-	ret = i915_gem_object_get_pages(obj);
+	ret = i915_gem_object_pin_pages(obj);
  	if (ret)
  		goto err_unlock;
 
-	i915_gem_object_pin_pages(obj);
-
  	/* Copy sg so that we make an independent mapping */
  	st = kmalloc(sizeof(struct sg_table), GFP_KERNEL);
  	if (st == NULL) {
@@ -61,13 +59,13 @@ static struct sg_table *i915_gem_map_dma_buf(struct dma_buf_attachment *attachme
  		goto err_unpin;
  	}
 
-	ret = sg_alloc_table(st, obj->pages->nents, GFP_KERNEL);
+	ret = sg_alloc_table(st, obj->mm.pages->nents, GFP_KERNEL);
  	if (ret)
  		goto err_free;
 
-	src = obj->pages->sgl;
+	src = obj->mm.pages->sgl;
  	dst = st->sgl;
-	for (i = 0; i < obj->pages->nents; i++) {
+	for (i = 0; i < obj->mm.pages->nents; i++) {
  		sg_set_page(dst, sg_page(src), src->length, 0);
  		dst = sg_next(dst);
  		src = sg_next(src);
@@ -299,14 +297,14 @@ static int i915_gem_object_get_pages_dmabuf(struct drm_i915_gem_object *obj)
  	if (IS_ERR(sg))
  		return PTR_ERR(sg);
 
-	obj->pages = sg;
+	obj->mm.pages = sg;
  	return 0;
  }
 
  static void i915_gem_object_put_pages_dmabuf(struct drm_i915_gem_object *obj)
  {
  	dma_buf_unmap_attachment(obj->base.import_attach,
-				 obj->pages, DMA_BIDIRECTIONAL);
+				 obj->mm.pages, DMA_BIDIRECTIONAL);
  }
 
  static const struct drm_i915_gem_object_ops i915_gem_object_dmabuf_ops = {
drivers/gpu/drm/i915/i915_gem_execbuffer.c +1 -1
index 0deecd4e3b6c..062e098b0909 100644
@@ -1290,7 +1290,7 @@ void i915_vma_move_to_active(struct i915_vma *vma,
 
  	GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
 
-	obj->dirty = 1; /* be paranoid  */
+	obj->mm.dirty = true; /* be paranoid  */
 
  	/* Add a reference if we're newly entering the active list.
  	 * The order in which we add operations to the retirement queue is
drivers/gpu/drm/i915/i915_gem_fence.c +2 -2
index 8df1fa7234e8..398856160656 100644
@@ -648,7 +648,7 @@ i915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object *obj)
  		return;
 
  	i = 0;
-	for_each_sgt_page(page, sgt_iter, obj->pages) {
+	for_each_sgt_page(page, sgt_iter, obj->mm.pages) {
  		char new_bit_17 = page_to_phys(page) >> 17;
  		if ((new_bit_17 & 0x1) !=
  		    (test_bit(i, obj->bit_17) != 0)) {
@@ -687,7 +687,7 @@ i915_gem_object_save_bit_17_swizzle(struct drm_i915_gem_object *obj)
 
  	i = 0;
 
-	for_each_sgt_page(page, sgt_iter, obj->pages) {
+	for_each_sgt_page(page, sgt_iter, obj->mm.pages) {
  		if (page_to_phys(page) & (1 << 17))
  			__set_bit(i, obj->bit_17);
  		else
drivers/gpu/drm/i915/i915_gem_gtt.c +5 -5
index 7e0c98576e72..3c711e0b8a3f 100644
@@ -175,7 +175,7 @@ static int ppgtt_bind_vma(struct i915_vma *vma,
  {
  	u32 pte_flags = 0;
 
-	vma->pages = vma->obj->pages;
+	vma->pages = vma->obj->mm.pages;
 
  	/* Currently applicable only to VLV */
  	if (vma->obj->gt_ro)
@@ -2295,7 +2295,7 @@ void i915_gem_suspend_gtt_mappings(struct drm_device *dev)
  int i915_gem_gtt_prepare_object(struct drm_i915_gem_object *obj)
  {
  	if (!dma_map_sg(&obj->base.dev->pdev->dev,
-			obj->pages->sgl, obj->pages->nents,
+			obj->mm.pages->sgl, obj->mm.pages->nents,
  			PCI_DMA_BIDIRECTIONAL))
  		return -ENOSPC;
 
@@ -2685,7 +2685,7 @@ void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj)
  		}
  	}
 
-	dma_unmap_sg(kdev, obj->pages->sgl, obj->pages->nents,
+	dma_unmap_sg(kdev, obj->mm.pages->sgl, obj->mm.pages->nents,
  		     PCI_DMA_BIDIRECTIONAL);
  }
 
@@ -3526,7 +3526,7 @@ intel_rotate_fb_obj_pages(const struct intel_rotation_info *rot_info,
 
  	/* Populate source page list from the object. */
  	i = 0;
-	for_each_sgt_dma(dma_addr, sgt_iter, obj->pages)
+	for_each_sgt_dma(dma_addr, sgt_iter, obj->mm.pages)
  		page_addr_list[i++] = dma_addr;
 
  	GEM_BUG_ON(i != n_pages);
@@ -3617,7 +3617,7 @@ i915_get_ggtt_vma_pages(struct i915_vma *vma)
  		return 0;
 
  	if (vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL)
-		vma->pages = vma->obj->pages;
+		vma->pages = vma->obj->mm.pages;
  	else if (vma->ggtt_view.type == I915_GGTT_VIEW_ROTATED)
  		vma->pages =
  			intel_rotate_fb_obj_pages(&vma->ggtt_view.params.rotated, vma->obj);
drivers/gpu/drm/i915/i915_gem_internal.c +10 -9
index 534a61c1aba2..24eb347f9e0a 100644
@@ -28,10 +28,11 @@
 
  static void internal_free_pages(struct sg_table *st)
  {
-	struct sg_page_iter sg_iter;
+	struct sgt_iter iter;
+	struct page *page;
 
-	for_each_sg_page(st->sgl, &sg_iter, st->nents, 0)
-		put_page(sg_page_iter_page(&sg_iter));
+	for_each_sgt_page(page, iter, st)
+		put_page(page);
 
  	sg_free_table(st);
  	kfree(st);
@@ -94,10 +95,10 @@ static int i915_gem_object_get_pages_internal(struct drm_i915_gem_object *obj)
  	if (!swiotlb_nr_tbl())
  #endif
  		sg_mark_end(sg);
-	obj->pages = st;
+	obj->mm.pages = st;
 
  	if (i915_gem_gtt_prepare_object(obj)) {
-		obj->pages = NULL;
+		obj->mm.pages = NULL;
  		goto err;
  	}
 
@@ -106,7 +107,7 @@ static int i915_gem_object_get_pages_internal(struct drm_i915_gem_object *obj)
  	 * and the caller is expected to repopulate - the contents of this
  	 * object are only valid whilst active and pinned.
  	 */
-	obj->madv = I915_MADV_DONTNEED;
+	obj->mm.madv = I915_MADV_DONTNEED;
  	return 0;
 
  err:
@@ -117,10 +118,10 @@ err:
 
  static void i915_gem_object_put_pages_internal(struct drm_i915_gem_object *obj)
  {
-	internal_free_pages(obj->pages);
+	internal_free_pages(obj->mm.pages);
 
-	obj->dirty = 0;
-	obj->madv = I915_MADV_WILLNEED;
+	obj->mm.dirty = false;
+	obj->mm.madv = I915_MADV_WILLNEED;
  }
 
  static const struct drm_i915_gem_object_ops i915_gem_object_internal_ops = {
drivers/gpu/drm/i915/i915_gem_render_state.c +1 -1
index adb00c5125ad..64d9712ec789 100644
@@ -230,7 +230,7 @@ int i915_gem_render_state_emit(struct drm_i915_gem_request *req)
  		return 0;
 
  	/* Recreate the page after shrinking */
-	if (!so->vma->obj->pages)
+	if (!so->vma->obj->mm.pages)
  		so->batch_offset = -1;
 
  	ret = i915_vma_pin(so->vma, 0, 0, PIN_GLOBAL | PIN_HIGH);
drivers/gpu/drm/i915/i915_gem_shrinker.c +5 -5
index 1c237d02f30b..e0e6d68fe470 100644
@@ -78,7 +78,7 @@ static bool can_release_pages(struct drm_i915_gem_object *obj)
  	 * to the GPU, simply unbinding from the GPU is not going to succeed
  	 * in releasing our pin count on the pages themselves.
  	 */
-	if (obj->pages_pin_count > obj->bind_count)
+	if (obj->mm.pages_pin_count > obj->bind_count)
  		return false;
 
  	if (any_vma_pinned(obj))
@@ -88,7 +88,7 @@ static bool can_release_pages(struct drm_i915_gem_object *obj)
  	 * discard the contents (because the user has marked them as being
  	 * purgeable) or if we can move their contents out to swap.
  	 */
-	return swap_available() || obj->madv == I915_MADV_DONTNEED;
+	return swap_available() || obj->mm.madv == I915_MADV_DONTNEED;
  }
 
  /**
@@ -175,11 +175,11 @@ i915_gem_shrink(struct drm_i915_private *dev_priv,
  			list_move_tail(&obj->global_list, &still_in_list);
 
  			if (flags & I915_SHRINK_PURGEABLE &&
-			    obj->madv != I915_MADV_DONTNEED)
+			    obj->mm.madv != I915_MADV_DONTNEED)
  				continue;
 
  			if (flags & I915_SHRINK_VMAPS &&
-			    !is_vmalloc_addr(obj->mapping))
+			    !is_vmalloc_addr(obj->mm.mapping))
  				continue;
 
  			if ((flags & I915_SHRINK_ACTIVE) == 0 &&
@@ -193,7 +193,7 @@ i915_gem_shrink(struct drm_i915_private *dev_priv,
 
  			/* For the unbound phase, this should be a no-op! */
  			i915_gem_object_unbind(obj);
-			if (i915_gem_object_put_pages(obj) == 0)
+			if (__i915_gem_object_put_pages(obj) == 0)
  				count += obj->base.size >> PAGE_SHIFT;
 
  			i915_gem_object_put(obj);
drivers/gpu/drm/i915/i915_gem_stolen.c +13 -11
index 24bad4e60ef0..6d139d6b4609 100644
@@ -554,16 +554,17 @@ static int i915_gem_object_get_pages_stolen(struct drm_i915_gem_object *obj)
  static void i915_gem_object_put_pages_stolen(struct drm_i915_gem_object *obj)
  {
  	/* Should only be called during free */
-	sg_free_table(obj->pages);
-	kfree(obj->pages);
+	sg_free_table(obj->mm.pages);
+	kfree(obj->mm.pages);
  }
 
-
  static void
  i915_gem_object_release_stolen(struct drm_i915_gem_object *obj)
  {
  	struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
 
+	__i915_gem_object_unpin_pages(obj);
+
  	if (obj->stolen) {
  		i915_gem_stolen_remove_node(dev_priv, obj->stolen);
  		kfree(obj->stolen);
@@ -589,15 +590,16 @@ _i915_gem_object_create_stolen(struct drm_device *dev,
  	drm_gem_private_object_init(dev, &obj->base, stolen->size);
  	i915_gem_object_init(obj, &i915_gem_object_stolen_ops);
 
-	obj->pages = i915_pages_create_for_stolen(dev,
-						  stolen->start, stolen->size);
-	if (obj->pages == NULL)
+	obj->mm.pages = i915_pages_create_for_stolen(dev,
+						     stolen->start,
+						     stolen->size);
+	if (!obj->mm.pages)
  		goto cleanup;
 
-	obj->get_page.sg_pos = obj->pages->sgl;
-	obj->get_page.sg_idx = 0;
+	obj->mm.get_page.sg_pos = obj->mm.pages->sgl;
+	obj->mm.get_page.sg_idx = 0;
 
-	i915_gem_object_pin_pages(obj);
+	__i915_gem_object_pin_pages(obj);
  	obj->stolen = stolen;
 
  	obj->base.read_domains = I915_GEM_DOMAIN_CPU | I915_GEM_DOMAIN_GTT;
@@ -717,14 +719,14 @@ i915_gem_object_create_stolen_for_preallocated(struct drm_device *dev,
  		goto err;
  	}
 
-	vma->pages = obj->pages;
+	vma->pages = obj->mm.pages;
  	vma->flags |= I915_VMA_GLOBAL_BIND;
  	__i915_vma_set_map_and_fenceable(vma);
  	list_move_tail(&vma->vm_link, &ggtt->base.inactive_list);
  	obj->bind_count++;
 
  	list_add_tail(&obj->global_list, &dev_priv->mm.bound_list);
-	i915_gem_object_pin_pages(obj);
+	__i915_gem_object_pin_pages(obj);
 
  	return obj;
 
drivers/gpu/drm/i915/i915_gem_tiling.c +4 -4
index a14b1e3d4c78..7286de7bd25e 100644
@@ -260,13 +260,13 @@ i915_gem_set_tiling(struct drm_device *dev, void *data,
  		if (!err) {
  			struct i915_vma *vma;
 
-			if (obj->pages &&
-			    obj->madv == I915_MADV_WILLNEED &&
+			if (obj->mm.pages &&
+			    obj->mm.madv == I915_MADV_WILLNEED &&
  			    dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES) {
  				if (args->tiling_mode == I915_TILING_NONE)
-					i915_gem_object_unpin_pages(obj);
+					__i915_gem_object_unpin_pages(obj);
  				if (!i915_gem_object_is_tiled(obj))
-					i915_gem_object_pin_pages(obj);
+					__i915_gem_object_pin_pages(obj);
  			}
 
  			list_for_each_entry(vma, &obj->vma_list, obj_link) {
drivers/gpu/drm/i915/i915_gem_userptr.c +15 -15
index cb95789da76e..0fdd1d6723d1 100644
@@ -73,10 +73,10 @@ static void cancel_userptr(struct work_struct *work)
  	/* Cancel any active worker and force us to re-evaluate gup */
  	obj->userptr.work = NULL;
 
-	if (obj->pages != NULL) {
+	if (obj->mm.pages) {
  		/* We are inside a kthread context and can't be interrupted */
  		WARN_ON(i915_gem_object_unbind(obj));
-		WARN_ON(i915_gem_object_put_pages(obj));
+		WARN_ON(__i915_gem_object_put_pages(obj));
  	}
 
  	i915_gem_object_put(obj);
@@ -432,15 +432,15 @@ __i915_gem_userptr_set_pages(struct drm_i915_gem_object *obj,
  {
  	int ret;
 
-	ret = st_set_pages(&obj->pages, pvec, num_pages);
+	ret = st_set_pages(&obj->mm.pages, pvec, num_pages);
  	if (ret)
  		return ret;
 
  	ret = i915_gem_gtt_prepare_object(obj);
  	if (ret) {
-		sg_free_table(obj->pages);
-		kfree(obj->pages);
-		obj->pages = NULL;
+		sg_free_table(obj->mm.pages);
+		kfree(obj->mm.pages);
+		obj->mm.pages = NULL;
  	}
 
  	return ret;
@@ -526,8 +526,8 @@ __i915_gem_userptr_get_pages_worker(struct work_struct *_work)
  			if (ret == 0) {
  				list_add_tail(&obj->global_list,
  					      &to_i915(dev)->mm.unbound_list);
-				obj->get_page.sg_pos = obj->pages->sgl;
-				obj->get_page.sg_idx = 0;
+				obj->mm.get_page.sg_pos = obj->mm.pages->sgl;
+				obj->mm.get_page.sg_idx = 0;
  				pinned = 0;
  			}
  		}
@@ -668,22 +668,22 @@ i915_gem_userptr_put_pages(struct drm_i915_gem_object *obj)
  	BUG_ON(obj->userptr.work != NULL);
  	__i915_gem_userptr_set_active(obj, false);
 
-	if (obj->madv != I915_MADV_WILLNEED)
-		obj->dirty = 0;
+	if (obj->mm.madv != I915_MADV_WILLNEED)
+		obj->mm.dirty = false;
 
  	i915_gem_gtt_finish_object(obj);
 
-	for_each_sgt_page(page, sgt_iter, obj->pages) {
-		if (obj->dirty)
+	for_each_sgt_page(page, sgt_iter, obj->mm.pages) {
+		if (obj->mm.dirty)
  			set_page_dirty(page);
 
  		mark_page_accessed(page);
  		put_page(page);
  	}
-	obj->dirty = 0;
+	obj->mm.dirty = false;
 
-	sg_free_table(obj->pages);
-	kfree(obj->pages);
+	sg_free_table(obj->mm.pages);
+	kfree(obj->mm.pages);
  }
 
  static void
drivers/gpu/drm/i915/i915_gpu_error.c +2 -2
index 6c22be28ba01..628d5cdf9200 100644
@@ -881,8 +881,8 @@ static void capture_bo(struct drm_i915_error_buffer *err,
  	err->write_domain = obj->base.write_domain;
  	err->fence_reg = vma->fence ? vma->fence->id : -1;
  	err->tiling = i915_gem_object_get_tiling(obj);
-	err->dirty = obj->dirty;
-	err->purgeable = obj->madv != I915_MADV_WILLNEED;
+	err->dirty = obj->mm.dirty;
+	err->purgeable = obj->mm.madv != I915_MADV_WILLNEED;
  	err->userptr = obj->userptr.mm != NULL;
  	err->cache_level = obj->cache_level;
  }
drivers/gpu/drm/i915/intel_lrc.c +2 -2
index 02c48d390148..6acecfc41f5e 100644
@@ -744,7 +744,7 @@ static int intel_lr_context_pin(struct i915_gem_context *ctx,
  	ce->lrc_reg_state[CTX_RING_BUFFER_START+1] =
  		i915_ggtt_offset(ce->ring->vma);
 
-	ce->state->obj->dirty = true;
+	ce->state->obj->mm.dirty = true;
 
  	/* Invalidate GuC TLB. */
  	if (i915.enable_guc_submission) {
@@ -2042,7 +2042,7 @@ populate_lr_context(struct i915_gem_context *ctx,
  		DRM_DEBUG_DRIVER("Could not map object pages! (%d)\n", ret);
  		return ret;
  	}
-	ctx_obj->dirty = true;
+	ctx_obj->mm.dirty = true;
 
  	/* The second page of the context object contains some fields which must
  	 * be set up prior to the first execution. */

drivers/gpu/drm/i915/intel_lrc.c void intel_lr_context_resume(struct drm_i915_private *dev_priv) · line 2185

Quoted 8 lines
drivers/gpu/drm/i915/intel_lrc.c +1 -1
@@ -2179,7 +2179,7 @@ void intel_lr_context_resume(struct drm_i915_private *dev_priv)
  			reg[CTX_RING_HEAD+1] = 0;
  			reg[CTX_RING_TAIL+1] = 0;
 
-			ce->state->obj->dirty = true;
+			ce->state->obj->mm.dirty = true;
  			i915_gem_object_unpin_map(ce->state->obj);
 
  			ce->ring->head = ce->ring->tail = 0;

Did not spot any issues. Hopefully CI is good enough to helps us if we missed something.

It even looks better (simpler) than the current API which was always slightly confusing.

Regards,

Tvrtko

_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx

maintainer Reviewed-by on v1 16/42
#51 lore raw

Re: [PATCH 16/42] drm/i915: Refactor object page API

In reply to Chris Wilson

drivers/gpu/drm/i915/i915_drv.h dma_addr_t · line 3196

Quoted 6 lines · Chris Wilson
drivers/gpu/drm/i915/i915_drv.h +2 -0
+static inline int __must_check
+i915_gem_object_pin_pages(struct drm_i915_gem_object *obj)

 {

drivers/gpu/drm/i915/i915_drv.h +1 -2
-	BUG_ON(obj->pages == NULL);
-	obj->pages_pin_count++;
+	lockdep_assert_held(&obj->base.dev->struct_mutex);

\n here.

drivers/gpu/drm/i915/i915_drv.h dma_addr_t · line 3200

Quoted 4 lines
drivers/gpu/drm/i915/i915_drv.h +4 -0
+	if (obj->mm.pages_pin_count++)
+		return 0;
+
+	return __i915_gem_object_get_pages(obj);

I also second John here, what's up with the dummy wrapper convetion. Add TODO comments if you intend to introduce new stuff relying on that.

drivers/gpu/drm/i915/i915_drv.h dma_addr_t · line 3206

Quoted 6 lines
drivers/gpu/drm/i915/i915_drv.h +6 -0
+}
+
+static inline void
+__i915_gem_object_pin_pages(struct drm_i915_gem_object *obj)
+{
+	lockdep_assert_held(&obj->base.dev->struct_mutex);

Ditto.

Quoted 6 lines
drivers/gpu/drm/i915/i915_drv.h +6 -0
+	GEM_BUG_ON(!obj->mm.pages);
+	obj->mm.pages_pin_count++;
+}
+
+static inline bool
+i915_gem_object_has_pinned_pages(struct drm_i915_gem_object *obj)

drivers/gpu/drm/i915/i915_drv.h dma_addr_t · line 3222

Quoted 10 lines
drivers/gpu/drm/i915/i915_drv.h +10 -0
+{
+	return obj->mm.pages_pin_count;
+}
+
+static inline void
+__i915_gem_object_unpin_pages(struct drm_i915_gem_object *obj)
+{
+	lockdep_assert_held(&obj->base.dev->struct_mutex);
+	GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));
+	GEM_BUG_ON(!obj->mm.pages);

\n at least here.

Quoted 3 lines
drivers/gpu/drm/i915/i915_drv.h +1 -0
+	obj->mm.pages_pin_count--;

 }

<SNIP>

Review

Quoted 6 lines
drivers/gpu/drm/i915/i915_gem.c +0 -0
@@ -812,7 +809,7 @@ int i915_gem_obj_prepare_shmem_write(struct drm_i915_gem_object *obj,
  		obj->cache_dirty = true;


  intel_fb_obj_invalidate(obj, ORIGIN_CPU);

(quoted diff) +1 -1
-	obj->dirty = 1;
+	obj->mm.dirty = true;

The type is not bool dirty : 1, do we get yelled at by compiler?

drivers/gpu/drm/i915/i915_gem.c i915_gem_gtt_pwrite_fast(struct drm_i915_private *i915, · line 1264

Quoted 6 lines
drivers/gpu/drm/i915/i915_gem.c +0 -0
@@ -1268,7 +1261,7 @@ i915_gem_gtt_pwrite_fast(struct drm_i915_private *i915,
  		goto out_unpin;


  intel_fb_obj_invalidate(obj, ORIGIN_CPU);

drivers/gpu/drm/i915/i915_gem.c +1 -1
-	obj->dirty = true;
+	obj->mm.dirty = true;

I assume not as this has not been changed. Consistency is good anyway.

Review

Quoted 12 lines
(quoted diff) +0 -0
@@ -2483,24 +2474,25 @@ i915_gem_object_get_pages(struct drm_i915_gem_object *obj)

lockdep_assert_held(&obj->base.dev->struct_mutex);

drivers/gpu/drm/i915/i915_gem.c +1 -1
-	if (obj->pages)
+	if (obj->mm.pages)

  return 0;

drivers/gpu/drm/i915/i915_gem.c +1 -1
-	if (obj->madv != I915_MADV_WILLNEED) {
+	if (obj->mm.madv != I915_MADV_WILLNEED) {

  DRM_DEBUG("Attempting to obtain a purgeable object\n");

(quoted diff) +1 -0
+		__i915_gem_object_unpin_pages(obj);

Confusing to have teardown of another function in here.

drivers/gpu/drm/i915/i915_gem.c i915_gem_object_get_pages(struct drm_i915_gem_object *obj) · line 2488

Quoted 9 lines

  return -EFAULT;
  }

drivers/gpu/drm/i915/i915_gem.c +0 -2
-	BUG_ON(obj->pages_pin_count);
-

  ret = ops->get_pages(obj);

drivers/gpu/drm/i915/i915_gem.c +2 -1
-	if (ret)
+	if (ret) {
+		__i915_gem_object_unpin_pages(obj);

And if you like *really* have to, at least try not to duplicate code. Bonus points form moving this to be proper teardown path where it has a counter-part.

Strange side effects on failing function call do not lead to very understandable code structure, but major spaghetti. Can be fixed in follow-up patch too.

Regards, Joonas

Signature · 7 lines
Joonas Lahtinen
Open Source Technology Center
Intel Corporation
_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
#52 lore raw

Re: [PATCH 16/42] drm/i915: Refactor object page API

Quoted 16 lines · Joonas Lahtinen

Review

Quoted 12 lines · Chris Wilson
(quoted diff) +0 -0
@@ -2483,24 +2474,25 @@ i915_gem_object_get_pages(struct drm_i915_gem_object *obj)

lockdep_assert_held(&obj->base.dev->struct_mutex);

drivers/gpu/drm/i915/i915_gem.c +1 -1
-	if (obj->pages)
+	if (obj->mm.pages)

  return 0;

drivers/gpu/drm/i915/i915_gem.c +1 -1
-	if (obj->madv != I915_MADV_WILLNEED) {
+	if (obj->mm.madv != I915_MADV_WILLNEED) {

  DRM_DEBUG("Attempting to obtain a purgeable object\n");

(quoted diff) +1 -0
+		__i915_gem_object_unpin_pages(obj);

Confusing to have teardown of another function in here.

Quoted 13 lines

drivers/gpu/drm/i915/i915_gem.c i915_gem_object_get_pages(struct drm_i915_gem_object *obj) · line 2488

Quoted 9 lines

  return -EFAULT;
  }

drivers/gpu/drm/i915/i915_gem.c +0 -2
-	BUG_ON(obj->pages_pin_count);
-

  ret = ops->get_pages(obj);

drivers/gpu/drm/i915/i915_gem.c +2 -1
-	if (ret)
+	if (ret) {
+		__i915_gem_object_unpin_pages(obj);

And if you like *really* have to, at least try not to duplicate code. Bonus points form moving this to be proper teardown path where it has a counter-part.

It is here, so that the unlikely error handling is not inside the inlined function, but where we expect the code to grow to handle the more complex locking requirements.
-Chris

Signature · 5 lines
Chris Wilson, Intel Open Source Technology Centre
_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
#53 lore raw

PATCH 17/42 drm/i915: Pass around sg_table to get_pages/put_pages backend

In reply to Chris Wilson

The plan is to move obj->pages out from under the struct_mutex into its own per-object lock. We need to prune any assumption of the struct_mutex from the get_pages/put_pages backends, and to make it easier we pass around the sg_table to operate on rather than indirectly via the obj.

---
 drivers/gpu/drm/i915/i915_drv.h          |  36 +++++--
 drivers/gpu/drm/i915/i915_gem.c          | 172 +++++++++++++++----------------
 drivers/gpu/drm/i915/i915_gem_dmabuf.c   |  20 ++--
 drivers/gpu/drm/i915/i915_gem_fence.c    |  18 ++--
 drivers/gpu/drm/i915/i915_gem_gtt.c      |  19 ++--
 drivers/gpu/drm/i915/i915_gem_gtt.h      |   6 +-
 drivers/gpu/drm/i915/i915_gem_internal.c |  22 ++--
 drivers/gpu/drm/i915/i915_gem_shrinker.c |  11 +-
 drivers/gpu/drm/i915/i915_gem_stolen.c   |  43 ++++----
 drivers/gpu/drm/i915/i915_gem_userptr.c  |  88 ++++++++--------
 10 files changed, 227 insertions(+), 208 deletions(-)
drivers/gpu/drm/i915/i915_drv.h +27 -9
index 3c22d49005fe..271e63c8f037 100644
@@ -2175,8 +2175,8 @@ struct drm_i915_gem_object_ops {
 	 * being released or under memory pressure (where we attempt to
 	 * reap pages for the shrinker).
 	 */
-	int (*get_pages)(struct drm_i915_gem_object *);
-	void (*put_pages)(struct drm_i915_gem_object *);
+	struct sg_table *(*get_pages)(struct drm_i915_gem_object *);
+	void (*put_pages)(struct drm_i915_gem_object *, struct sg_table *);
 
 	int (*dmabuf_export)(struct drm_i915_gem_object *);
 	void (*release)(struct drm_i915_gem_object *);
@@ -2313,8 +2313,6 @@ struct drm_i915_gem_object {
 	struct i915_gem_userptr {
 		uintptr_t ptr;
 		unsigned read_only :1;
-		unsigned workers :4;
-#define I915_GEM_USERPTR_MAX_WORKERS 15
 
 		struct i915_mm_struct *mm;
 		struct i915_mmu_object *mmu_object;
@@ -2376,6 +2374,19 @@ __deprecated
 extern void drm_gem_object_unreference_unlocked(struct drm_gem_object *);
 
 static inline bool
+i915_gem_object_is_dead(const struct drm_i915_gem_object *obj)
+{
+	return atomic_read(&obj->base.refcount.refcount) == 0;
+}
+
+#if IS_ENABLED(CONFIG_LOCKDEP)
+#define lockdep_assert_held_unless(lock, cond) \
+	GEM_BUG_ON(!lockdep_is_held(lock) && !(cond))
+#else
+#define lockdep_assert_held_unless(lock, cond)
+#endif
+
+static inline bool
 i915_gem_object_has_struct_page(const struct drm_i915_gem_object *obj)
 {
 	return obj->ops->flags & I915_GEM_OBJECT_HAS_STRUCT_PAGE;
@@ -3188,6 +3199,8 @@ dma_addr_t
 i915_gem_object_get_dma_address(struct drm_i915_gem_object *obj,
 				unsigned long n);
 
+void __i915_gem_object_set_pages(struct drm_i915_gem_object *obj,
+				 struct sg_table *pages);
 int __i915_gem_object_get_pages(struct drm_i915_gem_object *obj);
 
 static inline int __must_check
@@ -3203,7 +3216,8 @@ i915_gem_object_pin_pages(struct drm_i915_gem_object *obj)
 static inline void
 __i915_gem_object_pin_pages(struct drm_i915_gem_object *obj)
 {
-	lockdep_assert_held(&obj->base.dev->struct_mutex);
+	lockdep_assert_held_unless(&obj->base.dev->struct_mutex,
+				   i915_gem_object_is_dead(obj));
 	GEM_BUG_ON(!obj->mm.pages);
 	obj->mm.pages_pin_count++;
 }
@@ -3217,7 +3231,8 @@ i915_gem_object_has_pinned_pages(struct drm_i915_gem_object *obj)
 static inline void
 __i915_gem_object_unpin_pages(struct drm_i915_gem_object *obj)
 {
-	lockdep_assert_held(&obj->base.dev->struct_mutex);
+	lockdep_assert_held_unless(&obj->base.dev->struct_mutex,
+				   i915_gem_object_is_dead(obj));
 	GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));
 	GEM_BUG_ON(!obj->mm.pages);
 	obj->mm.pages_pin_count--;
@@ -3228,7 +3243,8 @@ static inline void i915_gem_object_unpin_pages(struct drm_i915_gem_object *obj)
 	__i915_gem_object_unpin_pages(obj);
 }
 
-int __i915_gem_object_put_pages(struct drm_i915_gem_object *obj);
+void __i915_gem_object_put_pages(struct drm_i915_gem_object *obj);
+void __i915_gem_object_invalidate(struct drm_i915_gem_object *obj);
 
 enum i915_map_type {
 	I915_MAP_WB = 0,
@@ -3451,8 +3467,10 @@ i915_vma_unpin_fence(struct i915_vma *vma)
 void i915_gem_restore_fences(struct drm_device *dev);
 
 void i915_gem_detect_bit_6_swizzle(struct drm_device *dev);
-void i915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object *obj);
-void i915_gem_object_save_bit_17_swizzle(struct drm_i915_gem_object *obj);
+void i915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object *obj,
+				       struct sg_table *pages);
+void i915_gem_object_save_bit_17_swizzle(struct drm_i915_gem_object *obj,
+					 struct sg_table *pages);
 
 /* i915_gem_context.c */
 int __must_check i915_gem_context_init(struct drm_device *dev);
drivers/gpu/drm/i915/i915_gem.c +83 -89

338 lines not on this page · raw message

drivers/gpu/drm/i915/i915_gem_dmabuf.c +8 -12
index 10441dc72e73..2abd524aba14 100644
@@ -289,22 +289,18 @@ struct dma_buf *i915_gem_prime_export(struct drm_device *dev,
 	return dma_buf;
 }
 
-static int i915_gem_object_get_pages_dmabuf(struct drm_i915_gem_object *obj)
+static struct sg_table *
+i915_gem_object_get_pages_dmabuf(struct drm_i915_gem_object *obj)
 {
-	struct sg_table *sg;
-
-	sg = dma_buf_map_attachment(obj->base.import_attach, DMA_BIDIRECTIONAL);
-	if (IS_ERR(sg))
-		return PTR_ERR(sg);
-
-	obj->mm.pages = sg;
-	return 0;
+	return dma_buf_map_attachment(obj->base.import_attach,
+				      DMA_BIDIRECTIONAL);
 }
 
-static void i915_gem_object_put_pages_dmabuf(struct drm_i915_gem_object *obj)
+static void i915_gem_object_put_pages_dmabuf(struct drm_i915_gem_object *obj,
+					     struct sg_table *pages)
 {
-	dma_buf_unmap_attachment(obj->base.import_attach,
-				 obj->mm.pages, DMA_BIDIRECTIONAL);
+	dma_buf_unmap_attachment(obj->base.import_attach, pages,
+				 DMA_BIDIRECTIONAL);
 }
 
 static const struct drm_i915_gem_object_ops i915_gem_object_dmabuf_ops = {
drivers/gpu/drm/i915/i915_gem_fence.c +10 -8
index 398856160656..834d649496f3 100644
@@ -628,6 +628,7 @@ i915_gem_swizzle_page(struct page *page)
 /**
  * i915_gem_object_do_bit_17_swizzle - fixup bit 17 swizzling
  * @obj: i915 GEM buffer object
+ * @pages: the scattergather list of physical pages
  *
  * This function fixes up the swizzling in case any page frame number for this
  * object has changed in bit 17 since that state has been saved with
@@ -638,7 +639,8 @@ i915_gem_swizzle_page(struct page *page)
  * by swapping them out and back in again).
  */
 void
-i915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object *obj)
+i915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object *obj,
+				  struct sg_table *pages)
 {
 	struct sgt_iter sgt_iter;
 	struct page *page;
@@ -648,10 +650,9 @@ i915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object *obj)
 		return;
 
 	i = 0;
-	for_each_sgt_page(page, sgt_iter, obj->mm.pages) {
+	for_each_sgt_page(page, sgt_iter, pages) {
 		char new_bit_17 = page_to_phys(page) >> 17;
-		if ((new_bit_17 & 0x1) !=
-		    (test_bit(i, obj->bit_17) != 0)) {
+		if ((new_bit_17 & 0x1) != (test_bit(i, obj->bit_17) != 0)) {
 			i915_gem_swizzle_page(page);
 			set_page_dirty(page);
 		}
@@ -662,21 +663,22 @@ i915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object *obj)
 /**
  * i915_gem_object_save_bit_17_swizzle - save bit 17 swizzling
  * @obj: i915 GEM buffer object
+ * @pages: the scattergather list of physical pages
  *
  * This function saves the bit 17 of each page frame number so that swizzling
  * can be fixed up later on with i915_gem_object_do_bit_17_swizzle(). This must
  * be called before the backing storage can be unpinned.
  */
 void
-i915_gem_object_save_bit_17_swizzle(struct drm_i915_gem_object *obj)
+i915_gem_object_save_bit_17_swizzle(struct drm_i915_gem_object *obj,
+				    struct sg_table *pages)
 {
 	struct sgt_iter sgt_iter;
 	struct page *page;
-	int page_count = obj->base.size >> PAGE_SHIFT;
 	int i;
 
 	if (obj->bit_17 == NULL) {
-		obj->bit_17 = kcalloc(BITS_TO_LONGS(page_count),
+		obj->bit_17 = kcalloc(BITS_TO_LONGS(obj->base.size >> PAGE_SHIFT),
 				      sizeof(long), GFP_KERNEL);
 		if (obj->bit_17 == NULL) {
 			DRM_ERROR("Failed to allocate memory for bit 17 "
@@ -687,7 +689,7 @@ i915_gem_object_save_bit_17_swizzle(struct drm_i915_gem_object *obj)
 
 	i = 0;
 
-	for_each_sgt_page(page, sgt_iter, obj->mm.pages) {
+	for_each_sgt_page(page, sgt_iter, pages) {
 		if (page_to_phys(page) & (1 << 17))
 			__set_bit(i, obj->bit_17);
 		else
drivers/gpu/drm/i915/i915_gem_gtt.c +10 -9
index 3c711e0b8a3f..578026584f42 100644
@@ -2292,14 +2292,15 @@ void i915_gem_suspend_gtt_mappings(struct drm_device *dev)
 	i915_ggtt_flush(dev_priv);
 }
 
-int i915_gem_gtt_prepare_object(struct drm_i915_gem_object *obj)
+int i915_gem_gtt_prepare_pages(struct drm_i915_gem_object *obj,
+			       struct sg_table *pages)
 {
-	if (!dma_map_sg(&obj->base.dev->pdev->dev,
-			obj->mm.pages->sgl, obj->mm.pages->nents,
-			PCI_DMA_BIDIRECTIONAL))
-		return -ENOSPC;
+	if (dma_map_sg(&obj->base.dev->pdev->dev,
+		       pages->sgl, pages->nents,
+		       PCI_DMA_BIDIRECTIONAL))
+		return 0;
 
-	return 0;
+	return -ENOSPC;
 }
 
 static void gen8_set_pte(void __iomem *addr, gen8_pte_t pte)
@@ -2671,7 +2672,8 @@ static void ggtt_unbind_vma(struct i915_vma *vma)
 					 true);
 }
 
-void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj)
+void i915_gem_gtt_finish_pages(struct drm_i915_gem_object *obj,
+			       struct sg_table *pages)
 {
 	struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
 	struct device *kdev = &dev_priv->drm.pdev->dev;
@@ -2685,8 +2687,7 @@ void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj)
 		}
 	}
 
-	dma_unmap_sg(kdev, obj->mm.pages->sgl, obj->mm.pages->nents,
-		     PCI_DMA_BIDIRECTIONAL);
+	dma_unmap_sg(kdev, pages->sgl, pages->nents, PCI_DMA_BIDIRECTIONAL);
 }
 
 static void i915_gtt_color_adjust(struct drm_mm_node *node,
drivers/gpu/drm/i915/i915_gem_gtt.h +4 -2
index bd93fb8f99d2..c34945f21323 100644
@@ -629,8 +629,10 @@ void i915_check_and_clear_faults(struct drm_i915_private *dev_priv);
 void i915_gem_suspend_gtt_mappings(struct drm_device *dev);
 void i915_gem_restore_gtt_mappings(struct drm_device *dev);
 
-int __must_check i915_gem_gtt_prepare_object(struct drm_i915_gem_object *obj);
-void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj);
+int __must_check i915_gem_gtt_prepare_pages(struct drm_i915_gem_object *obj,
+					    struct sg_table *pages);
+void i915_gem_gtt_finish_pages(struct drm_i915_gem_object *obj,
+			       struct sg_table *pages);
 
 /* Flags used by pin/bind&friends. */
 #define PIN_NONBLOCK		BIT(0)
drivers/gpu/drm/i915/i915_gem_internal.c +11 -11
index 24eb347f9e0a..b0c56b311248 100644
@@ -38,7 +38,8 @@ static void internal_free_pages(struct sg_table *st)
 	kfree(st);
 }
 
-static int i915_gem_object_get_pages_internal(struct drm_i915_gem_object *obj)
+static struct sg_table *
+i915_gem_object_get_pages_internal(struct drm_i915_gem_object *obj)
 {
 	const unsigned int npages = obj->base.size / PAGE_SIZE;
 	struct sg_table *st;
@@ -49,11 +50,11 @@ static int i915_gem_object_get_pages_internal(struct drm_i915_gem_object *obj)
 
 	st = kmalloc(sizeof(*st), GFP_KERNEL);
 	if (!st)
-		return -ENOMEM;
+		return ERR_PTR(-ENOMEM);
 
 	if (sg_alloc_table(st, npages, GFP_KERNEL)) {
 		kfree(st);
-		return -ENOMEM;
+		return ERR_PTR(-ENOMEM);
 	}
 
 	sg = st->sgl;
@@ -95,12 +96,9 @@ static int i915_gem_object_get_pages_internal(struct drm_i915_gem_object *obj)
 	if (!swiotlb_nr_tbl())
 #endif
 		sg_mark_end(sg);
-	obj->mm.pages = st;
 
-	if (i915_gem_gtt_prepare_object(obj)) {
-		obj->mm.pages = NULL;
+	if (i915_gem_gtt_prepare_pages(obj, st))
 		goto err;
-	}
 
 	/* Mark the pages as dontneed whilst they are still pinned. As soon
 	 * as they are unpinned they are allowed to be reaped by the shrinker,
@@ -108,17 +106,19 @@ static int i915_gem_object_get_pages_internal(struct drm_i915_gem_object *obj)
 	 * object are only valid whilst active and pinned.
 	 */
 	obj->mm.madv = I915_MADV_DONTNEED;
-	return 0;
+	return st;
 
 err:
 	sg_mark_end(sg);
 	internal_free_pages(st);
-	return -ENOMEM;
+	return ERR_PTR(-ENOMEM);
 }
 
-static void i915_gem_object_put_pages_internal(struct drm_i915_gem_object *obj)
+static void i915_gem_object_put_pages_internal(struct drm_i915_gem_object *obj,
+					       struct sg_table *pages)
 {
-	internal_free_pages(obj->mm.pages);
+	i915_gem_gtt_finish_pages(obj, pages);
+	internal_free_pages(pages);
 
 	obj->mm.dirty = false;
 	obj->mm.madv = I915_MADV_WILLNEED;
drivers/gpu/drm/i915/i915_gem_shrinker.c +8 -3
index e0e6d68fe470..a8c3d37b95b9 100644
@@ -91,6 +91,13 @@ static bool can_release_pages(struct drm_i915_gem_object *obj)
 	return swap_available() || obj->mm.madv == I915_MADV_DONTNEED;
 }
 
+static bool unsafe_drop_pages(struct drm_i915_gem_object *obj)
+{
+	if (i915_gem_object_unbind(obj) == 0)
+		__i915_gem_object_put_pages(obj);
+	return !obj->mm.pages;
+}
+
 /**
  * i915_gem_shrink - Shrink buffer object caches
  * @dev_priv: i915 device
@@ -191,9 +198,7 @@ i915_gem_shrink(struct drm_i915_private *dev_priv,
 
 			i915_gem_object_get(obj);
 
-			/* For the unbound phase, this should be a no-op! */
-			i915_gem_object_unbind(obj);
-			if (__i915_gem_object_put_pages(obj) == 0)
+			if (unsafe_drop_pages(obj))
 				count += obj->base.size >> PAGE_SHIFT;
 
 			i915_gem_object_put(obj);
drivers/gpu/drm/i915/i915_gem_stolen.c +21 -22

89 lines not on this page · raw message

drivers/gpu/drm/i915/i915_gem_userptr.c +45 -43

196 lines not on this page · raw message

Signature · 6 lines
2.9.3

_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
maintainer Reviewed-by on v1 17/42
#54 lore raw

Re: [PATCH 17/42] drm/i915: Pass around sg_table to get_pages/put_pages backend

drivers/gpu/drm/i915/i915_drv.h __deprecated · line 2380

Quoted 8 lines · Chris Wilson
drivers/gpu/drm/i915/i915_drv.h +0 -0
@@ -2376,6 +2374,19 @@ __deprecated
  extern void drm_gem_object_unreference_unlocked(struct drm_gem_object *);


 static inline bool

drivers/gpu/drm/i915/i915_drv.h +4 -0
+i915_gem_object_is_dead(const struct drm_i915_gem_object *obj)
+{
+	return atomic_read(&obj->base.refcount.refcount) == 0;
+}

Seems like this ought to be in DRM? Todo item.

drivers/gpu/drm/i915/i915_drv.h __deprecated · line 2387

Quoted 7 lines
drivers/gpu/drm/i915/i915_drv.h +7 -0
+
+#if IS_ENABLED(CONFIG_LOCKDEP)
+#define lockdep_assert_held_unless(lock, cond) \
+	GEM_BUG_ON(!lockdep_is_held(lock) && !(cond))
+#else
+#define lockdep_assert_held_unless(lock, cond)
+#endif

Ugh.

drivers/gpu/drm/i915/i915_gem_userptr.c i915_gem_userptr_get_pages(struct drm_i915_gem_object *obj) · line 650

Quoted 6 lines
(quoted diff) +0 -0
@@ -646,21 +647,22 @@ i915_gem_userptr_get_pages(struct drm_i915_gem_object *obj)


  active = false;
  if (pinned < 0)

drivers/gpu/drm/i915/i915_gem_userptr.c +1 -1
-		ret = pinned, pinned = 0;
+		pages = ERR_PTR(pinned), pinned = 0;

Just convert to curly braces.

Quoted 14 lines

  else if (pinned < num_pages)

drivers/gpu/drm/i915/i915_gem_userptr.c +1 -1
-		ret = __i915_gem_userptr_get_pages_schedule(obj, &active);
+		pages = __i915_gem_userptr_get_pages_schedule(obj, &active);

  else

drivers/gpu/drm/i915/i915_gem_userptr.c +2 -2
-		ret = __i915_gem_userptr_set_pages(obj, pvec, num_pages);
-	if (ret) {
+		pages = __i915_gem_userptr_set_pages(obj, pvec, num_pages);
+	if (IS_ERR(pages)) {

  __i915_gem_userptr_set_active(obj, active); release_pages(pvec, pinned, 0);
  }
  drm_free_large(pvec);

(quoted diff) +1 -1
-	return ret;
+	return pages;
Quoted 2 lines

 }

Horrible patch to review, mix of different changes all in one.

Regars, Joonas

Signature · 11 lines
Joonas Lahtinen
Open Source Technology Center
Intel Corporation
-- 
Joonas Lahtinen
Open Source Technology Center
Intel Corporation
_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
#55 lore raw

Re: [PATCH 17/42] drm/i915: Pass around sg_table to get_pages/put_pages backend

Quoted 11 lines · Joonas Lahtinen

drivers/gpu/drm/i915/i915_drv.h __deprecated · line 2380

Quoted 8 lines · Chris Wilson
drivers/gpu/drm/i915/i915_drv.h +0 -0
@@ -2376,6 +2374,19 @@ __deprecated
  extern void drm_gem_object_unreference_unlocked(struct drm_gem_object *);


 static inline bool

drivers/gpu/drm/i915/i915_drv.h +4 -0
+i915_gem_object_is_dead(const struct drm_i915_gem_object *obj)
+{
+	return atomic_read(&obj->base.refcount.refcount) == 0;
+}

Seems like this ought to be in DRM? Todo item.

Quoted 9 lines

drivers/gpu/drm/i915/i915_drv.h __deprecated · line 2387

Quoted 7 lines
drivers/gpu/drm/i915/i915_drv.h +7 -0
+
+#if IS_ENABLED(CONFIG_LOCKDEP)
+#define lockdep_assert_held_unless(lock, cond) \
+	GEM_BUG_ON(!lockdep_is_held(lock) && !(cond))
+#else
+#define lockdep_assert_held_unless(lock, cond)
+#endif

Ugh.

These are temporaries to keep lockdep asserts working until removed later in the series (but I hope they illustrate the change in locking regime being prepared for).
-Chris

Signature · 5 lines
Chris Wilson, Intel Open Source Technology Centre
_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
#56 lore raw

Re: [PATCH 17/42] drm/i915: Pass around sg_table to get_pages/put_pages backend

In reply to Chris Wilson

[sending the draft reply until I get another look at it]

Quoted 24 lines · Chris Wilson

The plan is to move obj->pages out from under the struct_mutex into its own per-object lock. We need to prune any assumption of the struct_mutex from the get_pages/put_pages backends, and to make it easier we pass around the sg_table to operate on rather than indirectly via the obj.

---
  drivers/gpu/drm/i915/i915_drv.h          |  36 +++++--
  drivers/gpu/drm/i915/i915_gem.c          | 172 +++++++++++++++----------------
  drivers/gpu/drm/i915/i915_gem_dmabuf.c   |  20 ++--
  drivers/gpu/drm/i915/i915_gem_fence.c    |  18 ++--
  drivers/gpu/drm/i915/i915_gem_gtt.c      |  19 ++--
  drivers/gpu/drm/i915/i915_gem_gtt.h      |   6 +-
  drivers/gpu/drm/i915/i915_gem_internal.c |  22 ++--
  drivers/gpu/drm/i915/i915_gem_shrinker.c |  11 +-
  drivers/gpu/drm/i915/i915_gem_stolen.c   |  43 ++++----
  drivers/gpu/drm/i915/i915_gem_userptr.c  |  88 ++++++++--------
  10 files changed, 227 insertions(+), 208 deletions(-)
drivers/gpu/drm/i915/i915_drv.h +0 -0
index 3c22d49005fe..271e63c8f037 100644

drivers/gpu/drm/i915/i915_drv.h struct drm_i915_gem_object_ops { · line 2179

Quoted 7 lines
drivers/gpu/drm/i915/i915_drv.h +2 -2
@@ -2175,8 +2175,8 @@ struct drm_i915_gem_object_ops {
  	 * being released or under memory pressure (where we attempt to
  	 * reap pages for the shrinker).
  	 */
-	int (*get_pages)(struct drm_i915_gem_object *);
-	void (*put_pages)(struct drm_i915_gem_object *);
+	struct sg_table *(*get_pages)(struct drm_i915_gem_object *);
+	void (*put_pages)(struct drm_i915_gem_object *, struct sg_table *);

Idea is that put_pages vfunc does not need struct mutex? Or it acquires it on demand inside it, which means struct mutex will nest inside the mm.lock?

Would it be cleaner to have two vfuncs?

drivers/gpu/drm/i915/i915_drv.h struct drm_i915_gem_object { · line 2317

Quoted 9 lines
drivers/gpu/drm/i915/i915_drv.h +0 -2
  	int (*dmabuf_export)(struct drm_i915_gem_object *);
  	void (*release)(struct drm_i915_gem_object *);
@@ -2313,8 +2313,6 @@ struct drm_i915_gem_object {
  	struct i915_gem_userptr {
  		uintptr_t ptr;
  		unsigned read_only :1;
-		unsigned workers :4;
-#define I915_GEM_USERPTR_MAX_WORKERS 15

Why are you removing this? Perhaps I am misremembering that the reason for having it was to avoid queuing up to many userptr get_pages workers? It cannot happen any longer?

[this is as far as I got to the other day, then I started looking forward in patches to try to pick up the design but it became too much]

Regards,

Tvrtko

Quoted 996 lines
drivers/gpu/drm/i915/i915_drv.h +25 -5
  		struct i915_mm_struct *mm;
  		struct i915_mmu_object *mmu_object;
@@ -2376,6 +2374,19 @@ __deprecated
  extern void drm_gem_object_unreference_unlocked(struct drm_gem_object *);
 
  static inline bool
+i915_gem_object_is_dead(const struct drm_i915_gem_object *obj)
+{
+	return atomic_read(&obj->base.refcount.refcount) == 0;
+}
+
+#if IS_ENABLED(CONFIG_LOCKDEP)
+#define lockdep_assert_held_unless(lock, cond) \
+	GEM_BUG_ON(!lockdep_is_held(lock) && !(cond))
+#else
+#define lockdep_assert_held_unless(lock, cond)
+#endif
+
+static inline bool
  i915_gem_object_has_struct_page(const struct drm_i915_gem_object *obj)
  {
  	return obj->ops->flags & I915_GEM_OBJECT_HAS_STRUCT_PAGE;
@@ -3188,6 +3199,8 @@ dma_addr_t
  i915_gem_object_get_dma_address(struct drm_i915_gem_object *obj,
  				unsigned long n);
 
+void __i915_gem_object_set_pages(struct drm_i915_gem_object *obj,
+				 struct sg_table *pages);
  int __i915_gem_object_get_pages(struct drm_i915_gem_object *obj);
 
  static inline int __must_check
@@ -3203,7 +3216,8 @@ i915_gem_object_pin_pages(struct drm_i915_gem_object *obj)
  static inline void
  __i915_gem_object_pin_pages(struct drm_i915_gem_object *obj)
  {
-	lockdep_assert_held(&obj->base.dev->struct_mutex);
+	lockdep_assert_held_unless(&obj->base.dev->struct_mutex,
+				   i915_gem_object_is_dead(obj));
  	GEM_BUG_ON(!obj->mm.pages);
  	obj->mm.pages_pin_count++;
  }
@@ -3217,7 +3231,8 @@ i915_gem_object_has_pinned_pages(struct drm_i915_gem_object *obj)
  static inline void
  __i915_gem_object_unpin_pages(struct drm_i915_gem_object *obj)
  {
-	lockdep_assert_held(&obj->base.dev->struct_mutex);
+	lockdep_assert_held_unless(&obj->base.dev->struct_mutex,
+				   i915_gem_object_is_dead(obj));
  	GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));
  	GEM_BUG_ON(!obj->mm.pages);
  	obj->mm.pages_pin_count--;
@@ -3228,7 +3243,8 @@ static inline void i915_gem_object_unpin_pages(struct drm_i915_gem_object *obj)
  	__i915_gem_object_unpin_pages(obj);
  }
 
-int __i915_gem_object_put_pages(struct drm_i915_gem_object *obj);
+void __i915_gem_object_put_pages(struct drm_i915_gem_object *obj);
+void __i915_gem_object_invalidate(struct drm_i915_gem_object *obj);
 
  enum i915_map_type {
  	I915_MAP_WB = 0,
@@ -3451,8 +3467,10 @@ i915_vma_unpin_fence(struct i915_vma *vma)
  void i915_gem_restore_fences(struct drm_device *dev);
 
  void i915_gem_detect_bit_6_swizzle(struct drm_device *dev);
-void i915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object *obj);
-void i915_gem_object_save_bit_17_swizzle(struct drm_i915_gem_object *obj);
+void i915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object *obj,
+				       struct sg_table *pages);
+void i915_gem_object_save_bit_17_swizzle(struct drm_i915_gem_object *obj,
+					 struct sg_table *pages);
 
  /* i915_gem_context.c */
  int __must_check i915_gem_context_init(struct drm_device *dev);
drivers/gpu/drm/i915/i915_gem.c +83 -89
index df774ddf62ae..620fe8ac6477 100644
@@ -169,7 +169,7 @@ i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
  	return 0;
  }
 
-static int
+static struct sg_table *
  i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj)
  {
  	struct address_space *mapping = obj->base.filp->f_mapping;
@@ -179,7 +179,7 @@ i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj)
  	int i;
 
  	if (WARN_ON(i915_gem_object_needs_bit17_swizzle(obj)))
-		return -EINVAL;
+		return ERR_PTR(-EINVAL);
 
  	for (i = 0; i < obj->base.size / PAGE_SIZE; i++) {
  		struct page *page;
@@ -187,7 +187,7 @@ i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj)
 
  		page = shmem_read_mapping_page(mapping, i);
  		if (IS_ERR(page))
-			return PTR_ERR(page);
+			return ERR_CAST(page);
 
  		src = kmap_atomic(page);
  		memcpy(vaddr, src, PAGE_SIZE);
@@ -202,11 +202,11 @@ i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj)
 
  	st = kmalloc(sizeof(*st), GFP_KERNEL);
  	if (st == NULL)
-		return -ENOMEM;
+		return ERR_PTR(-ENOMEM);
 
  	if (sg_alloc_table(st, 1, GFP_KERNEL)) {
  		kfree(st);
-		return -ENOMEM;
+		return ERR_PTR(-ENOMEM);
  	}
 
  	sg = st->sgl;
@@ -216,28 +216,30 @@ i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj)
  	sg_dma_address(sg) = obj->phys_handle->busaddr;
  	sg_dma_len(sg) = obj->base.size;
 
-	obj->mm.pages = st;
-	return 0;
+	return st;
  }
 
  static void
-i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj)
+__i915_gem_object_release_shmem(struct drm_i915_gem_object *obj)
  {
-	int ret;
-
  	GEM_BUG_ON(obj->mm.madv == __I915_MADV_PURGED);
 
-	ret = i915_gem_object_set_to_cpu_domain(obj, true);
-	if (WARN_ON(ret)) {
-		/* In the event of a disaster, abandon all caches and
-		 * hope for the best.
-		 */
-		obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
-	}
-
  	if (obj->mm.madv == I915_MADV_DONTNEED)
  		obj->mm.dirty = false;
 
+	if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0)
+		i915_gem_clflush_object(obj, false);
+
+	obj->base.read_domains = I915_GEM_DOMAIN_CPU;
+	obj->base.write_domain = I915_GEM_DOMAIN_CPU;
+}
+
+static void
+i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj,
+			       struct sg_table *pages)
+{
+	__i915_gem_object_release_shmem(obj);
+
  	if (obj->mm.dirty) {
  		struct address_space *mapping = obj->base.filp->f_mapping;
  		char *vaddr = obj->phys_handle->vaddr;
@@ -265,8 +267,8 @@ i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj)
  		obj->mm.dirty = false;
  	}
 
-	sg_free_table(obj->mm.pages);
-	kfree(obj->mm.pages);
+	sg_free_table(pages);
+	kfree(pages);
  }
 
  static void
@@ -517,9 +519,9 @@ i915_gem_object_attach_phys(struct drm_i915_gem_object *obj,
  	if (ret)
  		return ret;
 
-	ret = __i915_gem_object_put_pages(obj);
-	if (ret)
-		return ret;
+	__i915_gem_object_put_pages(obj);
+	if (obj->mm.pages)
+		return -EBUSY;
 
  	/* create a new object */
  	phys = drm_pci_alloc(obj->base.dev, obj->base.size, align);
@@ -535,7 +537,7 @@ i915_gem_object_attach_phys(struct drm_i915_gem_object *obj,
  static int
  i915_gem_phys_pwrite(struct drm_i915_gem_object *obj,
  		     struct drm_i915_gem_pwrite *args,
-		     struct drm_file *file_priv)
+		     struct drm_file *file)
  {
  	struct drm_device *dev = obj->base.dev;
  	void *vaddr = obj->phys_handle->vaddr + args->offset;
@@ -551,7 +553,7 @@ i915_gem_phys_pwrite(struct drm_i915_gem_object *obj,
  				   I915_WAIT_LOCKED |
  				   I915_WAIT_ALL,
  				   MAX_SCHEDULE_TIMEOUT,
-				   to_rps_client(file_priv));
+				   to_rps_client(file));
  	if (ret)
  		return ret;
 
@@ -2225,8 +2227,7 @@ i915_gem_object_truncate(struct drm_i915_gem_object *obj)
  }
 
  /* Try to discard unwanted pages */
-static void
-i915_gem_object_invalidate(struct drm_i915_gem_object *obj)
+void __i915_gem_object_invalidate(struct drm_i915_gem_object *obj)
  {
  	struct address_space *mapping;
 
@@ -2245,32 +2246,20 @@ i915_gem_object_invalidate(struct drm_i915_gem_object *obj)
  }
 
  static void
-i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
+i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj,
+			      struct sg_table *pages)
  {
  	struct sgt_iter sgt_iter;
  	struct page *page;
-	int ret;
 
-	GEM_BUG_ON(obj->mm.madv == __I915_MADV_PURGED);
-
-	ret = i915_gem_object_set_to_cpu_domain(obj, true);
-	if (WARN_ON(ret)) {
-		/* In the event of a disaster, abandon all caches and
-		 * hope for the best.
-		 */
-		i915_gem_clflush_object(obj, true);
-		obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
-	}
+	__i915_gem_object_release_shmem(obj);
 
-	i915_gem_gtt_finish_object(obj);
+	i915_gem_gtt_finish_pages(obj, pages);
 
  	if (i915_gem_object_needs_bit17_swizzle(obj))
-		i915_gem_object_save_bit_17_swizzle(obj);
-
-	if (obj->mm.madv == I915_MADV_DONTNEED)
-		obj->mm.dirty = false;
+		i915_gem_object_save_bit_17_swizzle(obj, pages);
 
-	for_each_sgt_page(page, sgt_iter, obj->mm.pages) {
+	for_each_sgt_page(page, sgt_iter, pages) {
  		if (obj->mm.dirty)
  			set_page_dirty(page);
 
@@ -2281,8 +2270,8 @@ i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
  	}
  	obj->mm.dirty = false;
 
-	sg_free_table(obj->mm.pages);
-	kfree(obj->mm.pages);
+	sg_free_table(pages);
+	kfree(pages);
  }
 
  static void __i915_gem_object_reset_page_iter(struct drm_i915_gem_object *obj)
@@ -2294,24 +2283,22 @@ static void __i915_gem_object_reset_page_iter(struct drm_i915_gem_object *obj)
  		radix_tree_delete(&obj->mm.get_page.radix, iter.index);
  }
 
-int __i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
+void __i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
  {
-	const struct drm_i915_gem_object_ops *ops = obj->ops;
+	struct sg_table *pages;
 
  	lockdep_assert_held(&obj->base.dev->struct_mutex);
 
-	if (!obj->mm.pages)
-		return 0;
-
  	if (i915_gem_object_has_pinned_pages(obj))
-		return -EBUSY;
+		return;
 
  	GEM_BUG_ON(obj->bind_count);
 
  	/* ->put_pages might need to allocate memory for the bit17 swizzle
  	 * array, hence protect them from being reaped by removing them from gtt
  	 * lists early. */
-	list_del(&obj->global_list);
+	pages = fetch_and_zero(&obj->mm.pages);
+	GEM_BUG_ON(!pages);
 
  	if (obj->mm.mapping) {
  		void *ptr;
@@ -2327,15 +2314,10 @@ int __i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
 
  	__i915_gem_object_reset_page_iter(obj);
 
-	ops->put_pages(obj);
-	obj->mm.pages = NULL;
-
-	i915_gem_object_invalidate(obj);
-
-	return 0;
+	obj->ops->put_pages(obj, pages);
  }
 
-static int
+static struct sg_table *
  i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
  {
  	struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
@@ -2353,17 +2335,17 @@ i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
  	 * wasn't in the GTT, there shouldn't be any way it could have been in
  	 * a GPU cache
  	 */
-	BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
-	BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
+	GEM_BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
+	GEM_BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
 
  	st = kmalloc(sizeof(*st), GFP_KERNEL);
  	if (st == NULL)
-		return -ENOMEM;
+		return ERR_PTR(-ENOMEM);
 
  	page_count = obj->base.size / PAGE_SIZE;
  	if (sg_alloc_table(st, page_count, GFP_KERNEL)) {
  		kfree(st);
-		return -ENOMEM;
+		return ERR_PTR(-ENOMEM);
  	}
 
  	/* Get the list of pages out of our struct file.  They'll be pinned
@@ -2423,20 +2405,19 @@ i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
  	if (!swiotlb_nr_tbl())
  #endif
  		sg_mark_end(sg);
-	obj->mm.pages = st;
 
-	ret = i915_gem_gtt_prepare_object(obj);
+	ret = i915_gem_gtt_prepare_pages(obj, st);
  	if (ret)
  		goto err_pages;
 
  	if (i915_gem_object_needs_bit17_swizzle(obj))
-		i915_gem_object_do_bit_17_swizzle(obj);
+		i915_gem_object_do_bit_17_swizzle(obj, st);
 
  	if (i915_gem_object_is_tiled(obj) &&
  	    dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES)
  		__i915_gem_object_pin_pages(obj);
 
-	return 0;
+	return st;
 
  err_pages:
  	sg_mark_end(sg);
@@ -2456,7 +2437,35 @@ err_pages:
  	if (ret == -ENOSPC)
  		ret = -ENOMEM;
 
-	return ret;
+	return ERR_PTR(ret);
+}
+
+void __i915_gem_object_set_pages(struct drm_i915_gem_object *obj,
+				 struct sg_table *pages)
+{
+	lockdep_assert_held(&obj->base.dev->struct_mutex);
+
+	obj->mm.get_page.sg_pos = pages->sgl;
+	obj->mm.get_page.sg_idx = 0;
+
+	obj->mm.pages = pages;
+}
+
+static int ____i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
+{
+	struct sg_table *pages;
+
+	if (unlikely(obj->mm.madv != I915_MADV_WILLNEED)) {
+		DRM_DEBUG("Attempting to obtain a purgeable object\n");
+		return -EFAULT;
+	}
+
+	pages = obj->ops->get_pages(obj);
+	if (unlikely(IS_ERR(pages)))
+		return PTR_ERR(pages);
+
+	__i915_gem_object_set_pages(obj, pages);
+	return 0;
  }
 
  /* Ensure that the associated pages are gathered from the backing storage
@@ -2468,33 +2477,18 @@ err_pages:
   */
  int __i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
  {
-	struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
-	const struct drm_i915_gem_object_ops *ops = obj->ops;
-	int ret;
+	int err;
 
  	lockdep_assert_held(&obj->base.dev->struct_mutex);
 
  	if (obj->mm.pages)
  		return 0;
 
-	if (obj->mm.madv != I915_MADV_WILLNEED) {
-		DRM_DEBUG("Attempting to obtain a purgeable object\n");
-		__i915_gem_object_unpin_pages(obj);
-		return -EFAULT;
-	}
-
-	ret = ops->get_pages(obj);
-	if (ret) {
+	err = ____i915_gem_object_get_pages(obj);
+	if (err)
  		__i915_gem_object_unpin_pages(obj);
-		return ret;
-	}
 
-	list_add_tail(&obj->global_list, &dev_priv->mm.unbound_list);
-
-	obj->mm.get_page.sg_pos = obj->mm.pages->sgl;
-	obj->mm.get_page.sg_idx = 0;
-
-	return 0;
+	return err;
  }
 
  /* The 'mapping' part of i915_gem_object_pin_map() below */
drivers/gpu/drm/i915/i915_gem_dmabuf.c +8 -12
index 10441dc72e73..2abd524aba14 100644
@@ -289,22 +289,18 @@ struct dma_buf *i915_gem_prime_export(struct drm_device *dev,
  	return dma_buf;
  }
 
-static int i915_gem_object_get_pages_dmabuf(struct drm_i915_gem_object *obj)
+static struct sg_table *
+i915_gem_object_get_pages_dmabuf(struct drm_i915_gem_object *obj)
  {
-	struct sg_table *sg;
-
-	sg = dma_buf_map_attachment(obj->base.import_attach, DMA_BIDIRECTIONAL);
-	if (IS_ERR(sg))
-		return PTR_ERR(sg);
-
-	obj->mm.pages = sg;
-	return 0;
+	return dma_buf_map_attachment(obj->base.import_attach,
+				      DMA_BIDIRECTIONAL);
  }
 
-static void i915_gem_object_put_pages_dmabuf(struct drm_i915_gem_object *obj)
+static void i915_gem_object_put_pages_dmabuf(struct drm_i915_gem_object *obj,
+					     struct sg_table *pages)
  {
-	dma_buf_unmap_attachment(obj->base.import_attach,
-				 obj->mm.pages, DMA_BIDIRECTIONAL);
+	dma_buf_unmap_attachment(obj->base.import_attach, pages,
+				 DMA_BIDIRECTIONAL);
  }
 
  static const struct drm_i915_gem_object_ops i915_gem_object_dmabuf_ops = {
drivers/gpu/drm/i915/i915_gem_fence.c +10 -8
index 398856160656..834d649496f3 100644
@@ -628,6 +628,7 @@ i915_gem_swizzle_page(struct page *page)
  /**
   * i915_gem_object_do_bit_17_swizzle - fixup bit 17 swizzling
   * @obj: i915 GEM buffer object
+ * @pages: the scattergather list of physical pages
   *
   * This function fixes up the swizzling in case any page frame number for this
   * object has changed in bit 17 since that state has been saved with
@@ -638,7 +639,8 @@ i915_gem_swizzle_page(struct page *page)
   * by swapping them out and back in again).
   */
  void
-i915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object *obj)
+i915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object *obj,
+				  struct sg_table *pages)
  {
  	struct sgt_iter sgt_iter;
  	struct page *page;
@@ -648,10 +650,9 @@ i915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object *obj)
  		return;
 
  	i = 0;
-	for_each_sgt_page(page, sgt_iter, obj->mm.pages) {
+	for_each_sgt_page(page, sgt_iter, pages) {
  		char new_bit_17 = page_to_phys(page) >> 17;
-		if ((new_bit_17 & 0x1) !=
-		    (test_bit(i, obj->bit_17) != 0)) {
+		if ((new_bit_17 & 0x1) != (test_bit(i, obj->bit_17) != 0)) {
  			i915_gem_swizzle_page(page);
  			set_page_dirty(page);
  		}
@@ -662,21 +663,22 @@ i915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object *obj)
  /**
   * i915_gem_object_save_bit_17_swizzle - save bit 17 swizzling
   * @obj: i915 GEM buffer object
+ * @pages: the scattergather list of physical pages
   *
   * This function saves the bit 17 of each page frame number so that swizzling
   * can be fixed up later on with i915_gem_object_do_bit_17_swizzle(). This must
   * be called before the backing storage can be unpinned.
   */
  void
-i915_gem_object_save_bit_17_swizzle(struct drm_i915_gem_object *obj)
+i915_gem_object_save_bit_17_swizzle(struct drm_i915_gem_object *obj,
+				    struct sg_table *pages)
  {
  	struct sgt_iter sgt_iter;
  	struct page *page;
-	int page_count = obj->base.size >> PAGE_SHIFT;
  	int i;
 
  	if (obj->bit_17 == NULL) {
-		obj->bit_17 = kcalloc(BITS_TO_LONGS(page_count),
+		obj->bit_17 = kcalloc(BITS_TO_LONGS(obj->base.size >> PAGE_SHIFT),
  				      sizeof(long), GFP_KERNEL);
  		if (obj->bit_17 == NULL) {
  			DRM_ERROR("Failed to allocate memory for bit 17 "
@@ -687,7 +689,7 @@ i915_gem_object_save_bit_17_swizzle(struct drm_i915_gem_object *obj)
 
  	i = 0;
 
-	for_each_sgt_page(page, sgt_iter, obj->mm.pages) {
+	for_each_sgt_page(page, sgt_iter, pages) {
  		if (page_to_phys(page) & (1 << 17))
  			__set_bit(i, obj->bit_17);
  		else
drivers/gpu/drm/i915/i915_gem_gtt.c +10 -9
index 3c711e0b8a3f..578026584f42 100644
@@ -2292,14 +2292,15 @@ void i915_gem_suspend_gtt_mappings(struct drm_device *dev)
  	i915_ggtt_flush(dev_priv);
  }
 
-int i915_gem_gtt_prepare_object(struct drm_i915_gem_object *obj)
+int i915_gem_gtt_prepare_pages(struct drm_i915_gem_object *obj,
+			       struct sg_table *pages)
  {
-	if (!dma_map_sg(&obj->base.dev->pdev->dev,
-			obj->mm.pages->sgl, obj->mm.pages->nents,
-			PCI_DMA_BIDIRECTIONAL))
-		return -ENOSPC;
+	if (dma_map_sg(&obj->base.dev->pdev->dev,
+		       pages->sgl, pages->nents,
+		       PCI_DMA_BIDIRECTIONAL))
+		return 0;
 
-	return 0;
+	return -ENOSPC;
  }
 
  static void gen8_set_pte(void __iomem *addr, gen8_pte_t pte)
@@ -2671,7 +2672,8 @@ static void ggtt_unbind_vma(struct i915_vma *vma)
  					 true);
  }
 
-void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj)
+void i915_gem_gtt_finish_pages(struct drm_i915_gem_object *obj,
+			       struct sg_table *pages)
  {
  	struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
  	struct device *kdev = &dev_priv->drm.pdev->dev;
@@ -2685,8 +2687,7 @@ void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj)
  		}
  	}
 
-	dma_unmap_sg(kdev, obj->mm.pages->sgl, obj->mm.pages->nents,
-		     PCI_DMA_BIDIRECTIONAL);
+	dma_unmap_sg(kdev, pages->sgl, pages->nents, PCI_DMA_BIDIRECTIONAL);
  }
 
  static void i915_gtt_color_adjust(struct drm_mm_node *node,
drivers/gpu/drm/i915/i915_gem_gtt.h +4 -2
index bd93fb8f99d2..c34945f21323 100644
@@ -629,8 +629,10 @@ void i915_check_and_clear_faults(struct drm_i915_private *dev_priv);
  void i915_gem_suspend_gtt_mappings(struct drm_device *dev);
  void i915_gem_restore_gtt_mappings(struct drm_device *dev);
 
-int __must_check i915_gem_gtt_prepare_object(struct drm_i915_gem_object *obj);
-void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj);
+int __must_check i915_gem_gtt_prepare_pages(struct drm_i915_gem_object *obj,
+					    struct sg_table *pages);
+void i915_gem_gtt_finish_pages(struct drm_i915_gem_object *obj,
+			       struct sg_table *pages);
 
  /* Flags used by pin/bind&friends. */
  #define PIN_NONBLOCK		BIT(0)
drivers/gpu/drm/i915/i915_gem_internal.c +11 -11
index 24eb347f9e0a..b0c56b311248 100644
@@ -38,7 +38,8 @@ static void internal_free_pages(struct sg_table *st)
  	kfree(st);
  }
 
-static int i915_gem_object_get_pages_internal(struct drm_i915_gem_object *obj)
+static struct sg_table *
+i915_gem_object_get_pages_internal(struct drm_i915_gem_object *obj)
  {
  	const unsigned int npages = obj->base.size / PAGE_SIZE;
  	struct sg_table *st;
@@ -49,11 +50,11 @@ static int i915_gem_object_get_pages_internal(struct drm_i915_gem_object *obj)
 
  	st = kmalloc(sizeof(*st), GFP_KERNEL);
  	if (!st)
-		return -ENOMEM;
+		return ERR_PTR(-ENOMEM);
 
  	if (sg_alloc_table(st, npages, GFP_KERNEL)) {
  		kfree(st);
-		return -ENOMEM;
+		return ERR_PTR(-ENOMEM);
  	}
 
  	sg = st->sgl;
@@ -95,12 +96,9 @@ static int i915_gem_object_get_pages_internal(struct drm_i915_gem_object *obj)
  	if (!swiotlb_nr_tbl())
  #endif
  		sg_mark_end(sg);
-	obj->mm.pages = st;
 
-	if (i915_gem_gtt_prepare_object(obj)) {
-		obj->mm.pages = NULL;
+	if (i915_gem_gtt_prepare_pages(obj, st))
  		goto err;
-	}
 
  	/* Mark the pages as dontneed whilst they are still pinned. As soon
  	 * as they are unpinned they are allowed to be reaped by the shrinker,
@@ -108,17 +106,19 @@ static int i915_gem_object_get_pages_internal(struct drm_i915_gem_object *obj)
  	 * object are only valid whilst active and pinned.
  	 */
  	obj->mm.madv = I915_MADV_DONTNEED;
-	return 0;
+	return st;
 
  err:
  	sg_mark_end(sg);
  	internal_free_pages(st);
-	return -ENOMEM;
+	return ERR_PTR(-ENOMEM);
  }
 
-static void i915_gem_object_put_pages_internal(struct drm_i915_gem_object *obj)
+static void i915_gem_object_put_pages_internal(struct drm_i915_gem_object *obj,
+					       struct sg_table *pages)
  {
-	internal_free_pages(obj->mm.pages);
+	i915_gem_gtt_finish_pages(obj, pages);
+	internal_free_pages(pages);
 
  	obj->mm.dirty = false;
  	obj->mm.madv = I915_MADV_WILLNEED;
drivers/gpu/drm/i915/i915_gem_shrinker.c +8 -3
index e0e6d68fe470..a8c3d37b95b9 100644
@@ -91,6 +91,13 @@ static bool can_release_pages(struct drm_i915_gem_object *obj)
  	return swap_available() || obj->mm.madv == I915_MADV_DONTNEED;
  }
 
+static bool unsafe_drop_pages(struct drm_i915_gem_object *obj)
+{
+	if (i915_gem_object_unbind(obj) == 0)
+		__i915_gem_object_put_pages(obj);
+	return !obj->mm.pages;
+}
+
  /**
   * i915_gem_shrink - Shrink buffer object caches
   * @dev_priv: i915 device
@@ -191,9 +198,7 @@ i915_gem_shrink(struct drm_i915_private *dev_priv,
 
  			i915_gem_object_get(obj);
 
-			/* For the unbound phase, this should be a no-op! */
-			i915_gem_object_unbind(obj);
-			if (__i915_gem_object_put_pages(obj) == 0)
+			if (unsafe_drop_pages(obj))
  				count += obj->base.size >> PAGE_SHIFT;
 
  			i915_gem_object_put(obj);
drivers/gpu/drm/i915/i915_gem_stolen.c +21 -22
index 6d139d6b4609..4166eafa25dd 100644
@@ -545,17 +545,20 @@ i915_pages_create_for_stolen(struct drm_device *dev,
  	return st;
  }
 
-static int i915_gem_object_get_pages_stolen(struct drm_i915_gem_object *obj)
+static struct sg_table *
+i915_gem_object_get_pages_stolen(struct drm_i915_gem_object *obj)
  {
-	BUG();
-	return -EINVAL;
+	return i915_pages_create_for_stolen(obj->base.dev,
+					    obj->stolen->start,
+					    obj->stolen->size);
  }
 
-static void i915_gem_object_put_pages_stolen(struct drm_i915_gem_object *obj)
+static void i915_gem_object_put_pages_stolen(struct drm_i915_gem_object *obj,
+					     struct sg_table *pages)
  {
  	/* Should only be called during free */
-	sg_free_table(obj->mm.pages);
-	kfree(obj->mm.pages);
+	sg_free_table(pages);
+	kfree(pages);
  }
 
  static void
@@ -590,21 +593,13 @@ _i915_gem_object_create_stolen(struct drm_device *dev,
  	drm_gem_private_object_init(dev, &obj->base, stolen->size);
  	i915_gem_object_init(obj, &i915_gem_object_stolen_ops);
 
-	obj->mm.pages = i915_pages_create_for_stolen(dev,
-						     stolen->start,
-						     stolen->size);
-	if (!obj->mm.pages)
-		goto cleanup;
-
-	obj->mm.get_page.sg_pos = obj->mm.pages->sgl;
-	obj->mm.get_page.sg_idx = 0;
-
-	__i915_gem_object_pin_pages(obj);
  	obj->stolen = stolen;
-
  	obj->base.read_domains = I915_GEM_DOMAIN_CPU | I915_GEM_DOMAIN_GTT;
  	obj->cache_level = HAS_LLC(dev) ? I915_CACHE_LLC : I915_CACHE_NONE;
 
+	if (i915_gem_object_pin_pages(obj))
+		goto cleanup;
+
  	return obj;
 
  cleanup:
@@ -699,10 +694,14 @@ i915_gem_object_create_stolen_for_preallocated(struct drm_device *dev,
  	if (gtt_offset == I915_GTT_OFFSET_NONE)
  		return obj;
 
+	ret = i915_gem_object_pin_pages(obj);
+	if (ret)
+		goto err;
+
  	vma = i915_gem_obj_lookup_or_create_vma(obj, &ggtt->base, NULL);
  	if (IS_ERR(vma)) {
  		ret = PTR_ERR(vma);
-		goto err;
+		goto err_pages;
  	}
 
  	/* To simplify the initialisation sequence between KMS and GTT,
@@ -716,20 +715,20 @@ i915_gem_object_create_stolen_for_preallocated(struct drm_device *dev,
  	ret = drm_mm_reserve_node(&ggtt->base.mm, &vma->node);
  	if (ret) {
  		DRM_DEBUG_KMS("failed to allocate stolen GTT space\n");
-		goto err;
+		goto err_pages;
  	}
 
  	vma->pages = obj->mm.pages;
  	vma->flags |= I915_VMA_GLOBAL_BIND;
  	__i915_vma_set_map_and_fenceable(vma);
  	list_move_tail(&vma->vm_link, &ggtt->base.inactive_list);
+	list_move_tail(&obj->global_list, &dev_priv->mm.bound_list);
  	obj->bind_count++;
 
-	list_add_tail(&obj->global_list, &dev_priv->mm.bound_list);
-	__i915_gem_object_pin_pages(obj);
-
  	return obj;
 
+err_pages:
+	i915_gem_object_unpin_pages(obj);
  err:
  	i915_gem_object_put(obj);
  	return NULL;
drivers/gpu/drm/i915/i915_gem_userptr.c +43 -41
index 0fdd1d6723d1..e97c7b589439 100644
@@ -73,11 +73,14 @@ static void cancel_userptr(struct work_struct *work)
  	/* Cancel any active worker and force us to re-evaluate gup */
  	obj->userptr.work = NULL;
 
-	if (obj->mm.pages) {
-		/* We are inside a kthread context and can't be interrupted */
-		WARN_ON(i915_gem_object_unbind(obj));
-		WARN_ON(__i915_gem_object_put_pages(obj));
-	}
+	/* We are inside a kthread context and can't be interrupted */
+	if (i915_gem_object_unbind(obj) == 0)
+		__i915_gem_object_put_pages(obj);
+	WARN_ONCE(obj->mm.pages,
+		  "Failed to release pages: bind_count=%d, pages_pin_count=%d, pin_display=%d\n",
+		  obj->bind_count,
+		  obj->mm.pages_pin_count,
+		  obj->pin_display);
 
  	i915_gem_object_put(obj);
  	mutex_unlock(&dev->struct_mutex);
@@ -426,24 +429,25 @@ err:
  	return ret;
  }
 
-static int
+static struct sg_table *
  __i915_gem_userptr_set_pages(struct drm_i915_gem_object *obj,
  			     struct page **pvec, int num_pages)
  {
+	struct sg_table *pages;
  	int ret;
 
-	ret = st_set_pages(&obj->mm.pages, pvec, num_pages);
+	ret = st_set_pages(&pages, pvec, num_pages);
  	if (ret)
-		return ret;
+		return ERR_PTR(ret);
 
-	ret = i915_gem_gtt_prepare_object(obj);
+	ret = i915_gem_gtt_prepare_pages(obj, pages);
  	if (ret) {
-		sg_free_table(obj->mm.pages);
-		kfree(obj->mm.pages);
-		obj->mm.pages = NULL;
+		sg_free_table(pages);
+		kfree(pages);
+		return ERR_PTR(ret);
  	}
 
-	return ret;
+	return pages;
  }
 
  static int
@@ -521,20 +525,20 @@ __i915_gem_userptr_get_pages_worker(struct work_struct *_work)
 
  	mutex_lock(&dev->struct_mutex);
  	if (obj->userptr.work == &work->work) {
+		struct sg_table *pages = ERR_PTR(ret);
+
  		if (pinned == npages) {
-			ret = __i915_gem_userptr_set_pages(obj, pvec, npages);
-			if (ret == 0) {
-				list_add_tail(&obj->global_list,
-					      &to_i915(dev)->mm.unbound_list);
-				obj->mm.get_page.sg_pos = obj->mm.pages->sgl;
-				obj->mm.get_page.sg_idx = 0;
+			pages = __i915_gem_userptr_set_pages(obj, pvec, npages);
+			if (!IS_ERR(pages)) {
+				__i915_gem_object_set_pages(obj, pages);
  				pinned = 0;
+				pages = NULL;
  			}
  		}
-		obj->userptr.work = ERR_PTR(ret);
+
+		obj->userptr.work = ERR_CAST(pages);
  	}
 
-	obj->userptr.workers--;
  	i915_gem_object_put(obj);
  	mutex_unlock(&dev->struct_mutex);
 
@@ -545,7 +549,7 @@ __i915_gem_userptr_get_pages_worker(struct work_struct *_work)
  	kfree(work);
  }
 
-static int
+static struct sg_table *
  __i915_gem_userptr_get_pages_schedule(struct drm_i915_gem_object *obj,
  				      bool *active)
  {
@@ -570,15 +574,11 @@ __i915_gem_userptr_get_pages_schedule(struct drm_i915_gem_object *obj,
  	 * that error back to this function through
  	 * obj->userptr.work = ERR_PTR.
  	 */
-	if (obj->userptr.workers >= I915_GEM_USERPTR_MAX_WORKERS)
-		return -EAGAIN;
-
  	work = kmalloc(sizeof(*work), GFP_KERNEL);
  	if (work == NULL)
-		return -ENOMEM;
+		return ERR_PTR(-ENOMEM);
 
  	obj->userptr.work = &work->work;
-	obj->userptr.workers++;
 
  	work->obj = i915_gem_object_get(obj);
 
@@ -589,14 +589,15 @@ __i915_gem_userptr_get_pages_schedule(struct drm_i915_gem_object *obj,
  	schedule_work(&work->work);
 
  	*active = true;
-	return -EAGAIN;
+	return ERR_PTR(-EAGAIN);
  }
 
-static int
+static struct sg_table *
  i915_gem_userptr_get_pages(struct drm_i915_gem_object *obj)
  {
  	const int num_pages = obj->base.size >> PAGE_SHIFT;
  	struct page **pvec;
+	struct sg_table *pages;
  	int pinned, ret;
  	bool active;
 
@@ -620,15 +621,15 @@ i915_gem_userptr_get_pages(struct drm_i915_gem_object *obj)
  	if (obj->userptr.work) {
  		/* active flag should still be held for the pending work */
  		if (IS_ERR(obj->userptr.work))
-			return PTR_ERR(obj->userptr.work);
+			return ERR_CAST(obj->userptr.work);
  		else
-			return -EAGAIN;
+			return ERR_PTR(-EAGAIN);
  	}
 
  	/* Let the mmu-notifier know that we have begun and need cancellation */
  	ret = __i915_gem_userptr_set_active(obj, true);
  	if (ret)
-		return ret;
+		return ERR_PTR(ret);
 
  	pvec = NULL;
  	pinned = 0;
@@ -637,7 +638,7 @@ i915_gem_userptr_get_pages(struct drm_i915_gem_object *obj)
  				      GFP_TEMPORARY);
  		if (pvec == NULL) {
  			__i915_gem_userptr_set_active(obj, false);
-			return -ENOMEM;
+			return ERR_PTR(-ENOMEM);
  		}
 
  		pinned = __get_user_pages_fast(obj->userptr.ptr, num_pages,
@@ -646,21 +647,22 @@ i915_gem_userptr_get_pages(struct drm_i915_gem_object *obj)
 
  	active = false;
  	if (pinned < 0)
-		ret = pinned, pinned = 0;
+		pages = ERR_PTR(pinned), pinned = 0;
  	else if (pinned < num_pages)
-		ret = __i915_gem_userptr_get_pages_schedule(obj, &active);
+		pages = __i915_gem_userptr_get_pages_schedule(obj, &active);
  	else
-		ret = __i915_gem_userptr_set_pages(obj, pvec, num_pages);
-	if (ret) {
+		pages = __i915_gem_userptr_set_pages(obj, pvec, num_pages);
+	if (IS_ERR(pages)) {
  		__i915_gem_userptr_set_active(obj, active);
  		release_pages(pvec, pinned, 0);
  	}
  	drm_free_large(pvec);
-	return ret;
+	return pages;
  }
 
  static void
-i915_gem_userptr_put_pages(struct drm_i915_gem_object *obj)
+i915_gem_userptr_put_pages(struct drm_i915_gem_object *obj,
+			   struct sg_table *pages)
  {
  	struct sgt_iter sgt_iter;
  	struct page *page;
@@ -671,9 +673,9 @@ i915_gem_userptr_put_pages(struct drm_i915_gem_object *obj)
  	if (obj->mm.madv != I915_MADV_WILLNEED)
  		obj->mm.dirty = false;
 
-	i915_gem_gtt_finish_object(obj);
+	i915_gem_gtt_finish_pages(obj, pages);
 
-	for_each_sgt_page(page, sgt_iter, obj->mm.pages) {
+	for_each_sgt_page(page, sgt_iter, pages) {
  		if (obj->mm.dirty)
  			set_page_dirty(page);
 

drivers/gpu/drm/i915/i915_gem_userptr.c i915_gem_userptr_put_pages(struct drm_i915_gem_object *obj) · line 691

Quoted 10 lines
drivers/gpu/drm/i915/i915_gem_userptr.c +2 -2
@@ -682,8 +684,8 @@ i915_gem_userptr_put_pages(struct drm_i915_gem_object *obj)
  	}
  	obj->mm.dirty = false;
 
-	sg_free_table(obj->mm.pages);
-	kfree(obj->mm.pages);
+	sg_free_table(pages);
+	kfree(pages);
  }
 
  static void

_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx

#57 lore raw

Re: [PATCH 17/42] drm/i915: Pass around sg_table to get_pages/put_pages backend

Quoted 8 lines · Tvrtko Ursulin

diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h index 3c22d49005fe..271e63c8f037 100644
--- a/drivers/gpu/drm/i915/i915_drv.h
+++ b/drivers/gpu/drm/i915/i915_drv.h
@@ -2175,8 +2175,8 @@ struct drm_i915_gem_object_ops {
* being released or under memory pressure (where we attempt to
* reap pages for the shrinker).
*/

Quoted 8 lines
 -       int (*get_pages)(struct drm_i915_gem_object *);
 -       void (*put_pages)(struct drm_i915_gem_object *);
 +       struct sg_table *(*get_pages)(struct drm_i915_gem_object *);
 +       void (*put_pages)(struct drm_i915_gem_object *, struct sg_table *);

Idea is that put_pages vfunc does not need struct mutex?  Or it acquires it on demand inside it, which means struct mutex will nest inside the mm.lock?

Right, it loses the mutex within get/put pages and I am no longer concerned about the number of workers. Though that is mainly because of another patch to improve execbuf + userptr workloads.
-Chris

Signature · 5 lines
Chris Wilson, Intel Open Source Technology Centre
_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
#58 lore raw

Re: [PATCH 17/42] drm/i915: Pass around sg_table to get_pages/put_pages backend

Quoted 14 lines · Chris Wilson
Quoted 13 lines · Tvrtko Ursulin

diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h index 3c22d49005fe..271e63c8f037 100644
--- a/drivers/gpu/drm/i915/i915_drv.h
+++ b/drivers/gpu/drm/i915/i915_drv.h
@@ -2175,8 +2175,8 @@ struct drm_i915_gem_object_ops {
* being released or under memory pressure (where we attempt to
* reap pages for the shrinker).
*/

  -       int (*get_pages)(struct drm_i915_gem_object *);
  -       void (*put_pages)(struct drm_i915_gem_object *);
  +       struct sg_table *(*get_pages)(struct drm_i915_gem_object *);
  +       void (*put_pages)(struct drm_i915_gem_object *, struct sg_table *);
Quoted 6 lines
Quoted 3 lines

Idea is that put_pages vfunc does not need struct mutex? Or it acquires it on demand inside it, which means struct mutex will nest inside the mm.lock?

Right, it loses the mutex within get/put pages and I am no longer concerned about the number of workers. Though that is mainly because of another patch to improve execbuf + userptr workloads.

Is that other patch before or after this one? Or in a different series altogether? In any case, why should this change be in this patch, especially when it is not commented anywhere?

Regards,

Tvrtko

_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx

#59 lore raw

Re: [PATCH 17/42] drm/i915: Pass around sg_table to get_pages/put_pages backend

Quoted 19 lines · Tvrtko Ursulin
Quoted 17 lines · Chris Wilson
Quoted 16 lines · Tvrtko Ursulin

diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h index 3c22d49005fe..271e63c8f037 100644
--- a/drivers/gpu/drm/i915/i915_drv.h
+++ b/drivers/gpu/drm/i915/i915_drv.h
@@ -2175,8 +2175,8 @@ struct drm_i915_gem_object_ops {
* being released or under memory pressure (where we attempt to
* reap pages for the shrinker).
*/

 -       int (*get_pages)(struct drm_i915_gem_object *);
 -       void (*put_pages)(struct drm_i915_gem_object *);
 +       struct sg_table *(*get_pages)(struct drm_i915_gem_object *);
 +       void (*put_pages)(struct drm_i915_gem_object *, struct sg_table *);

Idea is that put_pages vfunc does not need struct mutex? Or it acquires it on demand inside it, which means struct mutex will nest inside the mm.lock?

Quoted 7 lines
Quoted 3 lines

Right, it loses the mutex within get/put pages and I am no longer concerned about the number of workers. Though that is mainly because of another patch to improve execbuf + userptr workloads.

Is that other patch before or after this one? Or in a different series altogether? In any case, why should this change be in this patch, especially when it is not commented anywhere?

Actually, it follows from

commit 364c8172edb5ff0b4650bbd4c45eaab46b84b008

Author: Chris Wilson <[email protected]>
Date:   Thu Aug 18 17:16:58 2016 +0100

drm/i915/userptr: Make gup errors stickier

which was the prelude to the patch I was thinking of. The current code already doesn't use more than one worker simultaneously. -Chris

Signature · 5 lines
Chris Wilson, Intel Open Source Technology Centre
_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
#60 lore raw

PATCH 18/42 drm/i915: Move object backing storage manipulation to its own locking

In reply to Chris Wilson

Break the allocation of the backing storage away from struct_mutex into a per-object lock. This allows parallel page allocation, provided we can do so outside of struct_mutex (i.e. set-domain-ioctl, pwrite, GTT fault), i.e. before execbuf! The increased cost of the atomic counters are hidden behind i915_vma_pin() for the typical case of execbuf, i.e. as the object is typically bound between execbufs, the page_pin_count is static. The cost will be felt around set-domain and pwrite, but offset by the improvement from reduced struct_mutex contention.

---
 drivers/gpu/drm/i915/i915_drv.h          | 26 ++++------
 drivers/gpu/drm/i915/i915_gem.c          | 89 +++++++++++++++++++++-----------
 drivers/gpu/drm/i915/i915_gem_shrinker.c | 51 ++++++++++++------
 drivers/gpu/drm/i915/i915_gem_tiling.c   |  2 +
 drivers/gpu/drm/i915/i915_gem_userptr.c  | 10 ++--
 5 files changed, 110 insertions(+), 68 deletions(-)
drivers/gpu/drm/i915/i915_drv.h +10 -16
index 271e63c8f037..7e9df190c891 100644
@@ -2266,7 +2266,8 @@ struct drm_i915_gem_object {
 	unsigned int pin_display;
 
 	struct {
-		unsigned int pages_pin_count;
+		struct mutex lock;
+		atomic_t pages_pin_count;
 
 		struct sg_table *pages;
 		void *mapping;
@@ -3206,8 +3207,7 @@ int __i915_gem_object_get_pages(struct drm_i915_gem_object *obj);
 static inline int __must_check
 i915_gem_object_pin_pages(struct drm_i915_gem_object *obj)
 {
-	lockdep_assert_held(&obj->base.dev->struct_mutex);
-	if (obj->mm.pages_pin_count++)
+	if (atomic_inc_not_zero(&obj->mm.pages_pin_count))
 		return 0;
 
 	return __i915_gem_object_get_pages(obj);
@@ -3216,29 +3216,26 @@ i915_gem_object_pin_pages(struct drm_i915_gem_object *obj)
 static inline void
 __i915_gem_object_pin_pages(struct drm_i915_gem_object *obj)
 {
-	lockdep_assert_held_unless(&obj->base.dev->struct_mutex,
-				   i915_gem_object_is_dead(obj));
 	GEM_BUG_ON(!obj->mm.pages);
-	obj->mm.pages_pin_count++;
+	atomic_inc(&obj->mm.pages_pin_count);
 }
 
 static inline bool
 i915_gem_object_has_pinned_pages(struct drm_i915_gem_object *obj)
 {
-	return obj->mm.pages_pin_count;
+	return atomic_read(&obj->mm.pages_pin_count);
 }
 
 static inline void
 __i915_gem_object_unpin_pages(struct drm_i915_gem_object *obj)
 {
-	lockdep_assert_held_unless(&obj->base.dev->struct_mutex,
-				   i915_gem_object_is_dead(obj));
 	GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));
 	GEM_BUG_ON(!obj->mm.pages);
-	obj->mm.pages_pin_count--;
+	atomic_dec(&obj->mm.pages_pin_count);
 }
 
-static inline void i915_gem_object_unpin_pages(struct drm_i915_gem_object *obj)
+static inline void
+i915_gem_object_unpin_pages(struct drm_i915_gem_object *obj)
 {
 	__i915_gem_object_unpin_pages(obj);
 }
@@ -3261,8 +3258,8 @@ enum i915_map_type {
  * the kernel address space. Based on the @type of mapping, the PTE will be
  * set to either WriteBack or WriteCombine (via pgprot_t).
  *
- * The caller must hold the struct_mutex, and is responsible for calling
- * i915_gem_object_unpin_map() when the mapping is no longer required.
+ * The caller is responsible for calling i915_gem_object_unpin_map() when the
+ * mapping is no longer required.
  *
  * Returns the pointer through which to access the mapped object, or an
  * ERR_PTR() on error.
@@ -3278,12 +3275,9 @@ void *__must_check i915_gem_object_pin_map(struct drm_i915_gem_object *obj,
  * with your access, call i915_gem_object_unpin_map() to release the pin
  * upon the mapping. Once the pin count reaches zero, that mapping may be
  * removed.
- *
- * The caller must hold the struct_mutex.
  */
 static inline void i915_gem_object_unpin_map(struct drm_i915_gem_object *obj)
 {
-	lockdep_assert_held(&obj->base.dev->struct_mutex);
 	i915_gem_object_unpin_pages(obj);
 }
 
drivers/gpu/drm/i915/i915_gem.c +60 -29

190 lines not on this page · raw message

drivers/gpu/drm/i915/i915_gem_shrinker.c +34 -17

99 lines not on this page · raw message

drivers/gpu/drm/i915/i915_gem_tiling.c +2 -0
index 7286de7bd25e..11e2e0f57ac1 100644
@@ -260,6 +260,7 @@ i915_gem_set_tiling(struct drm_device *dev, void *data,
 		if (!err) {
 			struct i915_vma *vma;
 
+			mutex_lock(&obj->mm.lock);
 			if (obj->mm.pages &&
 			    obj->mm.madv == I915_MADV_WILLNEED &&
 			    dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES) {
@@ -268,6 +269,7 @@ i915_gem_set_tiling(struct drm_device *dev, void *data,
 				if (!i915_gem_object_is_tiled(obj))
 					__i915_gem_object_pin_pages(obj);
 			}
+			mutex_unlock(&obj->mm.lock);
 
 			list_for_each_entry(vma, &obj->vma_list, obj_link) {
 				if (!vma->fence)
drivers/gpu/drm/i915/i915_gem_userptr.c +4 -6
index e97c7b589439..136c493b15b2 100644
@@ -79,7 +79,7 @@ static void cancel_userptr(struct work_struct *work)
 	WARN_ONCE(obj->mm.pages,
 		  "Failed to release pages: bind_count=%d, pages_pin_count=%d, pin_display=%d\n",
 		  obj->bind_count,
-		  obj->mm.pages_pin_count,
+		  atomic_read(&obj->mm.pages_pin_count),
 		  obj->pin_display);
 
 	i915_gem_object_put(obj);
@@ -491,7 +491,6 @@ __i915_gem_userptr_get_pages_worker(struct work_struct *_work)
 {
 	struct get_pages_work *work = container_of(_work, typeof(*work), work);
 	struct drm_i915_gem_object *obj = work->obj;
-	struct drm_device *dev = obj->base.dev;
 	const int npages = obj->base.size >> PAGE_SHIFT;
 	struct page **pvec;
 	int pinned, ret;
@@ -523,7 +522,7 @@ __i915_gem_userptr_get_pages_worker(struct work_struct *_work)
 		}
 	}
 
-	mutex_lock(&dev->struct_mutex);
+	mutex_lock(&obj->mm.lock);
 	if (obj->userptr.work == &work->work) {
 		struct sg_table *pages = ERR_PTR(ret);
 
@@ -538,13 +537,12 @@ __i915_gem_userptr_get_pages_worker(struct work_struct *_work)
 
 		obj->userptr.work = ERR_CAST(pages);
 	}
-
-	i915_gem_object_put(obj);
-	mutex_unlock(&dev->struct_mutex);
+	mutex_unlock(&obj->mm.lock);
 
 	release_pages(pvec, pinned, 0);
 	drm_free_large(pvec);
 
+	i915_gem_object_put_unlocked(obj);
 	put_task_struct(work->task);
 	kfree(work);
 }
Signature · 6 lines
2.9.3

_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
maintainer Reviewed-by on v1 18/42
#61 lore raw

Re: [PATCH 18/42] drm/i915: Move object backing storage manipulation to its own locking

Quoted 7 lines · Chris Wilson
(quoted diff) +0 -0
@@ -4211,10 +4240,10 @@ i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
  		i915_gem_object_truncate(obj);


  args->retained = obj->mm.madv != __I915_MADV_PURGED;

(quoted diff) +1 -0
+	mutex_unlock(&obj->mm.lock);


+err:

Shared, so 'out:'

Other than that, looks good.

Am I to assume that now when we introduced a new lock, every single function that assumes the lock taken is now tagged with assert_lockdep_held?

Regards, Joonas

Signature · 7 lines
Joonas Lahtinen
Open Source Technology Center
Intel Corporation
_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
#62 lore raw

Re: [PATCH 18/42] drm/i915: Move object backing storage manipulation to its own locking

Quoted 11 lines · Joonas Lahtinen
Quoted 7 lines · Chris Wilson
(quoted diff) +0 -0
@@ -4211,10 +4240,10 @@ i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
  		i915_gem_object_truncate(obj);


  args->retained = obj->mm.madv != __I915_MADV_PURGED;

(quoted diff) +1 -0
+	mutex_unlock(&obj->mm.lock);


+err:

Shared, so 'out:'

Quoted 7 lines

Other than that, looks good.

Am I to assume that now when we introduced a new lock, every single function that assumes the lock taken is now tagged with assert_lockdep_held?

There's are two external calls that assume the caller holds the obj->mm.lock, and yes they are so marked.
-Chris

Signature · 5 lines
Chris Wilson, Intel Open Source Technology Centre
_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
#63 lore raw

PATCH 19/42 drm/i915/dmabuf: Acquire the backing storage outside of struct_mutex

In reply to Chris Wilson

Use the per-object mm.lock to allocate the backing storage (and hold a reference to it across the dmabuf access) without resorting to struct_mutex.

---
 drivers/gpu/drm/i915/i915_gem_dmabuf.c | 51 ++++++++++++++--------------------
 1 file changed, 21 insertions(+), 30 deletions(-)
drivers/gpu/drm/i915/i915_gem_dmabuf.c +21 -30
index 2abd524aba14..9023110cf1ca 100644
@@ -44,19 +44,15 @@ static struct sg_table *i915_gem_map_dma_buf(struct dma_buf_attachment *attachme
 	struct scatterlist *src, *dst;
 	int ret, i;
 
-	ret = i915_mutex_lock_interruptible(obj->base.dev);
-	if (ret)
-		goto err;
-
 	ret = i915_gem_object_pin_pages(obj);
 	if (ret)
-		goto err_unlock;
+		goto err;
 
 	/* Copy sg so that we make an independent mapping */
 	st = kmalloc(sizeof(struct sg_table), GFP_KERNEL);
 	if (st == NULL) {
 		ret = -ENOMEM;
-		goto err_unpin;
+		goto err_put_pages;
 	}
 
 	ret = sg_alloc_table(st, obj->mm.pages->nents, GFP_KERNEL);
@@ -72,21 +68,18 @@ static struct sg_table *i915_gem_map_dma_buf(struct dma_buf_attachment *attachme
 	}
 
 	if (!dma_map_sg(attachment->dev, st->sgl, st->nents, dir)) {
-		ret =-ENOMEM;
+		ret = -ENOMEM;
 		goto err_free_sg;
 	}
 
-	mutex_unlock(&obj->base.dev->struct_mutex);
 	return st;
 
 err_free_sg:
 	sg_free_table(st);
 err_free:
 	kfree(st);
-err_unpin:
+err_put_pages:
 	i915_gem_object_unpin_pages(obj);
-err_unlock:
-	mutex_unlock(&obj->base.dev->struct_mutex);
 err:
 	return ERR_PTR(ret);
 }
@@ -101,36 +94,21 @@ static void i915_gem_unmap_dma_buf(struct dma_buf_attachment *attachment,
 	sg_free_table(sg);
 	kfree(sg);
 
-	mutex_lock(&obj->base.dev->struct_mutex);
 	i915_gem_object_unpin_pages(obj);
-	mutex_unlock(&obj->base.dev->struct_mutex);
 }
 
 static void *i915_gem_dmabuf_vmap(struct dma_buf *dma_buf)
 {
 	struct drm_i915_gem_object *obj = dma_buf_to_obj(dma_buf);
-	struct drm_device *dev = obj->base.dev;
-	void *addr;
-	int ret;
 
-	ret = i915_mutex_lock_interruptible(dev);
-	if (ret)
-		return ERR_PTR(ret);
-
-	addr = i915_gem_object_pin_map(obj, I915_MAP_WB);
-	mutex_unlock(&dev->struct_mutex);
-
-	return addr;
+	return i915_gem_object_pin_map(obj, I915_MAP_WB);
 }
 
 static void i915_gem_dmabuf_vunmap(struct dma_buf *dma_buf, void *vaddr)
 {
 	struct drm_i915_gem_object *obj = dma_buf_to_obj(dma_buf);
-	struct drm_device *dev = obj->base.dev;
 
-	mutex_lock(&dev->struct_mutex);
 	i915_gem_object_unpin_map(obj);
-	mutex_unlock(&dev->struct_mutex);
 }
 
 static void *i915_gem_dmabuf_kmap_atomic(struct dma_buf *dma_buf, unsigned long page_num)
@@ -177,15 +155,22 @@ static int i915_gem_begin_cpu_access(struct dma_buf *dma_buf, enum dma_data_dire
 {
 	struct drm_i915_gem_object *obj = dma_buf_to_obj(dma_buf);
 	struct drm_device *dev = obj->base.dev;
-	int ret;
 	bool write = (direction == DMA_BIDIRECTIONAL || direction == DMA_TO_DEVICE);
+	int ret;
 
-	ret = i915_mutex_lock_interruptible(dev);
+	ret = i915_gem_object_pin_pages(obj);
 	if (ret)
 		return ret;
 
+	ret = i915_mutex_lock_interruptible(dev);
+	if (ret)
+		goto err;
+
 	ret = i915_gem_object_set_to_cpu_domain(obj, write);
 	mutex_unlock(&dev->struct_mutex);
+
+err:
+	i915_gem_object_unpin_pages(obj);
 	return ret;
 }
 
@@ -195,13 +180,19 @@ static int i915_gem_end_cpu_access(struct dma_buf *dma_buf, enum dma_data_direct
 	struct drm_device *dev = obj->base.dev;
 	int ret;
 
-	ret = i915_mutex_lock_interruptible(dev);
+	ret = i915_gem_object_pin_pages(obj);
 	if (ret)
 		return ret;
 
+	ret = i915_mutex_lock_interruptible(dev);
+	if (ret)
+		goto err;
+
 	ret = i915_gem_object_set_to_gtt_domain(obj, false);
 	mutex_unlock(&dev->struct_mutex);
 
+err:
+	i915_gem_object_unpin_pages(obj);
 	return ret;
 }
 
Signature · 6 lines
2.9.3

_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
maintainer Reviewed-by on v1 19/42
#64 lore raw

Re: [PATCH 19/42] drm/i915/dmabuf: Acquire the backing storage outside of struct_mutex

Quoted 18 lines · Chris Wilson
(quoted diff) +0 -0
@@ -72,21 +68,18 @@ static struct sg_table *i915_gem_map_dma_buf(struct dma_buf_attachment *attachme
  	}


  if (!dma_map_sg(attachment->dev, st->sgl, st->nents, dir)) {

drivers/gpu/drm/i915/i915_gem_dmabuf.c +1 -1
-		ret =-ENOMEM;
+		ret = -ENOMEM;

  goto err_free_sg;
  }

(quoted diff) +0 -1
-	mutex_unlock(&obj->base.dev->struct_mutex);

  return st;

 err_free_sg:
  sg_free_table(st);
 err_free:
  kfree(st);

drivers/gpu/drm/i915/i915_gem_dmabuf.c +1 -1
-err_unpin:
+err_put_pages:
Quoted 1 line

  i915_gem_object_unpin_pages(obj);

I do not quite follow the logic leading to rename in here.

Quoted 19 lines
drivers/gpu/drm/i915/i915_gem_dmabuf.c +1 -1
@@ -177,15 +155,22 @@ static int i915_gem_begin_cpu_access(struct dma_buf *dma_buf, enum dma_data_dire
  {
  	struct drm_i915_gem_object *obj = dma_buf_to_obj(dma_buf);
  	struct drm_device *dev = obj->base.dev;
-	int ret;
  	bool write = (direction == DMA_BIDIRECTIONAL || direction == DMA_TO_DEVICE);
+	int ret;

(quoted diff) +1 -1
-	ret = i915_mutex_lock_interruptible(dev);
+	ret = i915_gem_object_pin_pages(obj);

  if (ret)
  return ret;

drivers/gpu/drm/i915/i915_gem_dmabuf.c +4 -0
+	ret = i915_mutex_lock_interruptible(dev);
+	if (ret)
+		goto err;
+

  ret = i915_gem_object_set_to_cpu_domain(obj, write); mutex_unlock(&dev->struct_mutex);

Review

Quoted 2 lines
(quoted diff) +2 -0
+
+err:

combined path, 'out:'

Quoted 15 lines
drivers/gpu/drm/i915/i915_gem_dmabuf.c +0 -0
@@ -195,13 +180,19 @@ static int i915_gem_end_cpu_access(struct dma_buf *dma_buf, enum dma_data_direct
  	struct drm_device *dev = obj->base.dev;
  	int ret;

(quoted diff) +1 -1
-	ret = i915_mutex_lock_interruptible(dev);
+	ret = i915_gem_object_pin_pages(obj);

  if (ret)
  return ret;

drivers/gpu/drm/i915/i915_gem_dmabuf.c +4 -0
+	ret = i915_mutex_lock_interruptible(dev);
+	if (ret)
+		goto err;
+
 	ret = i915_gem_object_set_to_gtt_domain(obj, false);
	mutex_unlock(&dev->struct_mutex);
Quoted 2 lines


+err:

Ditto.

With above fixed;

Regards, Joonas

Signature · 7 lines
Joonas Lahtinen
Open Source Technology Center
Intel Corporation
_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
#65 lore raw

PATCH 20/42 drm/i915: Implement pread without struct-mutex

In reply to Chris Wilson

We only need struct_mutex within pread for a brief window where we need to serialise with rendering and control our cache domains. Elsewhere we can rely on the backing storage being pinned, and forgive userspace any races against us.

---
 drivers/gpu/drm/i915/i915_gem.c | 364 +++++++++++++++++-----------------------
 1 file changed, 158 insertions(+), 206 deletions(-)
drivers/gpu/drm/i915/i915_gem.c +158 -206
index f4a773ca9333..27f91714e82e 100644
@@ -63,13 +63,13 @@ static bool cpu_write_needs_clflush(struct drm_i915_gem_object *obj)
 }
 
 static int
-insert_mappable_node(struct drm_i915_private *i915,
+insert_mappable_node(struct i915_ggtt *ggtt,
                      struct drm_mm_node *node, u32 size)
 {
 	memset(node, 0, sizeof(*node));
-	return drm_mm_insert_node_in_range_generic(&i915->ggtt.base.mm, node,
-						   size, 0, 0, 0,
-						   i915->ggtt.mappable_end,
+	return drm_mm_insert_node_in_range_generic(&ggtt->base.mm, node,
+						   size, 0, -1,
+						   0, ggtt->mappable_end,
 						   DRM_MM_SEARCH_DEFAULT,
 						   DRM_MM_CREATE_DEFAULT);
 }
@@ -820,32 +820,6 @@ err_unpin:
 	return ret;
 }
 
-/* Per-page copy function for the shmem pread fastpath.
- * Flushes invalid cachelines before reading the target if
- * needs_clflush is set. */
-static int
-shmem_pread_fast(struct page *page, int shmem_page_offset, int page_length,
-		 char __user *user_data,
-		 bool page_do_bit17_swizzling, bool needs_clflush)
-{
-	char *vaddr;
-	int ret;
-
-	if (unlikely(page_do_bit17_swizzling))
-		return -EINVAL;
-
-	vaddr = kmap_atomic(page);
-	if (needs_clflush)
-		drm_clflush_virt_range(vaddr + shmem_page_offset,
-				       page_length);
-	ret = __copy_to_user_inatomic(user_data,
-				      vaddr + shmem_page_offset,
-				      page_length);
-	kunmap_atomic(vaddr);
-
-	return ret ? -EFAULT : 0;
-}
-
 static void
 shmem_clflush_swizzled_range(char *addr, unsigned long length,
 			     bool swizzled)
@@ -871,7 +845,7 @@ shmem_clflush_swizzled_range(char *addr, unsigned long length,
 /* Only difference to the fast-path function is that this can handle bit17
  * and uses non-atomic copy and kmap functions. */
 static int
-shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length,
+shmem_pread_slow(struct page *page, int offset, int length,
 		 char __user *user_data,
 		 bool page_do_bit17_swizzling, bool needs_clflush)
 {
@@ -880,60 +854,132 @@ shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length,
 
 	vaddr = kmap(page);
 	if (needs_clflush)
-		shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
-					     page_length,
+		shmem_clflush_swizzled_range(vaddr + offset, length,
 					     page_do_bit17_swizzling);
 
 	if (page_do_bit17_swizzling)
-		ret = __copy_to_user_swizzled(user_data,
-					      vaddr, shmem_page_offset,
-					      page_length);
+		ret = __copy_to_user_swizzled(user_data, vaddr, offset, length);
 	else
-		ret = __copy_to_user(user_data,
-				     vaddr + shmem_page_offset,
-				     page_length);
+		ret = __copy_to_user(user_data, vaddr + offset, length);
 	kunmap(page);
 
 	return ret ? - EFAULT : 0;
 }
 
-static inline unsigned long
-slow_user_access(struct io_mapping *mapping,
-		 uint64_t page_base, int page_offset,
-		 char __user *user_data,
-		 unsigned long length, bool pwrite)
+static int
+shmem_pread(struct page *page, int offset, int length, char __user *user_data,
+	    bool page_do_bit17_swizzling, bool needs_clflush)
 {
-	void __iomem *ioaddr;
+	int ret;
+
+	ret = -ENODEV;
+	if (!page_do_bit17_swizzling) {
+		char *vaddr = kmap_atomic(page);
+
+		if (needs_clflush)
+			drm_clflush_virt_range(vaddr + offset, length);
+		ret = __copy_to_user_inatomic(user_data, vaddr + offset, length);
+		kunmap_atomic(vaddr);
+	}
+	if (ret == 0)
+		return 0;
+
+	return shmem_pread_slow(page, offset, length, user_data,
+				page_do_bit17_swizzling, needs_clflush);
+}
+
+static int
+i915_gem_shmem_pread(struct drm_i915_gem_object *obj,
+		     struct drm_i915_gem_pread *args)
+{
+	char __user *user_data;
+	u64 remain;
+	unsigned int obj_do_bit17_swizzling;
+	unsigned int needs_clflush;
+	unsigned int idx, offset;
+	int ret;
+
+	obj_do_bit17_swizzling = 0;
+	if (i915_gem_object_needs_bit17_swizzle(obj))
+		obj_do_bit17_swizzling = 1 << 17;
+
+	ret = mutex_lock_interruptible(&obj->base.dev->struct_mutex);
+	if (ret)
+		return ret;
+
+	ret = i915_gem_obj_prepare_shmem_read(obj, &needs_clflush);
+	mutex_unlock(&obj->base.dev->struct_mutex);
+	if (ret)
+		return ret;
+
+	remain = args->size;
+	user_data = u64_to_user_ptr(args->data_ptr);
+	offset = offset_in_page(args->offset);
+	for (idx = args->offset >> PAGE_SHIFT; remain; idx++) {
+		struct page *page = i915_gem_object_get_page(obj, idx);
+		int length;
+
+		length = remain;
+		if (offset + length > PAGE_SIZE)
+			length = PAGE_SIZE - offset;
+
+		ret = shmem_pread(page, offset, length, user_data,
+				  page_to_phys(page) & obj_do_bit17_swizzling,
+				  needs_clflush);
+		if (ret)
+			break;
+
+		remain -= length;
+		user_data += length;
+		offset = 0;
+	}
+
+	i915_gem_obj_finish_shmem_access(obj);
+	return ret;
+}
+
+static inline int
+gtt_user_read(struct io_mapping *mapping,
+	      loff_t page_base, int page_offset,
+	      char __user *user_data,
+	      int length)
+{
+	void __iomem *vaddr_atomic;
 	void *vaddr;
-	uint64_t unwritten;
+	unsigned long unwritten;
 
-	ioaddr = io_mapping_map_wc(mapping, page_base, PAGE_SIZE);
+	vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
 	/* We can use the cpu mem copy function because this is X86. */
-	vaddr = (void __force *)ioaddr + page_offset;
-	if (pwrite)
-		unwritten = __copy_from_user(vaddr, user_data, length);
-	else
-		unwritten = __copy_to_user(user_data, vaddr, length);
-
-	io_mapping_unmap(ioaddr);
+	vaddr = (void __force *)vaddr_atomic + page_offset;
+	unwritten = __copy_to_user_inatomic(user_data, vaddr, length);
+	io_mapping_unmap_atomic(vaddr_atomic);
+	if (unwritten) {
+		vaddr_atomic = io_mapping_map_wc(mapping, page_base, PAGE_SIZE);
+		/* We can use the cpu mem copy function because this is X86. */
+		vaddr = (void __force *)vaddr_atomic + page_offset;
+		unwritten = copy_to_user(user_data, vaddr, length);
+		io_mapping_unmap(vaddr_atomic);
+	}
 	return unwritten;
 }
 
 static int
-i915_gem_gtt_pread(struct drm_device *dev,
-		   struct drm_i915_gem_object *obj, uint64_t size,
-		   uint64_t data_offset, uint64_t data_ptr)
+i915_gem_gtt_pread(struct drm_i915_gem_object *obj,
+		   const struct drm_i915_gem_pread *args)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct i915_ggtt *ggtt = &dev_priv->ggtt;
-	struct i915_vma *vma;
+	struct i915_ggtt *ggtt = &to_i915(obj->base.dev)->ggtt;
 	struct drm_mm_node node;
-	char __user *user_data;
-	uint64_t remain;
-	uint64_t offset;
+	struct i915_vma *vma;
+	void __user *user_data;
+	u64 remain, offset;
 	int ret;
 
-	vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, PIN_MAPPABLE);
+	ret = mutex_lock_interruptible(&obj->base.dev->struct_mutex);
+	if (ret)
+		return ret;
+
+	vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0,
+				       PIN_MAPPABLE | PIN_NONBLOCK);
 	if (!IS_ERR(vma)) {
 		node.start = i915_ggtt_offset(vma);
 		node.allocated = false;
@@ -944,33 +990,21 @@ i915_gem_gtt_pread(struct drm_device *dev,
 		}
 	}
 	if (IS_ERR(vma)) {
-		ret = insert_mappable_node(dev_priv, &node, PAGE_SIZE);
+		ret = insert_mappable_node(ggtt, &node, PAGE_SIZE);
 		if (ret)
-			goto out;
-
-		ret = i915_gem_object_pin_pages(obj);
-		if (ret) {
-			remove_mappable_node(&node);
-			goto out;
-		}
+			goto out_unlock;
+		GEM_BUG_ON(!node.allocated);
 	}
 
 	ret = i915_gem_object_set_to_gtt_domain(obj, false);
 	if (ret)
 		goto out_unpin;
 
-	user_data = u64_to_user_ptr(data_ptr);
-	remain = size;
-	offset = data_offset;
+	mutex_unlock(&obj->base.dev->struct_mutex);
 
-	mutex_unlock(&dev->struct_mutex);
-	if (likely(!i915.prefault_disable)) {
-		ret = fault_in_multipages_writeable(user_data, remain);
-		if (ret) {
-			mutex_lock(&dev->struct_mutex);
-			goto out_unpin;
-		}
-	}
+	user_data = u64_to_user_ptr(args->data_ptr);
+	remain = args->size;
+	offset = args->offset;
 
 	while (remain > 0) {
 		/* Operation in this page
@@ -987,19 +1021,14 @@ i915_gem_gtt_pread(struct drm_device *dev,
 			wmb();
 			ggtt->base.insert_page(&ggtt->base,
 					       i915_gem_object_get_dma_address(obj, offset >> PAGE_SHIFT),
-					       node.start,
-					       I915_CACHE_NONE, 0);
+					       node.start, I915_CACHE_NONE, 0);
 			wmb();
 		} else {
 			page_base += offset & PAGE_MASK;
 		}
-		/* This is a slow read/write as it tries to read from
-		 * and write to user memory which may result into page
-		 * faults, and so we cannot perform this under struct_mutex.
-		 */
-		if (slow_user_access(&ggtt->mappable, page_base,
-				     page_offset, user_data,
-				     page_length, false)) {
+
+		if (gtt_user_read(&ggtt->mappable, page_base, page_offset,
+				  user_data, page_length)) {
 			ret = -EFAULT;
 			break;
 		}
@@ -1009,111 +1038,19 @@ i915_gem_gtt_pread(struct drm_device *dev,
 		offset += page_length;
 	}
 
-	mutex_lock(&dev->struct_mutex);
-	if (ret == 0 && (obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0) {
-		/* The user has modified the object whilst we tried
-		 * reading from it, and we now have no idea what domain
-		 * the pages should be in. As we have just been touching
-		 * them directly, flush everything back to the GTT
-		 * domain.
-		 */
-		ret = i915_gem_object_set_to_gtt_domain(obj, false);
-	}
-
+	mutex_lock(&obj->base.dev->struct_mutex);
 out_unpin:
 	if (node.allocated) {
 		wmb();
 		ggtt->base.clear_range(&ggtt->base,
 				       node.start, node.size,
 				       true);
-		i915_gem_object_unpin_pages(obj);
 		remove_mappable_node(&node);
 	} else {
 		i915_vma_unpin(vma);
 	}
-out:
-	return ret;
-}
-
-static int
-i915_gem_shmem_pread(struct drm_device *dev,
-		     struct drm_i915_gem_object *obj,
-		     struct drm_i915_gem_pread *args,
-		     struct drm_file *file)
-{
-	char __user *user_data;
-	ssize_t remain;
-	loff_t offset;
-	int shmem_page_offset, page_length, ret = 0;
-	int obj_do_bit17_swizzling, page_do_bit17_swizzling;
-	int prefaulted = 0;
-	int needs_clflush = 0;
-	struct sg_page_iter sg_iter;
-
-	ret = i915_gem_obj_prepare_shmem_read(obj, &needs_clflush);
-	if (ret)
-		return ret;
-
-	obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
-	user_data = u64_to_user_ptr(args->data_ptr);
-	offset = args->offset;
-	remain = args->size;
-
-	for_each_sg_page(obj->mm.pages->sgl, &sg_iter, obj->mm.pages->nents,
-			 offset >> PAGE_SHIFT) {
-		struct page *page = sg_page_iter_page(&sg_iter);
-
-		if (remain <= 0)
-			break;
-
-		/* Operation in this page
-		 *
-		 * shmem_page_offset = offset within page in shmem file
-		 * page_length = bytes to copy for this page
-		 */
-		shmem_page_offset = offset_in_page(offset);
-		page_length = remain;
-		if ((shmem_page_offset + page_length) > PAGE_SIZE)
-			page_length = PAGE_SIZE - shmem_page_offset;
-
-		page_do_bit17_swizzling = obj_do_bit17_swizzling &&
-			(page_to_phys(page) & (1 << 17)) != 0;
-
-		ret = shmem_pread_fast(page, shmem_page_offset, page_length,
-				       user_data, page_do_bit17_swizzling,
-				       needs_clflush);
-		if (ret == 0)
-			goto next_page;
-
-		mutex_unlock(&dev->struct_mutex);
-
-		if (likely(!i915.prefault_disable) && !prefaulted) {
-			ret = fault_in_multipages_writeable(user_data, remain);
-			/* Userspace is tricking us, but we've already clobbered
-			 * its pages with the prefault and promised to write the
-			 * data up to the first fault. Hence ignore any errors
-			 * and just continue. */
-			(void)ret;
-			prefaulted = 1;
-		}
-
-		ret = shmem_pread_slow(page, shmem_page_offset, page_length,
-				       user_data, page_do_bit17_swizzling,
-				       needs_clflush);
-
-		mutex_lock(&dev->struct_mutex);
-
-		if (ret)
-			goto out;
-
-next_page:
-		remain -= page_length;
-		user_data += page_length;
-		offset += page_length;
-	}
-
-out:
-	i915_gem_obj_finish_shmem_access(obj);
+out_unlock:
+	mutex_unlock(&obj->base.dev->struct_mutex);
 
 	return ret;
 }
@@ -1132,7 +1069,7 @@ i915_gem_pread_ioctl(struct drm_device *dev, void *data,
 {
 	struct drm_i915_gem_pread *args = data;
 	struct drm_i915_gem_object *obj;
-	int ret = 0;
+	int ret;
 
 	if (args->size == 0)
 		return 0;
@@ -1150,7 +1087,7 @@ i915_gem_pread_ioctl(struct drm_device *dev, void *data,
 	if (args->offset > obj->base.size ||
 	    args->size > obj->base.size - args->offset) {
 		ret = -EINVAL;
-		goto err;
+		goto out;
 	}
 
 	trace_i915_gem_object_pread(obj, args->offset, args->size);
@@ -1160,28 +1097,21 @@ i915_gem_pread_ioctl(struct drm_device *dev, void *data,
 				   MAX_SCHEDULE_TIMEOUT,
 				   to_rps_client(file));
 	if (ret)
-		goto err;
+		goto out;
 
-	ret = i915_mutex_lock_interruptible(dev);
+	ret = i915_gem_object_pin_pages(obj);
 	if (ret)
-		goto err;
-
-	ret = i915_gem_shmem_pread(dev, obj, args, file);
+		goto out;
 
-	/* pread for non shmem backed objects */
+	ret = i915_gem_shmem_pread(obj, args);
 	if (ret == -EFAULT || ret == -ENODEV) {
 		intel_runtime_pm_get(to_i915(dev));
-		ret = i915_gem_gtt_pread(dev, obj, args->size,
-					args->offset, args->data_ptr);
+		ret = i915_gem_gtt_pread(obj, args);
 		intel_runtime_pm_put(to_i915(dev));
 	}
 
-	i915_gem_object_put(obj);
-	mutex_unlock(&dev->struct_mutex);
-
-	return ret;
-
-err:
+	i915_gem_object_unpin_pages(obj);
+out:
 	i915_gem_object_put_unlocked(obj);
 	return ret;
 }
@@ -1209,6 +1139,28 @@ fast_user_write(struct io_mapping *mapping,
 	return unwritten;
 }
 
+static inline unsigned long
+slow_user_access(struct io_mapping *mapping,
+		 unsigned long page_base, int page_offset,
+		 char __user *user_data,
+		 unsigned long length, bool pwrite)
+{
+	void __iomem *ioaddr;
+	void *vaddr;
+	unsigned long unwritten;
+
+	ioaddr = io_mapping_map_wc(mapping, page_base, PAGE_SIZE);
+	/* We can use the cpu mem copy function because this is X86. */
+	vaddr = (void __force *)ioaddr + page_offset;
+	if (pwrite)
+		unwritten = __copy_from_user(vaddr, user_data, length);
+	else
+		unwritten = __copy_to_user(user_data, vaddr, length);
+
+	io_mapping_unmap(ioaddr);
+	return unwritten;
+}
+
 /**
  * This is the fast pwrite path, where we copy the data directly from the
  * user into the GTT, uncached.
@@ -1247,7 +1199,7 @@ i915_gem_gtt_pwrite_fast(struct drm_i915_private *i915,
 		}
 	}
 	if (IS_ERR(vma)) {
-		ret = insert_mappable_node(i915, &node, PAGE_SIZE);
+		ret = insert_mappable_node(ggtt, &node, PAGE_SIZE);
 		if (ret)
 			goto out;
 
@@ -1276,8 +1228,8 @@ i915_gem_gtt_pwrite_fast(struct drm_i915_private *i915,
 		 * page_length = bytes to copy for this page
 		 */
 		u32 page_base = node.start;
-		unsigned page_offset = offset_in_page(offset);
-		unsigned page_length = PAGE_SIZE - page_offset;
+		unsigned int page_offset = offset_in_page(offset);
+		unsigned int page_length = PAGE_SIZE - page_offset;
 		page_length = remain < page_length ? remain : page_length;
 		if (node.allocated) {
 			wmb(); /* flush the write before we modify the GGTT */
Signature · 6 lines
2.9.3

_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
maintainer Reviewed-by on v1 20/42
#66 lore raw

Re: [PATCH 20/42] drm/i915: Implement pread without struct-mutex

drivers/gpu/drm/i915/i915_gem.c shmem_clflush_swizzled_range(char *addr, unsigned long length, · line 848

Quoted 6 lines · Chris Wilson
drivers/gpu/drm/i915/i915_gem.c +0 -0
@@ -871,7 +845,7 @@ shmem_clflush_swizzled_range(char *addr, unsigned long length,
  /* Only difference to the fast-path function is that this can handle bit17
   * and uses non-atomic copy and kmap functions. */

 static int

drivers/gpu/drm/i915/i915_gem.c +1 -1
-shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length,
+shmem_pread_slow(struct page *page, int offset, int length,

Lets maybe call this shmem_pread_cpu? I understand it's been faster on recent generations even for swizzling. Can be added as follow-up.

Quoted 4 lines
drivers/gpu/drm/i915/i915_gem.c +4 -0
+static int
+i915_gem_shmem_pread(struct drm_i915_gem_object *obj,
+		     struct drm_i915_gem_pread *args)
+{

drivers/gpu/drm/i915/i915_gem.c shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length, · line 904

Quoted 10 lines
drivers/gpu/drm/i915/i915_gem.c +10 -0
+	char __user *user_data;
+	u64 remain;
+	unsigned int obj_do_bit17_swizzling;
+	unsigned int needs_clflush;
+	unsigned int idx, offset;
+	int ret;
+
+	obj_do_bit17_swizzling = 0;
+	if (i915_gem_object_needs_bit17_swizzle(obj))
+		obj_do_bit17_swizzling = 1 << 17;

Could use BIT(17) to make it super explicit.

Regards, Joonas

PS. Something like C context-aware diff would be super, would make reading these much more fun. Or maybe patchwork 2-way diff view.

Signature · 7 lines
Joonas Lahtinen
Open Source Technology Center
Intel Corporation
_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
#67 lore raw

PATCH 21/42 drm/i915: Implement pwrite without struct-mutex

In reply to Chris Wilson

We only need struct_mutex within pwrite for a brief window where we need to serialise with rendering and control our cache domains. Elsewhere we can rely on the backing storage being pinned, and forgive userspace any races against us.

---
 drivers/gpu/drm/i915/i915_gem.c | 346 ++++++++++++++--------------------------
 1 file changed, 120 insertions(+), 226 deletions(-)
drivers/gpu/drm/i915/i915_gem.c +120 -226
index 27f91714e82e..81f88103e6f5 100644
@@ -1121,10 +1121,10 @@ out:
  */
 
 static inline int
-fast_user_write(struct io_mapping *mapping,
-		loff_t page_base, int page_offset,
-		char __user *user_data,
-		int length)
+ggtt_write(struct io_mapping *mapping,
+	   loff_t page_base, int page_offset,
+	   char __user *user_data,
+	   int length)
 {
 	void __iomem *vaddr_atomic;
 	void *vaddr;
@@ -1132,60 +1132,42 @@ fast_user_write(struct io_mapping *mapping,
 
 	vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
 	/* We can use the cpu mem copy function because this is X86. */
-	vaddr = (void __force*)vaddr_atomic + page_offset;
+	vaddr = (void __force *)vaddr_atomic + page_offset;
 	unwritten = __copy_from_user_inatomic_nocache(vaddr,
 						      user_data, length);
 	io_mapping_unmap_atomic(vaddr_atomic);
-	return unwritten;
-}
 
-static inline unsigned long
-slow_user_access(struct io_mapping *mapping,
-		 unsigned long page_base, int page_offset,
-		 char __user *user_data,
-		 unsigned long length, bool pwrite)
-{
-	void __iomem *ioaddr;
-	void *vaddr;
-	unsigned long unwritten;
-
-	ioaddr = io_mapping_map_wc(mapping, page_base, PAGE_SIZE);
-	/* We can use the cpu mem copy function because this is X86. */
-	vaddr = (void __force *)ioaddr + page_offset;
-	if (pwrite)
-		unwritten = __copy_from_user(vaddr, user_data, length);
-	else
-		unwritten = __copy_to_user(user_data, vaddr, length);
+	if (unwritten) {
+		vaddr_atomic = io_mapping_map_wc(mapping, page_base, PAGE_SIZE);
+		/* We can use the cpu mem copy function because this is X86. */
+		vaddr = (void __force *)vaddr_atomic + page_offset;
+		unwritten = copy_from_user(vaddr, user_data, length);
+		io_mapping_unmap(vaddr_atomic);
+	}
 
-	io_mapping_unmap(ioaddr);
 	return unwritten;
 }
 
 /**
  * This is the fast pwrite path, where we copy the data directly from the
  * user into the GTT, uncached.
- * @i915: i915 device private data
- * @obj: i915 gem object
+ * @obj: i915 GEM object
  * @args: pwrite arguments structure
- * @file: drm file pointer
  */
 static int
-i915_gem_gtt_pwrite_fast(struct drm_i915_private *i915,
-			 struct drm_i915_gem_object *obj,
-			 struct drm_i915_gem_pwrite *args,
-			 struct drm_file *file)
+i915_gem_gtt_pwrite_fast(struct drm_i915_gem_object *obj,
+			 const struct drm_i915_gem_pwrite *args)
 {
-	struct i915_ggtt *ggtt = &i915->ggtt;
-	struct drm_device *dev = obj->base.dev;
-	struct i915_vma *vma;
+	struct i915_ggtt *ggtt = &to_i915(obj->base.dev)->ggtt;
 	struct drm_mm_node node;
-	uint64_t remain, offset;
-	char __user *user_data;
+	struct i915_vma *vma;
+	u64 remain, offset;
+	void __user *user_data;
 	int ret;
-	bool hit_slow_path = false;
 
-	if (i915_gem_object_is_tiled(obj))
-		return -EFAULT;
+	ret = mutex_lock_interruptible(&obj->base.dev->struct_mutex);
+	if (ret)
+		return ret;
 
 	vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0,
 				       PIN_MAPPABLE | PIN_NONBLOCK);
@@ -1201,21 +1183,17 @@ i915_gem_gtt_pwrite_fast(struct drm_i915_private *i915,
 	if (IS_ERR(vma)) {
 		ret = insert_mappable_node(ggtt, &node, PAGE_SIZE);
 		if (ret)
-			goto out;
-
-		ret = i915_gem_object_pin_pages(obj);
-		if (ret) {
-			remove_mappable_node(&node);
-			goto out;
-		}
+			goto out_unlock;
+		GEM_BUG_ON(!node.allocated);
 	}
 
 	ret = i915_gem_object_set_to_gtt_domain(obj, true);
 	if (ret)
 		goto out_unpin;
 
+	mutex_unlock(&obj->base.dev->struct_mutex);
+
 	intel_fb_obj_invalidate(obj, ORIGIN_CPU);
-	obj->mm.dirty = true;
 
 	user_data = u64_to_user_ptr(args->data_ptr);
 	offset = args->offset;
@@ -1246,92 +1224,36 @@ i915_gem_gtt_pwrite_fast(struct drm_i915_private *i915,
 		 * If the object is non-shmem backed, we retry again with the
 		 * path that handles page fault.
 		 */
-		if (fast_user_write(&ggtt->mappable, page_base,
-				    page_offset, user_data, page_length)) {
-			hit_slow_path = true;
-			mutex_unlock(&dev->struct_mutex);
-			if (slow_user_access(&ggtt->mappable,
-					     page_base,
-					     page_offset, user_data,
-					     page_length, true)) {
-				ret = -EFAULT;
-				mutex_lock(&dev->struct_mutex);
-				goto out_flush;
-			}
-
-			mutex_lock(&dev->struct_mutex);
+		if (ggtt_write(&ggtt->mappable, page_base, page_offset,
+			       user_data, page_length)) {
+			ret = -EFAULT;
+			break;
 		}
 
 		remain -= page_length;
 		user_data += page_length;
 		offset += page_length;
 	}
-
-out_flush:
-	if (hit_slow_path) {
-		if (ret == 0 &&
-		    (obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0) {
-			/* The user has modified the object whilst we tried
-			 * reading from it, and we now have no idea what domain
-			 * the pages should be in. As we have just been touching
-			 * them directly, flush everything back to the GTT
-			 * domain.
-			 */
-			ret = i915_gem_object_set_to_gtt_domain(obj, false);
-		}
-	}
-
 	intel_fb_obj_flush(obj, false, ORIGIN_CPU);
+
+	mutex_lock(&obj->base.dev->struct_mutex);
 out_unpin:
 	if (node.allocated) {
 		wmb();
 		ggtt->base.clear_range(&ggtt->base,
 				       node.start, node.size,
 				       true);
-		i915_gem_object_unpin_pages(obj);
 		remove_mappable_node(&node);
 	} else {
 		i915_vma_unpin(vma);
 	}
-out:
+out_unlock:
+	mutex_unlock(&obj->base.dev->struct_mutex);
 	return ret;
 }
 
-/* Per-page copy function for the shmem pwrite fastpath.
- * Flushes invalid cachelines before writing to the target if
- * needs_clflush_before is set and flushes out any written cachelines after
- * writing if needs_clflush is set. */
-static int
-shmem_pwrite_fast(struct page *page, int shmem_page_offset, int page_length,
-		  char __user *user_data,
-		  bool page_do_bit17_swizzling,
-		  bool needs_clflush_before,
-		  bool needs_clflush_after)
-{
-	char *vaddr;
-	int ret;
-
-	if (unlikely(page_do_bit17_swizzling))
-		return -EINVAL;
-
-	vaddr = kmap_atomic(page);
-	if (needs_clflush_before)
-		drm_clflush_virt_range(vaddr + shmem_page_offset,
-				       page_length);
-	ret = __copy_from_user_inatomic(vaddr + shmem_page_offset,
-					user_data, page_length);
-	if (needs_clflush_after)
-		drm_clflush_virt_range(vaddr + shmem_page_offset,
-				       page_length);
-	kunmap_atomic(vaddr);
-
-	return ret ? -EFAULT : 0;
-}
-
-/* Only difference to the fast-path function is that this can handle bit17
- * and uses non-atomic copy and kmap functions. */
 static int
-shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length,
+shmem_pwrite_slow(struct page *page, int offset, int length,
 		  char __user *user_data,
 		  bool page_do_bit17_swizzling,
 		  bool needs_clflush_before,
@@ -1342,124 +1264,113 @@ shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length,
 
 	vaddr = kmap(page);
 	if (unlikely(needs_clflush_before || page_do_bit17_swizzling))
-		shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
-					     page_length,
+		shmem_clflush_swizzled_range(vaddr + offset, length,
 					     page_do_bit17_swizzling);
 	if (page_do_bit17_swizzling)
-		ret = __copy_from_user_swizzled(vaddr, shmem_page_offset,
-						user_data,
-						page_length);
+		ret = __copy_from_user_swizzled(vaddr, offset, user_data,
+						length);
 	else
-		ret = __copy_from_user(vaddr + shmem_page_offset,
-				       user_data,
-				       page_length);
+		ret = __copy_from_user(vaddr + offset, user_data, length);
 	if (needs_clflush_after)
-		shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
-					     page_length,
+		shmem_clflush_swizzled_range(vaddr + offset, length,
 					     page_do_bit17_swizzling);
 	kunmap(page);
 
 	return ret ? -EFAULT : 0;
 }
 
+/* Per-page copy function for the shmem pwrite fastpath.
+ * Flushes invalid cachelines before writing to the target if
+ * needs_clflush_before is set and flushes out any written cachelines after
+ * writing if needs_clflush is set.
+ */
 static int
-i915_gem_shmem_pwrite(struct drm_device *dev,
-		      struct drm_i915_gem_object *obj,
-		      struct drm_i915_gem_pwrite *args,
-		      struct drm_file *file)
+shmem_pwrite(struct page *page, int offset, int len, char __user *user_data,
+	     bool page_do_bit17_swizzling,
+	     bool needs_clflush_before,
+	     bool needs_clflush_after)
 {
-	ssize_t remain;
-	loff_t offset;
-	char __user *user_data;
-	int shmem_page_offset, page_length, ret = 0;
-	int obj_do_bit17_swizzling, page_do_bit17_swizzling;
-	int hit_slowpath = 0;
+	int ret;
+
+	ret = -ENODEV;
+	if (!page_do_bit17_swizzling) {
+		char *vaddr = kmap_atomic(page);
+
+		if (needs_clflush_before)
+			drm_clflush_virt_range(vaddr + offset, len);
+		ret = __copy_from_user_inatomic(vaddr + offset, user_data, len);
+		if (needs_clflush_after)
+			drm_clflush_virt_range(vaddr + offset, len);
+
+		kunmap_atomic(vaddr);
+	}
+	if (ret == 0)
+		return ret;
+
+	return shmem_pwrite_slow(page, offset, len, user_data,
+				 page_do_bit17_swizzling,
+				 needs_clflush_before,
+				 needs_clflush_after);
+}
+
+static int
+i915_gem_shmem_pwrite(struct drm_i915_gem_object *obj,
+		      const struct drm_i915_gem_pwrite *args)
+{
+	void __user *user_data;
+	u64 remain;
+	unsigned int obj_do_bit17_swizzling;
+	unsigned int partial_cacheline_write;
 	unsigned int needs_clflush;
-	struct sg_page_iter sg_iter;
+	unsigned int offset, idx;
+	int ret;
 
-	ret = i915_gem_obj_prepare_shmem_write(obj, &needs_clflush);
+	ret = mutex_lock_interruptible(&obj->base.dev->struct_mutex);
 	if (ret)
 		return ret;
 
-	obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
-	user_data = u64_to_user_ptr(args->data_ptr);
-	offset = args->offset;
-	remain = args->size;
+	ret = i915_gem_obj_prepare_shmem_write(obj, &needs_clflush);
+	mutex_unlock(&obj->base.dev->struct_mutex);
+	if (ret)
+		return ret;
 
-	for_each_sg_page(obj->mm.pages->sgl, &sg_iter, obj->mm.pages->nents,
-			 offset >> PAGE_SHIFT) {
-		struct page *page = sg_page_iter_page(&sg_iter);
-		int partial_cacheline_write;
+	obj_do_bit17_swizzling = 0;
+	if (i915_gem_object_needs_bit17_swizzle(obj))
+		obj_do_bit17_swizzling = 1 << 17;
 
-		if (remain <= 0)
-			break;
+	/* If we don't overwrite a cacheline completely we need to be
+	 * careful to have up-to-date data by first clflushing. Don't
+	 * overcomplicate things and flush the entire patch.
+	 */
+	partial_cacheline_write = 0;
+	if (needs_clflush & CLFLUSH_BEFORE)
+		partial_cacheline_write = boot_cpu_data.x86_clflush_size - 1;
 
-		/* Operation in this page
-		 *
-		 * shmem_page_offset = offset within page in shmem file
-		 * page_length = bytes to copy for this page
-		 */
-		shmem_page_offset = offset_in_page(offset);
-
-		page_length = remain;
-		if ((shmem_page_offset + page_length) > PAGE_SIZE)
-			page_length = PAGE_SIZE - shmem_page_offset;
-
-		/* If we don't overwrite a cacheline completely we need to be
-		 * careful to have up-to-date data by first clflushing. Don't
-		 * overcomplicate things and flush the entire patch. */
-		partial_cacheline_write = needs_clflush & CLFLUSH_BEFORE &&
-			((shmem_page_offset | page_length)
-				& (boot_cpu_data.x86_clflush_size - 1));
-
-		page_do_bit17_swizzling = obj_do_bit17_swizzling &&
-			(page_to_phys(page) & (1 << 17)) != 0;
-
-		ret = shmem_pwrite_fast(page, shmem_page_offset, page_length,
-					user_data, page_do_bit17_swizzling,
-					partial_cacheline_write,
-					needs_clflush & CLFLUSH_AFTER);
-		if (ret == 0)
-			goto next_page;
-
-		hit_slowpath = 1;
-		mutex_unlock(&dev->struct_mutex);
-		ret = shmem_pwrite_slow(page, shmem_page_offset, page_length,
-					user_data, page_do_bit17_swizzling,
-					partial_cacheline_write,
-					needs_clflush & CLFLUSH_AFTER);
+	user_data = u64_to_user_ptr(args->data_ptr);
+	remain = args->size;
+	offset = offset_in_page(args->offset);
+	for (idx = args->offset >> PAGE_SHIFT; remain; idx++) {
+		struct page *page = i915_gem_object_get_page(obj, idx);
+		int length;
 
-		mutex_lock(&dev->struct_mutex);
+		length = remain;
+		if (offset + length > PAGE_SIZE)
+			length = PAGE_SIZE - offset;
 
+		ret = shmem_pwrite(page, offset, length, user_data,
+				   page_to_phys(page) & obj_do_bit17_swizzling,
+				   (offset | length) & partial_cacheline_write,
+				   needs_clflush & CLFLUSH_AFTER);
 		if (ret)
-			goto out;
-
-next_page:
-		remain -= page_length;
-		user_data += page_length;
-		offset += page_length;
-	}
-
-out:
-	i915_gem_obj_finish_shmem_access(obj);
+			break;
 
-	if (hit_slowpath) {
-		/*
-		 * Fixup: Flush cpu caches in case we didn't flush the dirty
-		 * cachelines in-line while writing and the object moved
-		 * out of the cpu write domain while we've dropped the lock.
-		 */
-		if (!(needs_clflush & CLFLUSH_AFTER) &&
-		    obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
-			if (i915_gem_clflush_object(obj, obj->pin_display))
-				needs_clflush |= CLFLUSH_AFTER;
-		}
+		remain -= length;
+		user_data += length;
+		offset = 0;
 	}
 
-	if (needs_clflush & CLFLUSH_AFTER)
-		i915_gem_chipset_flush(to_i915(dev));
-
 	intel_fb_obj_flush(obj, false, ORIGIN_CPU);
+	i915_gem_obj_finish_shmem_access(obj);
 	return ret;
 }
 
@@ -1475,7 +1386,6 @@ int
 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
 		      struct drm_file *file)
 {
-	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct drm_i915_gem_pwrite *args = data;
 	struct drm_i915_gem_object *obj;
 	int ret;
@@ -1488,13 +1398,6 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
 		       args->size))
 		return -EFAULT;
 
-	if (likely(!i915.prefault_disable)) {
-		ret = fault_in_multipages_readable(u64_to_user_ptr(args->data_ptr),
-						   args->size);
-		if (ret)
-			return -EFAULT;
-	}
-
 	obj = i915_gem_object_lookup(file, args->handle);
 	if (!obj)
 		return -ENOENT;
@@ -1516,11 +1419,9 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
 	if (ret)
 		goto err;
 
-	intel_runtime_pm_get(dev_priv);
-
-	ret = i915_mutex_lock_interruptible(dev);
+	ret = i915_gem_object_pin_pages(obj);
 	if (ret)
-		goto err_rpm;
+		goto err;
 
 	ret = -EFAULT;
 	/* We can only do the GTT pwrite on untiled buffers, as otherwise
@@ -1531,7 +1432,7 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
 	 */
 	if (!i915_gem_object_has_struct_page(obj) ||
 	    cpu_write_needs_clflush(obj)) {
-		ret = i915_gem_gtt_pwrite_fast(dev_priv, obj, args, file);
+		ret = i915_gem_gtt_pwrite_fast(obj, args);
 		/* Note that the gtt paths might fail with non-page-backed user
 		 * pointers (e.g. gtt mappings when moving data between
 		 * textures). Fallback to the shmem path in that case. */
@@ -1541,17 +1442,10 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
 		if (obj->phys_handle)
 			ret = i915_gem_phys_pwrite(obj, args, file);
 		else
-			ret = i915_gem_shmem_pwrite(dev, obj, args, file);
+			ret = i915_gem_shmem_pwrite(obj, args);
 	}
 
-	i915_gem_object_put(obj);
-	mutex_unlock(&dev->struct_mutex);
-	intel_runtime_pm_put(dev_priv);
-
-	return ret;
-
-err_rpm:
-	intel_runtime_pm_put(dev_priv);
+	i915_gem_object_unpin_pages(obj);
 err:
 	i915_gem_object_put_unlocked(obj);
 	return ret;
Signature · 6 lines
2.9.3

_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
maintainer Reviewed-by on v1 21/42
#68 lore raw

Re: [PATCH 21/42] drm/i915: Implement pwrite without struct-mutex

Quoted 6 lines · Chris Wilson

+/* Per-page copy function for the shmem pwrite fastpath.
+ * Flushes invalid cachelines before writing to the target if
+ * needs_clflush_before is set and flushes out any written cachelines after
+ * writing if needs_clflush is set.
+ */
 static int

drivers/gpu/drm/i915/i915_gem.c shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length, · line 1291

Quoted 8 lines
drivers/gpu/drm/i915/i915_gem.c +4 -4
-i915_gem_shmem_pwrite(struct drm_device *dev,
-		      struct drm_i915_gem_object *obj,
-		      struct drm_i915_gem_pwrite *args,
-		      struct drm_file *file)
+shmem_pwrite(struct page *page, int offset, int len, char __user *user_data,
+	     bool page_do_bit17_swizzling,
+	     bool needs_clflush_before,
+	     bool needs_clflush_after)

I remember having complaints of two bool arguments in same func. Could these tree be fixed while mangling the code otherwise too? Or as a follow-up.

drivers/gpu/drm/i915/i915_gem.c shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length, · line 1323

Quoted 8 lines
drivers/gpu/drm/i915/i915_gem.c +8 -0
+static int
+i915_gem_shmem_pwrite(struct drm_i915_gem_object *obj,
+		      const struct drm_i915_gem_pwrite *args)
+{
+	void __user *user_data;
+	u64 remain;
+	unsigned int obj_do_bit17_swizzling;
+	unsigned int partial_cacheline_write;

partial_cacheline_mask might be more descriptive?

Regards, Joonas

Signature · 7 lines
Joonas Lahtinen
Open Source Technology Center
Intel Corporation
_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
#69 lore raw

Re: [PATCH 21/42] drm/i915: Implement pwrite without struct-mutex

Quoted 11 lines · Joonas Lahtinen
Quoted 10 lines · Chris Wilson

+/* Per-page copy function for the shmem pwrite fastpath.
+ * Flushes invalid cachelines before writing to the target if
+ * needs_clflush_before is set and flushes out any written cachelines after
+ * writing if needs_clflush is set.
+ */
 static int

drivers/gpu/drm/i915/i915_gem.c +0 -4
-i915_gem_shmem_pwrite(struct drm_device *dev,
-		      struct drm_i915_gem_object *obj,
-		      struct drm_i915_gem_pwrite *args,
-		      struct drm_file *file)
Quoted 8 lines

drivers/gpu/drm/i915/i915_gem.c shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length, · line 1291

Quoted 4 lines
drivers/gpu/drm/i915/i915_gem.c +4 -0
+shmem_pwrite(struct page *page, int offset, int len, char __user *user_data,
+	     bool page_do_bit17_swizzling,
+	     bool needs_clflush_before,
+	     bool needs_clflush_after)

I remember having complaints of two bool arguments in same func. Could these tree be fixed while mangling the code otherwise too? Or as a follow-up.

They are three different evaluations on each loop. Looks to be bit of a hassle to amalgamate them into one in the caller. Though

(pages_to_phys() & BIT(17)) |
((offset | length) & 63) |
(needs_clflush & 2)

almost works (just need to choose another bit for needs_clflush). I'm not convinced you'd like the unpacking in shmem_pwrite() ;) Something like:

static int
shmem_pwrite_slow(struct page *page, int offset, int length,
char __user *user_data,
unsigned flags)

{
        char *vaddr;
        int ret;

        vaddr = kmap(page);
        if (unlikely(flags & (BIT(17) | 63)))
                shmem_clflush_swizzled_range(vaddr + offset, length, flags);
        if (flags & BIT(17))
                ret = __copy_from_user_swizzled(vaddr, offset, user_data,
                                                length);
        else
                ret = __copy_from_user(vaddr + offset, user_data, length);
        if (flags & BIT(16))
                shmem_clflush_swizzled_range(vaddr + offset, length, flags));
        kunmap(page);

        return ret ? -EFAULT : 0;
}
Quoted 10 lines

drivers/gpu/drm/i915/i915_gem.c shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length, · line 1323

Quoted 8 lines
drivers/gpu/drm/i915/i915_gem.c +8 -0
+static int
+i915_gem_shmem_pwrite(struct drm_i915_gem_object *obj,
+		      const struct drm_i915_gem_pwrite *args)
+{
+	void __user *user_data;
+	u64 remain;
+	unsigned int obj_do_bit17_swizzling;
+	unsigned int partial_cacheline_write;

partial_cacheline_mask might be more descriptive?

To me that says we want to mask off the partial cacheline, or use them constructively. Imho,

len & partial_cacheline_write

reads more clearly as a boolean question than

len & partial_cacheline_mask.

-Chris

Signature · 5 lines
Chris Wilson, Intel Open Source Technology Centre
_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
maintainer
#70 lore raw

Re: [PATCH 21/42] drm/i915: Implement pwrite without struct-mutex

Quoted 8 lines · Chris Wilson

To me that says we want to mask off the partial cacheline, or use them constructively. Imho,

len & partial_cacheline_write

reads more clearly as a boolean question than

len & partial_cacheline_mask.

For me they convey the same value in this context, but alone, partial_cacheline_write sounds like bool. You can combine it too into write_mask as an option, too. But not a big deal after all.

Regards, joonas

Signature · 7 lines
Joonas Lahtinen
Open Source Technology Center
Intel Corporation
_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
#71 lore raw

PATCH 22/42 drm/i915: Acquire the backing storage outside of struct_mutex in set-domain

In reply to Chris Wilson

As we can locklessly (well struct_mutex-lessly) acquire the backing storage, do so in set-domain-ioctl to reduce the contention on the struct_mutex.

---
 drivers/gpu/drm/i915/i915_gem.c | 83 ++++++++++++++++++++++++++---------------
 1 file changed, 53 insertions(+), 30 deletions(-)
drivers/gpu/drm/i915/i915_gem.c +53 -30
index 81f88103e6f5..80069f0fa9ac 100644
@@ -1458,6 +1458,30 @@ write_origin(struct drm_i915_gem_object *obj, unsigned domain)
 		obj->frontbuffer_ggtt_origin : ORIGIN_CPU);
 }
 
+static void i915_gem_object_bump_inactive_ggtt(struct drm_i915_gem_object *obj)
+{
+	struct drm_i915_private *i915;
+	struct list_head *list;
+	struct i915_vma *vma;
+
+	list_for_each_entry(vma, &obj->vma_list, obj_link) {
+		if (!i915_vma_is_ggtt(vma))
+			continue;
+
+		if (i915_vma_is_active(vma))
+			continue;
+
+		if (!drm_mm_node_allocated(&vma->node))
+			continue;
+
+		list_move_tail(&vma->vm_link, &vma->vm->inactive_list);
+	}
+
+	i915 = to_i915(obj->base.dev);
+	list = obj->bind_count ? &i915->mm.bound_list : &i915->mm.unbound_list;
+	list_move_tail(&obj->global_list, list);
+}
+
 /**
  * Called when user space prepares to use an object with the CPU, either
  * through the mmap ioctl's mapping or a GTT mapping.
@@ -1499,25 +1523,40 @@ i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
 				   MAX_SCHEDULE_TIMEOUT,
 				   to_rps_client(file));
 	if (ret)
-		goto err;
+		goto err_unlocked;
+
+	/* Flush and acquire obj->pages so that we are coherent through
+	 * direct access in memory with previous cached writes through
+	 * shmemfs and that our cache domain tracking remains valid.
+	 * For example, if the obj->filp was moved to swap without us
+	 * being notified and releasing the pages, we would mistakenly
+	 * continue to assume that the obj remained out of the CPU cached
+	 * domain.
+	 */
+	ret = i915_gem_object_pin_pages(obj);
+	if (ret)
+		goto err_unlocked;
 
 	ret = i915_mutex_lock_interruptible(dev);
 	if (ret)
-		goto err;
+		goto err_pages;
 
 	if (read_domains & I915_GEM_DOMAIN_GTT)
 		ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
 	else
 		ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
 
-	if (write_domain != 0)
-		intel_fb_obj_invalidate(obj, write_origin(obj, write_domain));
+	/* And bump the LRU for this access */
+	i915_gem_object_bump_inactive_ggtt(obj);
 
-	i915_gem_object_put(obj);
 	mutex_unlock(&dev->struct_mutex);
-	return ret;
 
-err:
+	if (write_domain != 0)
+		intel_fb_obj_invalidate(obj, write_origin(obj, write_domain));
+
+err_pages:
+	i915_gem_object_unpin_pages(obj);
+err_unlocked:
 	i915_gem_object_put_unlocked(obj);
 	return ret;
 }
@@ -1740,6 +1779,10 @@ int i915_gem_fault(struct vm_area_struct *area, struct vm_fault *vmf)
 	if (ret)
 		goto err;
 
+	ret = i915_gem_object_pin_pages(obj);
+	if (ret)
+		goto err;
+
 	intel_runtime_pm_get(dev_priv);
 
 	ret = i915_mutex_lock_interruptible(dev);
@@ -1821,6 +1864,7 @@ err_unlock:
 	mutex_unlock(&dev->struct_mutex);
 err_rpm:
 	intel_runtime_pm_put(dev_priv);
+	i915_gem_object_unpin_pages(obj);
 err:
 	switch (ret) {
 	case -EIO:
@@ -3214,24 +3258,6 @@ i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
 					    I915_GEM_DOMAIN_CPU);
 }
 
-static void i915_gem_object_bump_inactive_ggtt(struct drm_i915_gem_object *obj)
-{
-	struct i915_vma *vma;
-
-	list_for_each_entry(vma, &obj->vma_list, obj_link) {
-		if (!i915_vma_is_ggtt(vma))
-			continue;
-
-		if (i915_vma_is_active(vma))
-			continue;
-
-		if (!drm_mm_node_allocated(&vma->node))
-			continue;
-
-		list_move_tail(&vma->vm_link, &vma->vm->inactive_list);
-	}
-}
-
 /**
  * Moves a single object to the GTT read, and possibly write domain.
  * @obj: object to act on
@@ -3287,7 +3313,7 @@ i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
 	/* It should now be out of any other write domains, and we can update
 	 * the domain values for our changes.
 	 */
-	BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
+	GEM_BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
 	obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
 	if (write) {
 		obj->base.read_domains = I915_GEM_DOMAIN_GTT;
@@ -3299,10 +3325,7 @@ i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
 					    old_read_domains,
 					    old_write_domain);
 
-	/* And bump the LRU for this access */
-	i915_gem_object_bump_inactive_ggtt(obj);
 	i915_gem_object_unpin_pages(obj);
-
 	return 0;
 }
 
@@ -3666,7 +3689,7 @@ i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
 	/* It should now be out of any other write domains, and we can update
 	 * the domain values for our changes.
 	 */
-	BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
+	GEM_BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
 
 	/* If we're writing through the CPU, then the GPU read domains will
 	 * need to be invalidated at next use.
Signature · 6 lines
2.9.3

_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
maintainer Reviewed-by on v1 22/42
#72 lore raw

Re: [PATCH 22/42] drm/i915: Acquire the backing storage outside of struct_mutex in set-domain

Quoted 38 lines · Chris Wilson
drivers/gpu/drm/i915/i915_gem.c +13 -1
@@ -1499,25 +1523,40 @@ i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
  				   MAX_SCHEDULE_TIMEOUT,
  				   to_rps_client(file));
  	if (ret)
-		goto err;
+		goto err_unlocked;
+
+	/* Flush and acquire obj->pages so that we are coherent through
+	 * direct access in memory with previous cached writes through
+	 * shmemfs and that our cache domain tracking remains valid.
+	 * For example, if the obj->filp was moved to swap without us
+	 * being notified and releasing the pages, we would mistakenly
+	 * continue to assume that the obj remained out of the CPU cached
+	 * domain.
+	 */
+	ret = i915_gem_object_pin_pages(obj);
+	if (ret)
+		goto err_unlocked;


  ret = i915_mutex_lock_interruptible(dev);
  if (ret)

(quoted diff) +1 -1
-		goto err;
+		goto err_pages;


  if (read_domains & I915_GEM_DOMAIN_GTT)
  ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0); else
  ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);

drivers/gpu/drm/i915/i915_gem.c +2 -2
-	if (write_domain != 0)
-		intel_fb_obj_invalidate(obj, write_origin(obj, write_domain));
+	/* And bump the LRU for this access */
+	i915_gem_object_bump_inactive_ggtt(obj);

drivers/gpu/drm/i915/i915_gem.c +0 -1
-	i915_gem_object_put(obj);

  mutex_unlock(&dev->struct_mutex);

drivers/gpu/drm/i915/i915_gem.c +0 -1
-	return ret;

drivers/gpu/drm/i915/i915_gem.c i915_gem_set_domain_ioctl(struct drm_device *dev, void *data, · line 1557

Quoted 5 lines
drivers/gpu/drm/i915/i915_gem.c +4 -1
-err:
+	if (write_domain != 0)
+		intel_fb_obj_invalidate(obj, write_origin(obj, write_domain));
+
+err_pages:

out_pages: as it's a shared path now.

Code motion would still be great as separate patches.

Regards, Joonas

Signature · 7 lines
Joonas Lahtinen
Open Source Technology Center
Intel Corporation
_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
#73 lore raw

PATCH 23/42 drm/i915: Move object release to a freelist + worker

In reply to Chris Wilson

We want to hide the latency of releasing objects and their backing storage from the submission, so we move the actual free to a worker. This allows us to switch to struct_mutex freeing of the object in the next patch.

Furthermore, if we know that the object we are dereferencing remains valid for the duration of our access, we can forgo the usual synchronisation barriers and atomic reference counting. To ensure this we defer freeing an object til after an RCU grace period, such that any lookup of the object within an RCU read critical section will remain valid until after we exit that critical section. We also employ this delay for rate-limiting the serialisation on reallocation - we have to slow down object creation in order to prevent resource starvation (in particular, files).

---
 drivers/gpu/drm/i915/i915_debugfs.c      |  15 ++-
 drivers/gpu/drm/i915/i915_drv.c          |   3 +
 drivers/gpu/drm/i915/i915_drv.h          |  44 +++++++-
 drivers/gpu/drm/i915/i915_gem.c          | 166 +++++++++++++++++++++----------
 drivers/gpu/drm/i915/i915_gem_shrinker.c |  14 ++-
 drivers/gpu/drm/i915/i915_gem_tiling.c   |  21 ++--
 6 files changed, 193 insertions(+), 70 deletions(-)
drivers/gpu/drm/i915/i915_debugfs.c +11 -4
index b807ddf65e04..144013875513 100644
@@ -4872,10 +4872,12 @@ DEFINE_SIMPLE_ATTRIBUTE(i915_ring_test_irq_fops,
 #define DROP_BOUND 0x2
 #define DROP_RETIRE 0x4
 #define DROP_ACTIVE 0x8
-#define DROP_ALL (DROP_UNBOUND | \
-		  DROP_BOUND | \
-		  DROP_RETIRE | \
-		  DROP_ACTIVE)
+#define DROP_FREED 0x10
+#define DROP_ALL (DROP_UNBOUND	| \
+		  DROP_BOUND	| \
+		  DROP_RETIRE	| \
+		  DROP_ACTIVE	| \
+		  DROP_FREED)
 static int
 i915_drop_caches_get(void *data, u64 *val)
 {
@@ -4919,6 +4921,11 @@ i915_drop_caches_set(void *data, u64 val)
 unlock:
 	mutex_unlock(&dev->struct_mutex);
 
+	if (val & DROP_FREED) {
+		synchronize_rcu();
+		flush_work(&dev_priv->mm.free_work);
+	}
+
 	return ret;
 }
 
drivers/gpu/drm/i915/i915_drv.c +3 -0
index 89d322215c84..f7d48f97993d 100644
@@ -563,6 +563,9 @@ static void i915_gem_fini(struct drm_device *dev)
 	i915_gem_context_fini(dev);
 	mutex_unlock(&dev->struct_mutex);
 
+	synchronize_rcu();
+	flush_work(&dev_priv->mm.free_work);
+
 	WARN_ON(!list_empty(&to_i915(dev)->context_list));
 }
 
drivers/gpu/drm/i915/i915_drv.h +41 -3

68 lines not on this page · raw message

drivers/gpu/drm/i915/i915_gem.c +116 -50

253 lines not on this page · raw message

drivers/gpu/drm/i915/i915_gem_shrinker.c +10 -4
index 868da0bae751..fa72473dc7f5 100644
@@ -201,6 +201,10 @@ i915_gem_shrink(struct drm_i915_private *dev_priv,
 						       typeof(*obj),
 						       global_list))) {
 			list_move_tail(&obj->global_list, &still_in_list);
+			if (!obj->mm.pages) {
+				list_del_init(&obj->global_list);
+				continue;
+			}
 
 			if (flags & I915_SHRINK_PURGEABLE &&
 			    obj->mm.madv != I915_MADV_DONTNEED)
@@ -217,8 +221,6 @@ i915_gem_shrink(struct drm_i915_private *dev_priv,
 			if (!can_release_pages(obj))
 				continue;
 
-			i915_gem_object_get(obj);
-
 			if (unsafe_drop_pages(obj)) {
 				mutex_lock(&obj->mm.lock);
 				if (!obj->mm.pages) {
@@ -227,8 +229,6 @@ i915_gem_shrink(struct drm_i915_private *dev_priv,
 				}
 				mutex_unlock(&obj->mm.lock);
 			}
-
-			i915_gem_object_put(obj);
 		}
 		list_splice(&still_in_list, phase->list);
 	}
@@ -395,12 +395,18 @@ i915_gem_shrinker_oom(struct notifier_block *nb, unsigned long event, void *ptr)
 	 */
 	unbound = bound = unevictable = 0;
 	list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_list) {
+		if (!obj->mm.pages)
+			continue;
+
 		if (!can_release_pages(obj))
 			unevictable += obj->base.size >> PAGE_SHIFT;
 		else
 			unbound += obj->base.size >> PAGE_SHIFT;
 	}
 	list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
+		if (!obj->mm.pages)
+			continue;
+
 		if (!can_release_pages(obj))
 			unevictable += obj->base.size >> PAGE_SHIFT;
 		else
drivers/gpu/drm/i915/i915_gem_tiling.c +12 -9
index 11e2e0f57ac1..ec48e403adfe 100644
@@ -328,12 +328,17 @@ i915_gem_get_tiling(struct drm_device *dev, void *data,
 	struct drm_i915_gem_get_tiling *args = data;
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct drm_i915_gem_object *obj;
+	int err = -ENOENT;
+
+	rcu_read_lock();
+	obj = i915_gem_object_lookup_rcu(file, args->handle);
+	if (obj) {
+		args->tiling_mode =
+			READ_ONCE(obj->tiling_and_stride) & TILING_MASK;
+		err = 0;
+	}
+	rcu_read_unlock();
 
-	obj = i915_gem_object_lookup(file, args->handle);
-	if (!obj)
-		return -ENOENT;
-
-	args->tiling_mode = READ_ONCE(obj->tiling_and_stride) & TILING_MASK;
 	switch (args->tiling_mode) {
 	case I915_TILING_X:
 		args->swizzle_mode = dev_priv->mm.bit_6_swizzle_x;
@@ -341,11 +346,10 @@ i915_gem_get_tiling(struct drm_device *dev, void *data,
 	case I915_TILING_Y:
 		args->swizzle_mode = dev_priv->mm.bit_6_swizzle_y;
 		break;
+	default:
 	case I915_TILING_NONE:
 		args->swizzle_mode = I915_BIT_6_SWIZZLE_NONE;
 		break;
-	default:
-		DRM_ERROR("unknown tiling mode\n");
 	}
 
 	/* Hide bit 17 from the user -- see comment in i915_gem_set_tiling */
@@ -358,6 +362,5 @@ i915_gem_get_tiling(struct drm_device *dev, void *data,
 	if (args->swizzle_mode == I915_BIT_6_SWIZZLE_9_10_17)
 		args->swizzle_mode = I915_BIT_6_SWIZZLE_9_10;
 
-	i915_gem_object_put_unlocked(obj);
-	return 0;
+	return err;
 }
Signature · 6 lines
2.9.3

_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
#74 lore raw

Re: [PATCH 23/42] drm/i915: Move object release to a freelist + worker

Quoted 477 lines · Chris Wilson

We want to hide the latency of releasing objects and their backing storage from the submission, so we move the actual free to a worker. This allows us to switch to struct_mutex freeing of the object in the next patch.

Furthermore, if we know that the object we are dereferencing remains valid for the duration of our access, we can forgo the usual synchronisation barriers and atomic reference counting. To ensure this we defer freeing an object til after an RCU grace period, such that any lookup of the object within an RCU read critical section will remain valid until after we exit that critical section. We also employ this delay for rate-limiting the serialisation on reallocation - we have to slow down object creation in order to prevent resource starvation (in particular, files).

---
  drivers/gpu/drm/i915/i915_debugfs.c      |  15 ++-
  drivers/gpu/drm/i915/i915_drv.c          |   3 +
  drivers/gpu/drm/i915/i915_drv.h          |  44 +++++++-
  drivers/gpu/drm/i915/i915_gem.c          | 166 +++++++++++++++++++++----------
  drivers/gpu/drm/i915/i915_gem_shrinker.c |  14 ++-
  drivers/gpu/drm/i915/i915_gem_tiling.c   |  21 ++--
  6 files changed, 193 insertions(+), 70 deletions(-)
drivers/gpu/drm/i915/i915_debugfs.c +11 -4
index b807ddf65e04..144013875513 100644
@@ -4872,10 +4872,12 @@ DEFINE_SIMPLE_ATTRIBUTE(i915_ring_test_irq_fops,
  #define DROP_BOUND 0x2
  #define DROP_RETIRE 0x4
  #define DROP_ACTIVE 0x8
-#define DROP_ALL (DROP_UNBOUND | \
-		  DROP_BOUND | \
-		  DROP_RETIRE | \
-		  DROP_ACTIVE)
+#define DROP_FREED 0x10
+#define DROP_ALL (DROP_UNBOUND	| \
+		  DROP_BOUND	| \
+		  DROP_RETIRE	| \
+		  DROP_ACTIVE	| \
+		  DROP_FREED)
  static int
  i915_drop_caches_get(void *data, u64 *val)
  {
@@ -4919,6 +4921,11 @@ i915_drop_caches_set(void *data, u64 val)
  unlock:
  	mutex_unlock(&dev->struct_mutex);
 
+	if (val & DROP_FREED) {
+		synchronize_rcu();
+		flush_work(&dev_priv->mm.free_work);
+	}
+
  	return ret;
  }
 
drivers/gpu/drm/i915/i915_drv.c +3 -0
index 89d322215c84..f7d48f97993d 100644
@@ -563,6 +563,9 @@ static void i915_gem_fini(struct drm_device *dev)
  	i915_gem_context_fini(dev);
  	mutex_unlock(&dev->struct_mutex);
 
+	synchronize_rcu();
+	flush_work(&dev_priv->mm.free_work);
+
  	WARN_ON(!list_empty(&to_i915(dev)->context_list));
  }
 
drivers/gpu/drm/i915/i915_drv.h +41 -3
index 7e9df190c891..e79a5cb78b5d 100644
@@ -1354,11 +1354,17 @@ struct i915_gem_mm {
  	struct list_head bound_list;
  	/**
  	 * List of objects which are not bound to the GTT (thus
-	 * are idle and not used by the GPU) but still have
-	 * (presumably uncached) pages still attached.
+	 * are idle and not used by the GPU). These objects may or may
+	 * not actually have any pages attached.
  	 */
  	struct list_head unbound_list;
 
+	/**
+	 * List of objects which are pending destruction.
+	 */
+	struct llist_head free_list;
+	struct work_struct free_work;
+
  	/** Usable portion of the GTT for GEM */
  	unsigned long stolen_base; /* limited to low memory (32-bit) */
 
@@ -2214,6 +2220,10 @@ struct drm_i915_gem_object {
  	/** Stolen memory for this object, instead of being backed by shmem. */
  	struct drm_mm_node *stolen;
  	struct list_head global_list;
+	union {
+		struct rcu_head rcu;
+		struct llist_node freed;
+	};
 
  	/** Used in execbuf to temporarily hold a ref */
  	struct list_head obj_exec_link;
@@ -2333,10 +2343,38 @@ to_intel_bo(struct drm_gem_object *gem)
  	return container_of(gem, struct drm_i915_gem_object, base);
  }
 
+/**
+ * i915_gem_object_lookup_rcu - look up a temporary GEM object from its handle
+ * @filp: DRM file private date
+ * @handle: userspace handle
+ *
+ * Returns:
+ *
+ * A pointer to the object named by the handle if such exists on @filp, NULL
+ * otherwise. This object is only valid whilst under the RCU read lock, and
+ * note carefully the object may be in the process of being destroyed.
+ */
+static inline struct drm_i915_gem_object *
+i915_gem_object_lookup_rcu(struct drm_file *file, u32 handle)
+{
+#ifdef CONFIG_LOCKDEP
+	WARN_ON(!lock_is_held(&rcu_lock_map));
+#endif
+	return idr_find(&file->object_idr, handle);
+}
+
  static inline struct drm_i915_gem_object *
  i915_gem_object_lookup(struct drm_file *file, u32 handle)
  {
-	return to_intel_bo(drm_gem_object_lookup(file, handle));
+	struct drm_i915_gem_object *obj;
+
+	rcu_read_lock();
+	obj = i915_gem_object_lookup_rcu(file, handle);
+	if (obj && !kref_get_unless_zero(&obj->base.refcount))
+		obj = NULL;
+	rcu_read_unlock();
+
+	return obj;
  }
 
  __deprecated
drivers/gpu/drm/i915/i915_gem.c +116 -50
index 80069f0fa9ac..7cd49dd1d3f8 100644
@@ -42,6 +42,7 @@
  #include <linux/pci.h>
  #include <linux/dma-buf.h>
 
+static void i915_gem_flush_free_objects(struct drm_i915_private *i915);
  static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
  static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
 
@@ -647,6 +648,8 @@ i915_gem_create_ioctl(struct drm_device *dev, void *data,
  {
  	struct drm_i915_gem_create *args = data;
 
+	i915_gem_flush_free_objects(to_i915(dev));
+
  	return i915_gem_create(file, dev,
  			       args->size, &args->handle);
  }
@@ -3469,10 +3472,14 @@ int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data,
  {
  	struct drm_i915_gem_caching *args = data;
  	struct drm_i915_gem_object *obj;
+	int err = 0;
 
-	obj = i915_gem_object_lookup(file, args->handle);
-	if (!obj)
-		return -ENOENT;
+	rcu_read_lock();
+	obj = i915_gem_object_lookup_rcu(file, args->handle);
+	if (!obj) {
+		err = -ENOENT;
+		goto out;
+	}
 
  	switch (obj->cache_level) {
  	case I915_CACHE_LLC:
@@ -3488,9 +3495,9 @@ int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data,
  		args->caching = I915_CACHING_NONE;
  		break;
  	}
-
-	i915_gem_object_put_unlocked(obj);
-	return 0;
+out:
+	rcu_read_unlock();
+	return err;
  }
 
  int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data,
@@ -4001,10 +4008,14 @@ i915_gem_busy_ioctl(struct drm_device *dev, void *data,
  	struct drm_i915_gem_busy *args = data;
  	struct drm_i915_gem_object *obj;
  	unsigned long active;
+	int err;
 
-	obj = i915_gem_object_lookup(file, args->handle);
-	if (!obj)
-		return -ENOENT;
+	rcu_read_lock();
+	obj = i915_gem_object_lookup_rcu(file, args->handle);
+	if (!obj) {
+		err = -ENOENT;
+		goto out;
+	}
 
  	args->busy = 0;
  	active = __I915_BO_ACTIVE(obj);
@@ -4034,7 +4045,6 @@ i915_gem_busy_ioctl(struct drm_device *dev, void *data,
  		 * are busy is not completely reliable - we only guarantee
  		 * that the object was busy.
  		 */
-		rcu_read_lock();
 
  		for_each_active(active, idx)
  			args->busy |= busy_check_reader(&obj->last_read[idx]);
@@ -4052,12 +4062,11 @@ i915_gem_busy_ioctl(struct drm_device *dev, void *data,
  		 * the result.
  		 */
  		args->busy |= busy_check_writer(&obj->last_write);
-
-		rcu_read_unlock();
  	}
 
-	i915_gem_object_put_unlocked(obj);
-	return 0;
+out:
+	rcu_read_unlock();
+	return err;
  }
 
  int
@@ -4204,7 +4213,6 @@ struct drm_i915_gem_object *i915_gem_object_create(struct drm_device *dev,
 
  fail:
  	i915_gem_object_free(obj);
-
  	return ERR_PTR(ret);
  }
 
@@ -4232,16 +4240,69 @@ static bool discard_backing_storage(struct drm_i915_gem_object *obj)
  	return atomic_long_read(&obj->base.filp->f_count) == 1;
  }
 
-void i915_gem_free_object(struct drm_gem_object *gem_obj)
+static void __i915_gem_free_objects(struct drm_i915_private *i915,
+				    struct llist_node *freed)
  {
-	struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
-	struct drm_device *dev = obj->base.dev;
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct i915_vma *vma, *next;
+	struct drm_i915_gem_object *obj, *on;
 
-	intel_runtime_pm_get(dev_priv);
+	mutex_lock(&i915->drm.struct_mutex);
+	intel_runtime_pm_get(i915);
+	llist_for_each_entry(obj, freed, freed) {
+		struct i915_vma *vma, *vn;
+
+		trace_i915_gem_object_destroy(obj);
+
+		GEM_BUG_ON(i915_gem_object_is_active(obj));
+		list_for_each_entry_safe(vma, vn,
+					 &obj->vma_list, obj_link) {
+			GEM_BUG_ON(!i915_vma_is_ggtt(vma));
+			GEM_BUG_ON(i915_vma_is_active(vma));
+			vma->flags &= ~I915_VMA_PIN_MASK;
+			i915_vma_close(vma);
+		}
+
+		list_del(&obj->global_list);
+	}
+	intel_runtime_pm_put(i915);
+	mutex_unlock(&i915->drm.struct_mutex);
+
+	llist_for_each_entry_safe(obj, on, freed, freed) {
+		GEM_BUG_ON(obj->bind_count);
+		GEM_BUG_ON(atomic_read(&obj->frontbuffer_bits));
+
+		if (obj->ops->release)
+			obj->ops->release(obj);
+
+		if (WARN_ON(i915_gem_object_has_pinned_pages(obj)))
+			atomic_set(&obj->mm.pages_pin_count, 0);
+		__i915_gem_object_put_pages(obj);
+		GEM_BUG_ON(obj->mm.pages);
+
+		if (obj->base.import_attach)
+			drm_prime_gem_destroy(&obj->base, NULL);
+
+		drm_gem_object_release(&obj->base);
+		i915_gem_info_remove_obj(i915, obj->base.size);
+
+		kfree(obj->bit_17);
+		i915_gem_object_free(obj);
+	}
+}
+
+static void i915_gem_flush_free_objects(struct drm_i915_private *i915)
+{
+	struct llist_node *freed;
 
-	trace_i915_gem_object_destroy(obj);
+	freed = llist_del_all(&i915->mm.free_list);
+	if (unlikely(freed))
+		__i915_gem_free_objects(i915, freed);
+}
+
+static void __i915_gem_free_work(struct work_struct *work)
+{
+	struct drm_i915_private *i915 =
+		container_of(work, struct drm_i915_private, mm.free_work);
+	struct llist_node *freed;
 
  	/* All file-owned VMA should have been released by this point through
  	 * i915_gem_close_object(), or earlier by i915_gem_context_close().
@@ -4250,42 +4311,44 @@ void i915_gem_free_object(struct drm_gem_object *gem_obj)
  	 * the GTT either for the user or for scanout). Those VMA still need to
  	 * unbound now.
  	 */
-	list_for_each_entry_safe(vma, next, &obj->vma_list, obj_link) {
-		GEM_BUG_ON(!i915_vma_is_ggtt(vma));
-		GEM_BUG_ON(i915_vma_is_active(vma));
-		vma->flags &= ~I915_VMA_PIN_MASK;
-		i915_vma_close(vma);
-	}
-	GEM_BUG_ON(obj->bind_count);
 
-	WARN_ON(atomic_read(&obj->frontbuffer_bits));
+	while ((freed = llist_del_all(&i915->mm.free_list)))
+		__i915_gem_free_objects(i915, freed);
+}
 
-	if (obj->mm.pages && obj->mm.madv == I915_MADV_WILLNEED &&
-	    dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES &&
-	    i915_gem_object_is_tiled(obj))
-		__i915_gem_object_unpin_pages(obj);
+static void __i915_gem_free_object_rcu(struct rcu_head *head)
+{
+	struct drm_i915_gem_object *obj =
+		container_of(head, typeof(*obj), rcu);
+	struct drm_i915_private *i915 = to_i915(obj->base.dev);
+
+	/* We can't simply use call_rcu() from i915_gem_free_object()
+	 * as we need to block whilst unbinding, and the call_rcu
+	 * task may be called from softirq context. So we take a
+	 * detour through a worker.
+	 */
+	if (llist_add(&obj->freed, &i915->mm.free_list))
+		schedule_work(&i915->mm.free_work);
+}
 
-	if (obj->ops->release)
-		obj->ops->release(obj);
+void i915_gem_free_object(struct drm_gem_object *gem_obj)
+{
+	struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
 
-	if (WARN_ON(i915_gem_object_has_pinned_pages(obj)))
-		atomic_set(&obj->mm.pages_pin_count, 0);
  	if (discard_backing_storage(obj))
  		obj->mm.madv = I915_MADV_DONTNEED;
-	__i915_gem_object_put_pages(obj);
 
-	GEM_BUG_ON(obj->mm.pages);
-
-	if (obj->base.import_attach)
-		drm_prime_gem_destroy(&obj->base, NULL);
-
-	drm_gem_object_release(&obj->base);
-	i915_gem_info_remove_obj(dev_priv, obj->base.size);
-
-	kfree(obj->bit_17);
-	i915_gem_object_free(obj);
+	if (obj->mm.pages && obj->mm.madv == I915_MADV_WILLNEED &&
+	    to_i915(obj->base.dev)->quirks & QUIRK_PIN_SWIZZLED_PAGES &&
+	    i915_gem_object_is_tiled(obj))
+		__i915_gem_object_unpin_pages(obj);
 
-	intel_runtime_pm_put(dev_priv);
+	/* Before we free the object, make sure any pure RCU-only
+	 * read-side critical sections are complete, e.g.
+	 * i915_gem_busy_ioctl(). For the corresponding synchronized
+	 * lookup see i915_gem_object_lookup_rcu().
+	 */
+	call_rcu(&obj->rcu, __i915_gem_free_object_rcu);
  }
 
  void __i915_gem_object_release_unless_active(struct drm_i915_gem_object *obj)
@@ -4334,6 +4397,7 @@ int i915_gem_suspend(struct drm_device *dev)
  	cancel_delayed_work_sync(&dev_priv->gpu_error.hangcheck_work);
  	cancel_delayed_work_sync(&dev_priv->gt.retire_work);
  	flush_delayed_work(&dev_priv->gt.idle_work);
+	flush_work(&dev_priv->mm.free_work);
 
  	/* Assert that we sucessfully flushed all the work and
  	 * reset the GPU back to its idle, low power state.
@@ -4630,6 +4694,8 @@ i915_gem_load_init(struct drm_device *dev)
  				  NULL);
 
  	INIT_LIST_HEAD(&dev_priv->context_list);
+	INIT_WORK(&dev_priv->mm.free_work, __i915_gem_free_work);
+	init_llist_head(&dev_priv->mm.free_list);
  	INIT_LIST_HEAD(&dev_priv->mm.unbound_list);
  	INIT_LIST_HEAD(&dev_priv->mm.bound_list);
  	INIT_LIST_HEAD(&dev_priv->mm.fence_list);
drivers/gpu/drm/i915/i915_gem_shrinker.c +10 -4
index 868da0bae751..fa72473dc7f5 100644
@@ -201,6 +201,10 @@ i915_gem_shrink(struct drm_i915_private *dev_priv,
  						       typeof(*obj),
  						       global_list))) {
  			list_move_tail(&obj->global_list, &still_in_list);
+			if (!obj->mm.pages) {
+				list_del_init(&obj->global_list);
+				continue;
+			}
 
  			if (flags & I915_SHRINK_PURGEABLE &&
  			    obj->mm.madv != I915_MADV_DONTNEED)
@@ -217,8 +221,6 @@ i915_gem_shrink(struct drm_i915_private *dev_priv,
  			if (!can_release_pages(obj))
  				continue;
 
-			i915_gem_object_get(obj);
-
  			if (unsafe_drop_pages(obj)) {
  				mutex_lock(&obj->mm.lock);
  				if (!obj->mm.pages) {
@@ -227,8 +229,6 @@ i915_gem_shrink(struct drm_i915_private *dev_priv,
  				}
  				mutex_unlock(&obj->mm.lock);
  			}
-
-			i915_gem_object_put(obj);
  		}
  		list_splice(&still_in_list, phase->list);
  	}
@@ -395,12 +395,18 @@ i915_gem_shrinker_oom(struct notifier_block *nb, unsigned long event, void *ptr)
  	 */
  	unbound = bound = unevictable = 0;
  	list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_list) {
+		if (!obj->mm.pages)
+			continue;
+
  		if (!can_release_pages(obj))
  			unevictable += obj->base.size >> PAGE_SHIFT;
  		else
  			unbound += obj->base.size >> PAGE_SHIFT;
  	}
  	list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
+		if (!obj->mm.pages)
+			continue;
+
  		if (!can_release_pages(obj))
  			unevictable += obj->base.size >> PAGE_SHIFT;
  		else
drivers/gpu/drm/i915/i915_gem_tiling.c +1 -0
index 11e2e0f57ac1..ec48e403adfe 100644
@@ -328,12 +328,17 @@ i915_gem_get_tiling(struct drm_device *dev, void *data,
  	struct drm_i915_gem_get_tiling *args = data;
  	struct drm_i915_private *dev_priv = to_i915(dev);
  	struct drm_i915_gem_object *obj;
+	int err = -ENOENT;

drivers/gpu/drm/i915/i915_gem_tiling.c i915_gem_get_tiling(struct drm_device *dev, void *data, · line 341

Quoted 10 lines
drivers/gpu/drm/i915/i915_gem_tiling.c +9 -0
+
+	rcu_read_lock();
+	obj = i915_gem_object_lookup_rcu(file, args->handle);
+	if (obj) {
+		args->tiling_mode =
+			READ_ONCE(obj->tiling_and_stride) & TILING_MASK;
+		err = 0;
+	}
+	rcu_read_unlock();
 

Should it not just return -ENOENT here if !obj rather than going through the rest of the function and having to modify that to ignore unknown tiling modes (because it is doing the switch on uninitialised data)?

Quoted 22 lines
drivers/gpu/drm/i915/i915_gem_tiling.c +1 -7
-	obj = i915_gem_object_lookup(file, args->handle);
-	if (!obj)
-		return -ENOENT;
-
-	args->tiling_mode = READ_ONCE(obj->tiling_and_stride) & TILING_MASK;
  	switch (args->tiling_mode) {
  	case I915_TILING_X:
  		args->swizzle_mode = dev_priv->mm.bit_6_swizzle_x;
@@ -341,11 +346,10 @@ i915_gem_get_tiling(struct drm_device *dev, void *data,
  	case I915_TILING_Y:
  		args->swizzle_mode = dev_priv->mm.bit_6_swizzle_y;
  		break;
+	default:
  	case I915_TILING_NONE:
  		args->swizzle_mode = I915_BIT_6_SWIZZLE_NONE;
  		break;
-	default:
-		DRM_ERROR("unknown tiling mode\n");
  	}
 
  	/* Hide bit 17 from the user -- see comment in i915_gem_set_tiling */

drivers/gpu/drm/i915/i915_gem_tiling.c i915_gem_get_tiling(struct drm_device *dev, void *data, · line 366

Quoted 7 lines
drivers/gpu/drm/i915/i915_gem_tiling.c +1 -2
@@ -358,6 +362,5 @@ i915_gem_get_tiling(struct drm_device *dev, void *data,
  	if (args->swizzle_mode == I915_BIT_6_SWIZZLE_9_10_17)
  		args->swizzle_mode = I915_BIT_6_SWIZZLE_9_10;
 
-	i915_gem_object_put_unlocked(obj);
-	return 0;
+	return err;
  }

_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx

#75 lore raw

PATCH 24/42 drm/i915: Treat a framebuffer reference as an active reference whilst shrinking

In reply to Chris Wilson

Treat a framebuffer reference with the same priority as an active reference whilst shrinking. Framebuffers are likely to be reused and typically cost more to migrate to and from GPU memory (on LLC architectures we need to clflush), so defer the temptation to purge them during a kswapd run until we have run out of cheap buffers.

---
 drivers/gpu/drm/i915/i915_gem_shrinker.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)
drivers/gpu/drm/i915/i915_gem_shrinker.c +3 -2
index fa72473dc7f5..0241658af16b 100644
@@ -214,8 +214,9 @@ i915_gem_shrink(struct drm_i915_private *dev_priv,
 			    !is_vmalloc_addr(obj->mm.mapping))
 				continue;
 
-			if ((flags & I915_SHRINK_ACTIVE) == 0 &&
-			    i915_gem_object_is_active(obj))
+			if (!(flags & I915_SHRINK_ACTIVE) &&
+			    (i915_gem_object_is_active(obj) ||
+			     obj->framebuffer_references))
 				continue;
 
 			if (!can_release_pages(obj))
Signature · 6 lines
2.9.3

_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
Reviewed-by on v1 24/42
#76 lore raw

Re: [PATCH 24/42] drm/i915: Treat a framebuffer reference as an active reference whilst shrinking

Quoted 17 lines · Chris Wilson

Treat a framebuffer reference with the same priority as an active reference whilst shrinking. Framebuffers are likely to be reused and typically cost more to migrate to and from GPU memory (on LLC architectures we need to clflush), so defer the temptation to purge them during a kswapd run until we have run out of cheap buffers.

---
  drivers/gpu/drm/i915/i915_gem_shrinker.c | 5 +++--
  1 file changed, 3 insertions(+), 2 deletions(-)
drivers/gpu/drm/i915/i915_gem_shrinker.c +0 -0
index fa72473dc7f5..0241658af16b 100644
@@ -214,8 +214,9 @@ i915_gem_shrink(struct drm_i915_private *dev_priv,
  			    !is_vmalloc_addr(obj->mm.mapping))

drivers/gpu/drm/i915/i915_gem_shrinker.c i915_gem_shrink(struct drm_i915_private *dev_priv, · line 222

Quoted 10 lines
drivers/gpu/drm/i915/i915_gem_shrinker.c +3 -2
  				continue;
 
-			if ((flags & I915_SHRINK_ACTIVE) == 0 &&
-			    i915_gem_object_is_active(obj))
+			if (!(flags & I915_SHRINK_ACTIVE) &&
+			    (i915_gem_object_is_active(obj) ||
+			     obj->framebuffer_references))
  				continue;
 
  			if (!can_release_pages(obj))

_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx