Explicit fencing on multiple timelines, again
Subsystems (17)
After a hiatus to fix up some fallout from the reset work, here we are again. Biggest features needing review are the changes to memory management to allow deferred freeing to prevent recursion whilst unbinding the active object, and the interaction with atomic modesets.
-Chris
_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH 1/42 drm/i915: Allow disabling error capture
We currently capture the GPU state after we detect a hang. This is vital for us to both triage and debug hangs in the wild (post-mortem debugging). However, it comes at the cost of running some potentially dangerous code (since it has to make very few assumption about the state of the driver) that is quite resource intensive.
--- drivers/gpu/drm/i915/Kconfig | 10 ++++++++++ drivers/gpu/drm/i915/i915_debugfs.c | 6 ++++++ drivers/gpu/drm/i915/i915_drv.h | 16 ++++++++++++++++ drivers/gpu/drm/i915/i915_gpu_error.c | 7 +++++++ drivers/gpu/drm/i915/i915_params.c | 9 +++++++++ drivers/gpu/drm/i915/i915_params.h | 1 + drivers/gpu/drm/i915/i915_sysfs.c | 8 ++++++++ drivers/gpu/drm/i915/intel_display.c | 4 ++++ drivers/gpu/drm/i915/intel_overlay.c | 4 ++++ 9 files changed, 65 insertions(+)
drivers/gpu/drm/i915/Kconfig +10 -0
@@ -46,6 +46,16 @@ config DRM_I915_PRELIMINARY_HW_SUPPORT If in doubt, say "N". +config DRM_I915_CAPTURE_ERROR + bool "Enable capturing GPU state following a hang" + depends on DRM_I915 + default y + help + This option enables capturing the GPU state when a hang is detected. + This information is vital for triaging hangs and assists in debugging. + + If in doubt, say "Y". + config DRM_I915_USERPTR bool "Always enable userptr support" depends on DRM_I915
drivers/gpu/drm/i915/i915_debugfs.c +6 -0
@@ -960,6 +960,8 @@ static int i915_hws_info(struct seq_file *m, void *data)return 0 ; } +#ifdef CONFIG_DRM_I915_CAPTURE_ERROR + static ssize_t i915_error_state_write(struct file *filp, const char __user *ubuf,
@@ -1042,6 +1044,8 @@ static const struct file_operations i915_error_state_fops = { .release = i915_error_state_release, }; +#endif +static int i915_next_seqno_get (void *data,u64 *val) {
@@ -5398,7 +5402,9 @@ static const struct i915_debugfs_files { {"i915_ring_missed_irq", &i915_ring_missed_irq_fops}, {"i915_ring_test_irq", &i915_ring_test_irq_fops}, {"i915_gem_drop_caches", &i915_drop_caches_fops}, +#ifdef CONFIG_DRM_I915_CAPTURE_ERROR {"i915_error_state", &i915_error_state_fops}, +#endif {"i915_next_seqno", &i915_next_seqno_fops}, {"i915_display_crc_ctl", &i915_display_crc_ctl_fops}, {"i915_pri_wm_latency", &i915_pri_wm_latency_fops},
drivers/gpu/drm/i915/i915_drv.h +16 -0
@@ -3544,6 +3544,8 @@ static inline void intel_display_crc_init(struct drm_i915_private *dev_priv) {}#endif /* i915_gpu_error.c */ +#ifdef CONFIG_DRM_I915_CAPTURE_ERROR + __printf(2, 3)void i915_error_printf (struct drm_i915_error_state_buf *e,const char *f, ...); int i915_error_state_to_str(struct drm_i915_error_state_buf *estr,
@@ -3564,6 +3566,20 @@ void i915_error_state_get(struct drm_device *dev,void i915_error_state_put (struct i915_error_state_file_priv *error_priv);void i915_destroy_error_state (struct drm_device *dev); +#else + +static inline void i915_capture_error_state (struct drm_i915_private *dev_priv, +u32 engine_mask, +const char *error_msg) +{ +} + +static inline void i915_destroy_error_state (struct drm_device *dev) +{ +} + +#endif +void i915_get_engine_instdone (struct drm_i915_private *dev_priv,enum intel_engine_id engine_id,struct intel_instdone *instdone);
drivers/gpu/drm/i915/i915_gpu_error.c +7 -0
@@ -30,6 +30,8 @@#include <generated/utsrelease.h> #include "i915_drv.h" +#ifdef CONFIG_DRM_I915_CAPTURE_ERROR +static const char *engine_str (int engine) {switch (engine) {
@@ -1464,6 +1466,9 @@ void i915_capture_error_state(struct drm_i915_private *dev_priv,struct drm_i915_error_state *error;unsigned long flags; +if (!i915.error_capture) +return ; +if (READ_ONCE (dev_priv->gpu_error.first_error))return ;
@@ -1549,6 +1554,8 @@ void i915_destroy_error_state(struct drm_device *dev) kref_put(&error->ref, i915_error_state_free); } +#endif +const char *i915_cache_level_str (struct drm_i915_private *i915,int type) {switch (type) {
drivers/gpu/drm/i915/i915_params.c +9 -0
@@ -47,6 +47,7 @@ struct i915_params i915 __read_mostly = { .load_detect_test = 0, .force_reset_modeset_test = 0, .reset = true, + .error_capture = true, .invert_brightness = 0, .disable_display = 0, .enable_cmd_parser = 1,
@@ -115,6 +116,14 @@ MODULE_PARM_DESC(vbt_sdvo_panel_type,module_param_named_unsafe (reset, i915.reset, bool,0600 );MODULE_PARM_DESC (reset,"Attempt GPU resets (default: true)" ); +#ifdef CONFIG_DRM_I915_CAPTURE_ERROR +module_param_named (error_capture, i915.error_capture, bool,0600 ); +MODULE_PARM_DESC (error_capture, +"Record the GPU state following a hang. " +"This information in /sys/class/drm/card<N>/error is vital for " +"triaging and debugging hangs." ); +#endif +module_param_named_unsafe (enable_hangcheck, i915.enable_hangcheck, bool,0644 );MODULE_PARM_DESC (enable_hangcheck, "Periodically check GPU activity for detecting hangs. "
drivers/gpu/drm/i915/i915_params.h +1 -0
@@ -59,6 +59,7 @@ struct i915_params {bool load_detect_test;bool force_reset_modeset_test;bool reset; +bool error_capture;bool disable_display;bool verbose_state_checks;bool nuclear_pageflip;
drivers/gpu/drm/i915/i915_sysfs.c +8 -0
@@ -514,6 +514,8 @@ static const struct attribute *vlv_attrs[] = { NULL, }; +#ifdef CONFIG_DRM_I915_CAPTURE_ERROR +static ssize_t error_state_read (struct file *filp,struct kobject *kobj,struct bin_attribute *attr,char *buf,loff_t off,size_t count)
@@ -571,6 +573,8 @@ static struct bin_attribute error_state_attr = { .write = error_state_write, }; +#endif +void i915_setup_sysfs (struct drm_i915_private *dev_priv) {struct device *kdev = dev_priv->drm.primary->kdev;
@@ -617,17 +621,21 @@ void i915_setup_sysfs(struct drm_i915_private *dev_priv)if (ret)DRM_ERROR ("RPS sysfs setup failed\n" ); +#ifdef CONFIG_DRM_I915_CAPTURE_ERROR ret =sysfs_create_bin_file (&kdev->kobj, &error_state_attr);if (ret)DRM_ERROR ("error_state sysfs setup failed\n" ); +#endif }void i915_teardown_sysfs (struct drm_i915_private *dev_priv) {struct device *kdev = dev_priv->drm.primary->kdev; +#ifdef CONFIG_DRM_I915_CAPTURE_ERROR sysfs_remove_bin_file (&kdev->kobj, &error_state_attr); +#endif if (IS_VALLEYVIEW (dev_priv) ||IS_CHERRYVIEW (dev_priv))sysfs_remove_files (&kdev->kobj, vlv_attrs);else
drivers/gpu/drm/i915/intel_display.c +4 -0
@@ -17096,6 +17096,8 @@ int intel_modeset_vga_set_state(struct drm_device *dev, bool state)return 0 ; } +#ifdef CONFIG_DRM_I915_CAPTURE_ERROR + struct intel_display_error_state { u32 power_well_driver;
@@ -17278,3 +17280,5 @@ intel_display_print_error_state(struct drm_i915_error_state_buf *m, err_printf(m, " VSYNC: %08x\n", error->transcoder[i].vsync); } } + +#endif
drivers/gpu/drm/i915/intel_overlay.c +4 -0
@@ -1470,6 +1470,8 @@ void intel_cleanup_overlay(struct drm_i915_private *dev_priv)kfree (dev_priv->overlay); } +#ifdef CONFIG_DRM_I915_CAPTURE_ERROR + struct intel_overlay_error_state { struct overlay_registers regs; unsigned long base;
@@ -1587,3 +1589,5 @@ intel_overlay_print_error_state(struct drm_i915_error_state_buf *m,P (UVSCALEV );#undef P } + +#endif
Signature · 6 lines
2.9.3 _______________________________________________ Intel-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH 2/42 drm/i915: Stop the machine whilst capturing the GPU crash dump
In reply to Chris WilsonThe error state is purposefully racy as we expect it to be called at any time and so have avoided any locking whilst capturing the crash dump. However, with multi-engine GPUs and multiple CPUs, those races can manifest into OOPSes as we attempt to chase dangling pointers freed on other CPUs. Under discussion are lots of ways to slow down normal operation in order to protect the post-mortem error capture, but what it we take the opposite approach and freeze the machine whilst the error capture runs (note the GPU may still running, but as long as we don't process any of the results the driver's bookkeeping will be static).
Note that by of itself, this is not a complete fix. It also depends on the compiler barriers in list_add/list_del to prevent traversing the lists into the void. We also depend that we only require state from carefully controlled sources - i.e. all the state we require for post-mortem debugging should be reachable from the request itself so that we only have to worry about retrieving the request carefully. Once we have the request, we know that all pointers from it are intact.
v2: Avoid drm_clflush_pages() inside stop_machine() as it may use stop_machine() itself for its wbinvd fallback.
--- drivers/gpu/drm/i915/Kconfig | 1 + drivers/gpu/drm/i915/i915_drv.h | 2 ++ drivers/gpu/drm/i915/i915_gpu_error.c | 46 +++++++++++++++++++++-------------- 3 files changed, 31 insertions(+), 18 deletions(-)
drivers/gpu/drm/i915/Kconfig +1 -0
@@ -4,6 +4,7 @@ config DRM_I915 depends on X86 && PCI select INTEL_GTT select INTERVAL_TREE + select STOP_MACHINE # we need shmfs for the swappable backing store, and in particular # the shmem_readpage() which depends upon tmpfs select SHMEM
drivers/gpu/drm/i915/i915_drv.h +2 -0
@@ -746,6 +746,8 @@ struct drm_i915_error_state {struct kref ref;struct timeval time; +struct drm_i915_private *i915; +char error_msg[128 ];bool simulated;int iommu;
drivers/gpu/drm/i915/i915_gpu_error.c +28 -18
87 lines not on this page · raw message
Signature · 6 lines
2.9.3 _______________________________________________ Intel-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/intel-gfx
Re: [PATCH 02/42] drm/i915: Stop the machine whilst capturing the GPU crash dump
Quoted 18 lines · Chris Wilson
The error state is purposefully racy as we expect it to be called at any time and so have avoided any locking whilst capturing the crash dump. However, with multi-engine GPUs and multiple CPUs, those races can manifest into OOPSes as we attempt to chase dangling pointers freed on other CPUs. Under discussion are lots of ways to slow down normal operation in order to protect the post-mortem error capture, but what it we take the opposite approach and freeze the machine whilst the error capture runs (note the GPU may still running, but as long as we don't process any of the results the driver's bookkeeping will be static).
Note that by of itself, this is not a complete fix. It also depends on the compiler barriers in list_add/list_del to prevent traversing the lists into the void. We also depend that we only require state from carefully controlled sources - i.e. all the state we require for post-mortem debugging should be reachable from the request itself so that we only have to worry about retrieving the request carefully. Once we have the request, we know that all pointers from it are intact.
Quoted 4 lines
v2: Avoid
drm_clflush_pages()insidestop_machine()as it may usestop_machine()itself for its wbinvd fallback.Signed-off-by Chris Wilson <[email protected]>
CC'in Daniel to add A-b.
Regards, Joonas
Signature · 7 lines
Joonas Lahtinen Open Source Technology Center Intel Corporation _______________________________________________ Intel-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH 3/42 drm/i915: Always use the GTT for error capture
In reply to Chris WilsonSince the GTT provides universal access to any GPU page, we can use it to reduce our plethora of read methods to just one. It also has the important characteristic of being exactly what the GPU sees - if there are incoherency problems, seeing the batch as executed (rather than as trapped inside the cpu cache) is important.
--- drivers/gpu/drm/i915/i915_gem_gtt.c | 43 ++++++++---- drivers/gpu/drm/i915/i915_gem_gtt.h | 2 + drivers/gpu/drm/i915/i915_gpu_error.c | 120 ++++++++++++---------------------- 3 files changed, 74 insertions(+), 91 deletions(-)
drivers/gpu/drm/i915/i915_gem_gtt.c +30 -13
@@ -2717,6 +2717,7 @@ int i915_gem_init_ggtt(struct drm_i915_private *dev_priv)*/ struct i915_ggtt *ggtt = &dev_priv->ggtt;unsigned long hole_start, hole_end; +struct i915_hw_ppgtt *ppgtt;struct drm_mm_node *entry;int ret;
@@ -2724,6 +2725,15 @@ int i915_gem_init_ggtt(struct drm_i915_private *dev_priv)if (ret)return ret; +/* Reserve a mappable slot for our lockless error capture */ + ret =drm_mm_insert_node_in_range_generic (&ggtt->base.mm, + &ggtt->error_capture, +4096 ,0 ,-1 , +0 , ggtt->mappable_end, +0 ,0 ); +if (ret) +return ret; +/* Clear any non-preallocated blocks */ drm_mm_for_each_hole (entry, &ggtt->base.mm, hole_start, hole_end) { DRM_DEBUG_KMS("clearing unused GTT space: [%lx, %lx]\n",
@@ -2738,25 +2748,21 @@ int i915_gem_init_ggtt(struct drm_i915_private *dev_priv) true);if (USES_PPGTT (dev_priv) && !USES_FULL_PPGTT (dev_priv)) { -struct i915_hw_ppgtt *ppgtt; - ppgtt =kzalloc (sizeof (*ppgtt),GFP_KERNEL ); -if (!ppgtt) -return -ENOMEM ; +if (!ppgtt) { + ret = -ENOMEM ; + goto err; + } ret =__hw_ppgtt_init (ppgtt, dev_priv); -if (ret) { -kfree (ppgtt); -return ret; - } +if (ret) + goto err_ppgtt; -if (ppgtt->base.allocate_va_range) +if (ppgtt->base.allocate_va_range) { ret = ppgtt->base.allocate_va_range (&ppgtt->base,0 , ppgtt->base.total); -if (ret) { - ppgtt->base.cleanup (&ppgtt->base); -kfree (ppgtt); -return ret; +if (ret) + goto err_ppgtt_cleanup; } ppgtt->base.clear_range(&ppgtt->base,
@@ -2770,6 +2776,14 @@ int i915_gem_init_ggtt(struct drm_i915_private *dev_priv) }return 0 ; + +err_ppgtt_cleanup: + ppgtt->base.cleanup (&ppgtt->base); +err_ppgtt: +kfree (ppgtt); +err: +drm_mm_remove_node (&ggtt->error_capture); +return ret; }/**
@@ -2788,6 +2802,9 @@ void i915_ggtt_cleanup_hw(struct drm_i915_private *dev_priv)i915_gem_cleanup_stolen (&dev_priv->drm); +if (drm_mm_node_allocated (&ggtt->error_capture)) +drm_mm_remove_node (&ggtt->error_capture); +if (drm_mm_initialized (&ggtt->base.mm)) {intel_vgt_deballoon (dev_priv);
drivers/gpu/drm/i915/i915_gem_gtt.h +2 -0
@@ -450,6 +450,8 @@ struct i915_ggtt {bool do_idle_maps;int mtrr; + +struct drm_mm_node error_capture; };struct i915_hw_ppgtt {
drivers/gpu/drm/i915/i915_gpu_error.c +42 -78
172 lines not on this page · raw message
Signature · 6 lines
2.9.3 _______________________________________________ Intel-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH 4/42 drm/i915: Consolidate error object printing
In reply to Chris WilsonLeave all the pretty printing to userspace and simplify the error capture to only have a single common object printer. It makes the kernel code more compact, and the refactoring allows us to apply more complex transformations like compressing the output.
--- drivers/gpu/drm/i915/i915_gpu_error.c | 100 +++++++++------------------------- 1 file changed, 25 insertions(+), 75 deletions(-)
drivers/gpu/drm/i915/i915_gpu_error.c +25 -75
@@ -343,10 +343,22 @@ void i915_error_printf(struct drm_i915_error_state_buf *e, const char *f, ...) }static void print_error_obj (struct drm_i915_error_state_buf *m, +struct intel_engine_cs *engine, +const char *name,struct drm_i915_error_object *obj) {int page, offset, elt; +if (!obj) +return ; + +if (name) { +err_printf (m,"%s --- %s = 0x%08x %08x\n" , + engine ? engine->name :"global" , name, +upper_32_bits (obj->gtt_offset), +lower_32_bits (obj->gtt_offset)); + } +for (page = offset =0 ; page < obj->page_count; page++) {for (elt =0 ; elt <PAGE_SIZE /4 ; elt++) { err_printf(m, "%08x : %08x\n", offset,
@@ -372,8 +384,8 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m,struct pci_dev *pdev = dev_priv->drm.pdev;struct drm_i915_error_state *error = error_priv->error;struct drm_i915_error_object *obj; -int i, j, offset, elt;int max_hangcheck_score; +int i, j;if (!error) {err_printf (m,"no error state collected\n" );
@@ -493,15 +505,7 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m,err_printf (m," --- gtt_offset = 0x%08x %08x\n" ,upper_32_bits (obj->gtt_offset),lower_32_bits (obj->gtt_offset)); -print_error_obj (m, obj); - } - - obj = ee->wa_batchbuffer; -if (obj) { -err_printf (m,"%s (w/a) --- gtt_offset = 0x%08x\n" , - dev_priv->engine[i].name, -lower_32_bits (obj->gtt_offset)); -print_error_obj (m, obj); +print_error_obj (m, &dev_priv->engine[i],NULL , obj); }if (ee->num_requests) {
@@ -533,77 +537,23 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m, } } -if ((obj = ee->ringbuffer)) { -err_printf (m,"%s --- ringbuffer = 0x%08x\n" , - dev_priv->engine[i].name, -lower_32_bits (obj->gtt_offset)); -print_error_obj (m, obj); - } +print_error_obj (m, &dev_priv->engine[i], +"ringbuffer" , ee->ringbuffer); -if ((obj = ee->hws_page)) { -u64 hws_offset = obj->gtt_offset; -u32 *hws_page = &obj->pages[0 ][0 ]; +print_error_obj (m, &dev_priv->engine[i], +"HW Status" , ee->hws_page); -if (i915.enable_execlists) { - hws_offset +=LRC_PPHWSP_PN *PAGE_SIZE ; - hws_page = &obj->pages[LRC_PPHWSP_PN ][0 ]; - } -err_printf (m,"%s --- HW Status = 0x%08llx\n" , - dev_priv->engine[i].name, hws_offset); - offset =0 ; -for (elt =0 ; elt <PAGE_SIZE /16 ; elt +=4 ) { -err_printf (m,"[%04x] %08x %08x %08x %08x\n" , - offset, - hws_page[elt], - hws_page[elt+1 ], - hws_page[elt+2 ], - hws_page[elt+3 ]); - offset +=16 ; - } - } +print_error_obj (m, &dev_priv->engine[i], +"HW context" , ee->ctx); - obj = ee->wa_ctx; -if (obj) { -u64 wa_ctx_offset = obj->gtt_offset; -u32 *wa_ctx_page = &obj->pages[0 ][0 ]; -struct intel_engine_cs *engine = &dev_priv->engine[RCS ]; -u32 wa_ctx_size = (engine->wa_ctx.indirect_ctx.size + - engine->wa_ctx.per_ctx.size); - -err_printf (m,"%s --- WA ctx batch buffer = 0x%08llx\n" , - dev_priv->engine[i].name, wa_ctx_offset); - offset =0 ; -for (elt =0 ; elt < wa_ctx_size; elt +=4 ) { -err_printf (m,"[%04x] %08x %08x %08x %08x\n" , - offset, - wa_ctx_page[elt +0 ], - wa_ctx_page[elt +1 ], - wa_ctx_page[elt +2 ], - wa_ctx_page[elt +3 ]); - offset +=16 ; - } - } +print_error_obj (m, &dev_priv->engine[i], +"WA context" , ee->wa_ctx); -if ((obj = ee->ctx)) { -err_printf (m,"%s --- HW Context = 0x%08x\n" , - dev_priv->engine[i].name, -lower_32_bits (obj->gtt_offset)); -print_error_obj (m, obj); - } +print_error_obj (m, &dev_priv->engine[i], +"WA batchbuffer" , ee->wa_batchbuffer); } -if ((obj = error->semaphore)) { -err_printf (m,"Semaphore page = 0x%08x\n" , -lower_32_bits (obj->gtt_offset)); -for (elt =0 ; elt <PAGE_SIZE /16 ; elt +=4 ) { -err_printf (m,"[%04x] %08x %08x %08x %08x\n" , - elt *4 , - obj->pages[0 ][elt], - obj->pages[0 ][elt+1 ], - obj->pages[0 ][elt+2 ], - obj->pages[0 ][elt+3 ]); - } - } +print_error_obj (m,NULL ,"Semaphores" , error->semaphore);if (error->overlay)intel_overlay_print_error_state (m, error->overlay);
Signature · 6 lines
2.9.3 _______________________________________________ Intel-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH 5/42 drm/i915: Compress GPU objects in error state
In reply to Chris WilsonOur error states are quickly growing, pinning kernel memory with them. The majority of the space is taken up by the error objects. These compress well using zlib and without decode are mostly meaningless, so encoding them does not hinder quickly parsing the error state for familiarity.
v2: Make the zlib dependency optional
--- drivers/gpu/drm/i915/Kconfig | 12 +++ drivers/gpu/drm/i915/i915_drv.h | 3 +- drivers/gpu/drm/i915/i915_gpu_error.c | 176 ++++++++++++++++++++++++++++++---- 3 files changed, 169 insertions(+), 22 deletions(-)
drivers/gpu/drm/i915/Kconfig +12 -0
@@ -57,6 +57,18 @@ config DRM_I915_CAPTURE_ERROR If in doubt, say "Y". +config DRM_I915_COMPRESS_ERROR + bool "Compress GPU error state" + depends on DRM_I915_CAPTURE_ERROR + select ZLIB_DEFLATE + default y + help + This option selects ZLIB_DEFLATE if it isn't already + selected and causes any error state captured upon a GPU hang + to be compressed using zlib. + + If in doubt, say "Y". + config DRM_I915_USERPTR bool "Always enable userptr support" depends on DRM_I915
drivers/gpu/drm/i915/i915_drv.h +2 -1
@@ -820,9 +820,10 @@ struct drm_i915_error_state {struct intel_instdone instdone;struct drm_i915_error_object { -int page_count;u64 gtt_offset;u64 gtt_size; +int page_count; +int unused;u32 *pages[0 ]; } *ringbuffer, *batchbuffer, *wa_batchbuffer, *ctx, *hws_page;
drivers/gpu/drm/i915/i915_gpu_error.c +155 -21
243 lines not on this page · raw message
Signature · 6 lines
2.9.3 _______________________________________________ Intel-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH 6/42 drm/i915: Support asynchronous waits on struct fence from i915_gem_request
In reply to Chris WilsonWe will need to wait on DMA completion (as signaled via struct fence) before executing our i915_gem_request. Therefore we want to expose a method for adding the await on the fence itself to the request.
--- drivers/gpu/drm/i915/i915_gem_request.c | 40 +++++++++++++++++++++++++++++++++ drivers/gpu/drm/i915/i915_gem_request.h | 2 ++ 2 files changed, 42 insertions(+)
drivers/gpu/drm/i915/i915_gem_request.c +40 -0
@@ -23,6 +23,7 @@*/ #include <linux/prefetch.h> +#include <linux/fence-array.h> #include "i915_drv.h"
@@ -495,6 +496,45 @@ i915_gem_request_await_request(struct drm_i915_gem_request *to,return 0 ; } +int +i915_gem_request_await_fence (struct drm_i915_gem_request *req, +struct fence *fence) +{ +struct fence_array *array; +int ret; +int i; + +if (test_bit (FENCE_FLAG_SIGNALED_BIT , &fence->flags)) +return 0 ; + +if (fence_is_i915 (fence)) +return i915_gem_request_await_request (req,to_request (fence)); + +if (!fence_is_array (fence)) { + ret =i915_sw_fence_await_dma_fence (&req->submit, + fence,10 *HZ , +GFP_KERNEL ); +return ret <0 ? ret :0 ; + } + + array =to_fence_array (fence); +for (i =0 ; i < array->num_fences; i++) { +struct fence *child = array->fences[i]; + +if (fence_is_i915 (child)) + ret =i915_gem_request_await_request (req, +to_request (child)); +else + ret =i915_sw_fence_await_dma_fence (&req->submit, + child,10 *HZ , +GFP_KERNEL ); +if (ret <0 ) +return ret; + } + +return 0 ; +} +/** * i915_gem_request_await_object - set this request to (async) wait upon a bo *
drivers/gpu/drm/i915/i915_gem_request.h +2 -0
@@ -214,6 +214,8 @@ int i915_gem_request_await_object(struct drm_i915_gem_request *to, struct drm_i915_gem_object *obj, bool write); +int i915_gem_request_await_fence (struct drm_i915_gem_request *req, +struct fence *fence);void __i915_add_request (struct drm_i915_gem_request *req,bool flush_caches);#define i915_add_request (req) \
Signature · 6 lines
2.9.3 _______________________________________________ Intel-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/intel-gfx
Re: [PATCH 06/42] drm/i915: Support asynchronous waits on struct fence from i915_gem_request
drivers/gpu/drm/i915/i915_gem_request.c i915_gem_request_await_request(struct drm_i915_gem_request *to, · line 519
Quoted 7 lines · Chris Wilson
drivers/gpu/drm/i915/i915_gem_request.c +7
-0+if (!fence_is_array (fence)) { + ret =i915_sw_fence_await_dma_fence (&req->submit, + fence, 10*HZ, + GFP_KERNEL); +return ret <0 ? ret :0 ; + } +
With the magic removed in two spots,
Regards, Joonas
Signature · 7 lines
Joonas Lahtinen Open Source Technology Center Intel Corporation _______________________________________________ Intel-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/intel-gfx
Re: [PATCH 06/42] drm/i915: Support asynchronous waits on struct fence from i915_gem_request
In reply to Chris WilsonQuoted 40 lines · Chris Wilson
We will need to wait on DMA completion (as signaled via struct fence) before executing our
i915_gem_request. Therefore we want to expose a method for adding the await on the fence itself to the request.Signed-off-by Chris Wilson <[email protected]>--- drivers/gpu/drm/i915/i915_gem_request.c | 40 +++++++++++++++++++++++++++++++++ drivers/gpu/drm/i915/i915_gem_request.h | 2 ++ 2 files changed, 42 insertions(+)drivers/gpu/drm/i915/i915_gem_request.c +15
-0@@ -23,6 +23,7 @@*/ #include <linux/prefetch.h> +#include <linux/fence-array.h> #include "i915_drv.h" @@ -495,6 +496,45 @@ i915_gem_request_await_request(struct drm_i915_gem_request *to,return 0 ; } +int +i915_gem_request_await_fence (struct drm_i915_gem_request *req, +struct fence *fence) +{ +struct fence_array *array; +int ret; +int i; + +if (test_bit (FENCE_FLAG_SIGNALED_BIT , &fence->flags)) +return 0 ; + +if (fence_is_i915 (fence)) +return i915_gem_request_await_request (req,to_request (fence)); +
drivers/gpu/drm/i915/i915_gem_request.c i915_gem_request_await_request(struct drm_i915_gem_request *to, · line 522
Quoted 10 lines
drivers/gpu/drm/i915/i915_gem_request.c +10
-0+if (!fence_is_array (fence)) { + ret =i915_sw_fence_await_dma_fence (&req->submit, + fence,10 *HZ , +GFP_KERNEL ); +return ret <0 ? ret :0 ; + } + + array =to_fence_array (fence); +for (i =0 ; i < array->num_fences; i++) { +struct fence *child = array->fences[i];
1. What is the advantage in manually waiting on array elements rather than just the array? Is the existance of signal_on_any a problem?
2. Can child be another array?
Regards,
Tvrtko
Quoted 23 lines
drivers/gpu/drm/i915/i915_gem_request.c +15
-0+ +if (fence_is_i915 (child)) + ret =i915_gem_request_await_request (req, +to_request (child)); +else + ret =i915_sw_fence_await_dma_fence (&req->submit, + child,10 *HZ , +GFP_KERNEL ); +if (ret <0 ) +return ret; + } + +return 0 ; +} +/** * i915_gem_request_await_object - set this request to (async) wait upon a bo * drivers/gpu/drm/i915/i915_gem_request.h +0
-0
drivers/gpu/drm/i915/i915_gem_request.h int · line 221
Quoted 8 lines
drivers/gpu/drm/i915/i915_gem_request.h +2
-0@@ -214,6 +214,8 @@ int i915_gem_request_await_object(struct drm_i915_gem_request *to, struct drm_i915_gem_object *obj, bool write); +int i915_gem_request_await_fence (struct drm_i915_gem_request *req, +struct fence *fence);void __i915_add_request (struct drm_i915_gem_request *req,bool flush_caches);#define i915_add_request (req) \
_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
Re: [PATCH 06/42] drm/i915: Support asynchronous waits on struct fence from i915_gem_request
Quoted 44 lines · Tvrtko Ursulin
Quoted 42 lines · Chris Wilson
We will need to wait on DMA completion (as signaled via struct fence) before executing our
i915_gem_request. Therefore we want to expose a method for adding the await on the fence itself to the request.Signed-off-by Chris Wilson <[email protected]>--- drivers/gpu/drm/i915/i915_gem_request.c | 40 +++++++++++++++++++++++++++++++++ drivers/gpu/drm/i915/i915_gem_request.h | 2 ++ 2 files changed, 42 insertions(+)drivers/gpu/drm/i915/i915_gem_request.c +21
-0@@ -23,6 +23,7 @@*/ #include <linux/prefetch.h> +#include <linux/fence-array.h> #include "i915_drv.h" @@ -495,6 +496,45 @@ i915_gem_request_await_request(struct drm_i915_gem_request *to,return 0 ; } +int +i915_gem_request_await_fence (struct drm_i915_gem_request *req, +struct fence *fence) +{ +struct fence_array *array; +int ret; +int i; + +if (test_bit (FENCE_FLAG_SIGNALED_BIT , &fence->flags)) +return 0 ; + +if (fence_is_i915 (fence)) +return i915_gem_request_await_request (req,to_request (fence)); + +if (!fence_is_array (fence)) { + ret =i915_sw_fence_await_dma_fence (&req->submit, + fence,10 *HZ , +GFP_KERNEL ); +return ret <0 ? ret :0 ; + }
Quoted 8 lines
drivers/gpu/drm/i915/i915_gem_request.c i915_gem_request_await_request(struct drm_i915_gem_request *to, · line 522
Quoted 4 lines
drivers/gpu/drm/i915/i915_gem_request.c +4
-0+ + array =to_fence_array (fence); +for (i =0 ; i < array->num_fences; i++) { +struct fence *child = array->fences[i];1. What is the advantage in manually waiting on array elements rather than just the array? Is the existance of
signal_on_anya problem?
We can wait on native fences much more efficiently. In the typical case, we don't even need the wait at all since the ordering is imposed by the timeline. Or we can hook into the submission fence to avoid the interrupt. Failing that we have to respond to the interrupt. Also note that foriegn fences handle autosignaling differently to native fences, as we cannot rely on third parties having a working hangcheck.
Quoted 1 line
2. Can child be another array?
Yes, but we don't want to recurse (or at least need to bound the recursion).
-Chris
Signature · 5 lines
Chris Wilson, Intel Open Source Technology Centre _______________________________________________ Intel-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/intel-gfx
Re: [PATCH 06/42] drm/i915: Support asynchronous waits on struct fence from i915_gem_request
Quoted 58 lines · Chris Wilson
Quoted 42 lines · Tvrtko Ursulin
Quoted 41 lines · Chris Wilson
We will need to wait on DMA completion (as signaled via struct fence) before executing our
i915_gem_request. Therefore we want to expose a method for adding the await on the fence itself to the request.Signed-off-by Chris Wilson <[email protected]>--- drivers/gpu/drm/i915/i915_gem_request.c | 40 +++++++++++++++++++++++++++++++++ drivers/gpu/drm/i915/i915_gem_request.h | 2 ++ 2 files changed, 42 insertions(+)drivers/gpu/drm/i915/i915_gem_request.c +20
-0@@ -23,6 +23,7 @@*/ #include <linux/prefetch.h> +#include <linux/fence-array.h> #include "i915_drv.h" @@ -495,6 +496,45 @@ i915_gem_request_await_request(struct drm_i915_gem_request *to,return 0 ; } +int +i915_gem_request_await_fence (struct drm_i915_gem_request *req, +struct fence *fence) +{ +struct fence_array *array; +int ret; +int i; + +if (test_bit (FENCE_FLAG_SIGNALED_BIT , &fence->flags)) +return 0 ; + +if (fence_is_i915 (fence)) +return i915_gem_request_await_request (req,to_request (fence)); + +if (!fence_is_array (fence)) { + ret =i915_sw_fence_await_dma_fence (&req->submit, + fence,10 *HZ , +GFP_KERNEL ); +return ret <0 ? ret :0 ;Quoted 8 lines
drivers/gpu/drm/i915/i915_gem_request.c i915_gem_request_await_request(struct drm_i915_gem_request *to, · line 522
Quoted 5 lines
drivers/gpu/drm/i915/i915_gem_request.c +5
-0+ } + + array =to_fence_array (fence); +for (i =0 ; i < array->num_fences; i++) { +struct fence *child = array->fences[i];1. What is the advantage in manually waiting on array elements rather than just the array? Is the existance of
signal_on_anya problem?We can wait on native fences much more efficiently. In the typical case, we don't even need the wait at all since the ordering is imposed by the timeline. Or we can hook into the submission fence to avoid the interrupt. Failing that we have to respond to the interrupt. Also note that foriegn fences handle autosignaling differently to native fences, as we cannot rely on third parties having a working hangcheck.
Quoted 3 lines
Quoted 1 line
2. Can child be another array?
Yes, but we don't want to recurse (or at least need to bound the recursion).
In that case could the array have been created in the signal_on_any mode and how would we handle that?
Regards,
Tvrtko
_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
Re: [PATCH 06/42] drm/i915: Support asynchronous waits on struct fence from i915_gem_request
Quoted 8 lines · Tvrtko Ursulin
Quoted 3 lines · Chris Wilson
Quoted 1 line
2. Can child be another array?
Yes, but we don't want to recurse (or at least need to bound the recursion).
In that case could the array have been created in the
signal_on_anymode and how would we handle that?
Hmm. Not along our paths, I think. The only fence-array we have are from sync-file which are signal-on-all (except... in the case of where it wraps a single fence, that fence could be a composite). signal-on-any is icky, to be complete we should not decompose it into its elements - however, whether or not a fence-array is operating in signal_on_any mode is not stored. So at the moment, the best I can do is offer a comment.
The only user of it at the moment is amdgpu waiting for the first of many VM manager to become available, not something we'll see immediately via sync-file.
-Chris
Signature · 5 lines
Chris Wilson, Intel Open Source Technology Centre _______________________________________________ Intel-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/intel-gfx
Re: [PATCH 06/42] drm/i915: Support asynchronous waits on struct fence from i915_gem_request
Quoted 5 lines · Chris Wilson
Quoted 4 lines · Tvrtko Ursulin
Quoted 3 lines · Chris Wilson
Quoted 1 line
2. Can child be another array?
Yes, but we don't want to recurse (or at least need to bound the recursion).
Quoted 8 lines
Quoted 2 lines
In that case could the array have been created in the
signal_on_anymode and how would we handle that?Hmm. Not along our paths, I think. The only fence-array we have are from sync-file which are signal-on-all (except... in the case of where it wraps a single fence, that fence could be a composite). signal-on-any is icky, to be complete we should not decompose it into its elements - however, whether or not a fence-array is operating in
signal_on_anymode is not stored. So at the moment, the best I can do is offer a comment.
Yes signal-on-any sounds extremely weird. It mandates you decompose 3rd party fences in all cases, with recursion.
Quoted 3 lines
The only user of it at the moment is amdgpu waiting for the first of many VM manager to become available, not something we'll see immediately via sync-file.
So userspace cannot engineer it to happen with some funky operations before giving us a fence?
Regards,
Tvrtko
_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
Re: [PATCH 06/42] drm/i915: Support asynchronous waits on struct fence from i915_gem_request
Quoted 19 lines · Tvrtko Ursulin
Quoted 5 lines · Chris Wilson
Quoted 4 lines · Tvrtko Ursulin
Quoted 3 lines · Chris Wilson
Quoted 1 line
2. Can child be another array?
Yes, but we don't want to recurse (or at least need to bound the recursion).
Quoted 8 lines
Quoted 2 lines
In that case could the array have been created in the
signal_on_anymode and how would we handle that?Hmm. Not along our paths, I think. The only fence-array we have are from sync-file which are signal-on-all (except... in the case of where it wraps a single fence, that fence could be a composite). signal-on-any is icky, to be complete we should not decompose it into its elements - however, whether or not a fence-array is operating in
signal_on_anymode is not stored. So at the moment, the best I can do is offer a comment.Yes signal-on-any sounds extremely weird. It mandates you decompose 3rd party fences in all cases, with recursion.
Quoted 6 lines
Quoted 3 lines
The only user of it at the moment is amdgpu waiting for the first of many VM manager to become available, not something we'll see immediately via sync-file.
So userspace cannot engineer it to happen with some funky operations before giving us a fence?
No. sync_file is a fence-array (in normal signal-on-all mode) or a single fence (which via merging sync_files can not be another fence-array, though the sync_file could be constructed pointing to a single fence-array fence). If that is a signal-on-any fence, it is broken if the sync_file is ever merged.... And reviewing the existing code, no one should be creating sync_file from a signal-on-any fence-array. There is no ABI to allow the user to create signal-on-any sync_file. I'm feeling safer that there are quite a few issues to resolve before signal-on-any fences are allowed to pollute the sync_file ABI.
-Chris
Signature · 5 lines
Chris Wilson, Intel Open Source Technology Centre _______________________________________________ Intel-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH 7/42 drm/i915: Allow i915_sw_fence_await_sw_fence() to allocate
In reply to Chris WilsonIn forthcoming patches, we want to be able to dynamically allocate the wait_queue_t used whilst awaiting. This is more convenient if we extend the i915_sw_fence_await_sw_fence() to perform the allocation for us if we pass in a gfp mask as an alternative than a preallocated struct.
--- drivers/gpu/drm/i915/i915_sw_fence.c | 40 ++++++++++++++++++++++++++++++++---- drivers/gpu/drm/i915/i915_sw_fence.h | 8 ++++++++ 2 files changed, 44 insertions(+), 4 deletions(-)
drivers/gpu/drm/i915/i915_sw_fence.c +36 -4
@@ -13,6 +13,8 @@#include "i915_sw_fence.h" +#define I915_SW_FENCE_FLAG_ALLOC BIT(0) +static DEFINE_SPINLOCK (i915_sw_fence_lock); static int __i915_sw_fence_notify(struct i915_sw_fence *fence,
@@ -135,6 +137,8 @@ static int i915_sw_fence_wake(wait_queue_t *wq, unsigned mode, int flags, void *list_del (&wq->task_list);__i915_sw_fence_complete (wq->private, key);i915_sw_fence_put (wq->private); +if (wq->flags &I915_SW_FENCE_FLAG_ALLOC ) +kfree (wq);return 0 ; }
@@ -192,9 +196,9 @@ static bool i915_sw_fence_check_if_after(struct i915_sw_fence *fence,return err; } -int i915_sw_fence_await_sw_fence (struct i915_sw_fence *fence, -struct i915_sw_fence *signaler, -wait_queue_t *wq) +static int __i915_sw_fence_await_sw_fence (struct i915_sw_fence *fence, +struct i915_sw_fence *signaler, +wait_queue_t *wq,gfp_t gfp) {unsigned long flags;int pending;
@@ -206,8 +210,22 @@ int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence,if (unlikely (i915_sw_fence_check_if_after (fence, signaler)))return -EINVAL ; + pending =0 ; +if (!wq) { + wq =kmalloc (sizeof (*wq), gfp); +if (!wq) { +if (!gfpflags_allow_blocking (gfp)) +return -ENOMEM ; + +i915_sw_fence_wait (signaler); +return 0 ; + } + + pending |=I915_SW_FENCE_FLAG_ALLOC ; + } +INIT_LIST_HEAD (&wq->task_list); - wq->flags =0 ; + wq->flags = pending; wq->func = i915_sw_fence_wake; wq->private =i915_sw_fence_get (fence);
@@ -226,6 +244,20 @@ int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence,return pending; } +int i915_sw_fence_await_sw_fence (struct i915_sw_fence *fence, +struct i915_sw_fence *signaler, +wait_queue_t *wq) +{ +return __i915_sw_fence_await_sw_fence (fence, signaler, wq,0 ); +} + +int i915_sw_fence_await_sw_fence_gfp (struct i915_sw_fence *fence, +struct i915_sw_fence *signaler, +gfp_t gfp) +{ +return __i915_sw_fence_await_sw_fence (fence, signaler,NULL , gfp); +} +struct dma_fence_cb {struct fence_cb base;struct i915_sw_fence *fence;
drivers/gpu/drm/i915/i915_sw_fence.h +8 -0
@@ -46,6 +46,9 @@ void i915_sw_fence_commit(struct i915_sw_fence *fence);int i915_sw_fence_await_sw_fence (struct i915_sw_fence *fence,struct i915_sw_fence *after,wait_queue_t *wq); +int i915_sw_fence_await_sw_fence_gfp (struct i915_sw_fence *fence, +struct i915_sw_fence *after, +gfp_t gfp); int i915_sw_fence_await_dma_fence(struct i915_sw_fence *fence, struct fence *dma, unsigned long timeout,
@@ -62,4 +65,9 @@ static inline bool i915_sw_fence_done(const struct i915_sw_fence *fence) return atomic_read(&fence->pending) < 0; } +static inline void i915_sw_fence_wait (struct i915_sw_fence *fence) +{ +wait_event (fence->wait,i915_sw_fence_done (fence)); +} +#endif /* _I915_SW_FENCE_H_ */
Signature · 6 lines
2.9.3 _______________________________________________ Intel-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/intel-gfx
Re: [PATCH 07/42] drm/i915: Allow i915_sw_fence_await_sw_fence() to allocate
Quoted 17 lines · Chris Wilson
In forthcoming patches, we want to be able to dynamically allocate the
wait_queue_tused whilst awaiting. This is more convenient if we extend thei915_sw_fence_await_sw_fence()to perform the allocation for us if we pass in a gfp mask as an alternative than a preallocated struct.Signed-off-by Chris Wilson <[email protected]>Reviewed-by Joonas Lahtinen <[email protected]>--- drivers/gpu/drm/i915/i915_sw_fence.c | 40 ++++++++++++++++++++++++++++++++---- drivers/gpu/drm/i915/i915_sw_fence.h | 8 ++++++++ 2 files changed, 44 insertions(+), 4 deletions(-)drivers/gpu/drm/i915/i915_sw_fence.c +0
-0
drivers/gpu/drm/i915/i915_sw_fence.c line 17
Quoted 5 lines
drivers/gpu/drm/i915/i915_sw_fence.c +2
-0@@ -13,6 +13,8 @@#include "i915_sw_fence.h" +#define I915_SW_FENCE_FLAG_ALLOC BIT(0) +
You collide with WQ_FLAG_EXCLUSIVE here, so I assume i915 will have complete control of the wq in question?
Quoted 27 lines
drivers/gpu/drm/i915/i915_sw_fence.c +5
-3static DEFINE_SPINLOCK (i915_sw_fence_lock); static int __i915_sw_fence_notify(struct i915_sw_fence *fence,@@ -135,6 +137,8 @@ static int i915_sw_fence_wake(wait_queue_t *wq, unsigned mode, int flags, void *list_del (&wq->task_list);__i915_sw_fence_complete (wq->private, key);i915_sw_fence_put (wq->private); +if (wq->flags &I915_SW_FENCE_FLAG_ALLOC ) +kfree (wq);return 0 ; }@@ -192,9 +196,9 @@ static bool i915_sw_fence_check_if_after(struct i915_sw_fence *fence,return err; } -int i915_sw_fence_await_sw_fence (struct i915_sw_fence *fence, -struct i915_sw_fence *signaler, -wait_queue_t *wq) +static int __i915_sw_fence_await_sw_fence (struct i915_sw_fence *fence, +struct i915_sw_fence *signaler, +wait_queue_t *wq,gfp_t gfp) {unsigned long flags;int pending;@@ -206,8 +210,22 @@ int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence,if (unlikely (i915_sw_fence_check_if_after (fence, signaler)))
drivers/gpu/drm/i915/i915_sw_fence.c int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence, · line 220
Quoted 10 lines
drivers/gpu/drm/i915/i915_sw_fence.c +8
-0return -EINVAL ; + pending =0 ; +if (!wq) { + wq =kmalloc (sizeof (*wq), gfp); +if (!wq) { +if (!gfpflags_allow_blocking (gfp)) +return -ENOMEM ; + +i915_sw_fence_wait (signaler);
This looks like a strange API, unless I am missing something - it normally sets things up for waiting, unless the allocation fails when it actually does the wait?
drivers/gpu/drm/i915/i915_sw_fence.c int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence, · line 224
Quoted 4 lines
drivers/gpu/drm/i915/i915_sw_fence.c +4
-0+return 0 ; + } + + pending |=I915_SW_FENCE_FLAG_ALLOC ;
Why is this stored in pending? Just because it is around as re-used?
Quoted 44 lines
drivers/gpu/drm/i915/i915_sw_fence.c +17
-1+ } +INIT_LIST_HEAD (&wq->task_list); - wq->flags =0 ; + wq->flags = pending; wq->func = i915_sw_fence_wake; wq->private =i915_sw_fence_get (fence);@@ -226,6 +244,20 @@ int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence,return pending; } +int i915_sw_fence_await_sw_fence (struct i915_sw_fence *fence, +struct i915_sw_fence *signaler, +wait_queue_t *wq) +{ +return __i915_sw_fence_await_sw_fence (fence, signaler, wq,0 ); +} + +int i915_sw_fence_await_sw_fence_gfp (struct i915_sw_fence *fence, +struct i915_sw_fence *signaler, +gfp_t gfp) +{ +return __i915_sw_fence_await_sw_fence (fence, signaler,NULL , gfp); +} +struct dma_fence_cb {struct fence_cb base;struct i915_sw_fence *fence;drivers/gpu/drm/i915/i915_sw_fence.h +3
-0@@ -46,6 +46,9 @@ void i915_sw_fence_commit(struct i915_sw_fence *fence);int i915_sw_fence_await_sw_fence (struct i915_sw_fence *fence,struct i915_sw_fence *after,wait_queue_t *wq); +int i915_sw_fence_await_sw_fence_gfp (struct i915_sw_fence *fence, +struct i915_sw_fence *after, +gfp_t gfp); int i915_sw_fence_await_dma_fence(struct i915_sw_fence *fence, struct fence *dma, unsigned long timeout,
drivers/gpu/drm/i915/i915_sw_fence.h static inline bool i915_sw_fence_done(const struct i915_sw_fence *fence) · line 73
Quoted 9 lines
drivers/gpu/drm/i915/i915_sw_fence.h +5
-0@@ -62,4 +65,9 @@ static inline bool i915_sw_fence_done(const struct i915_sw_fence *fence) return atomic_read(&fence->pending) < 0; } +static inline void i915_sw_fence_wait (struct i915_sw_fence *fence) +{ +wait_event (fence->wait,i915_sw_fence_done (fence)); +} +#endif /* _I915_SW_FENCE_H_ */
Regards,
Tvrtko
_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
Re: [PATCH 07/42] drm/i915: Allow i915_sw_fence_await_sw_fence() to allocate
Quoted 14 lines · Tvrtko Ursulin
Quoted 12 lines · Chris Wilson
In forthcoming patches, we want to be able to dynamically allocate the
wait_queue_tused whilst awaiting. This is more convenient if we extend thei915_sw_fence_await_sw_fence()to perform the allocation for us if we pass in a gfp mask as an alternative than a preallocated struct.Signed-off-by Chris Wilson <[email protected]>Reviewed-by Joonas Lahtinen <[email protected]>--- drivers/gpu/drm/i915/i915_sw_fence.c | 40 ++++++++++++++++++++++++++++++++---- drivers/gpu/drm/i915/i915_sw_fence.h | 8 ++++++++ 2 files changed, 44 insertions(+), 4 deletions(-)
Quoted 11 lines
drivers/gpu/drm/i915/i915_sw_fence.c line 15
Quoted 8 lines
drivers/gpu/drm/i915/i915_sw_fence.c +2
-0@@ -13,6 +13,8 @@#include "i915_sw_fence.h" +#define I915_SW_FENCE_FLAG_ALLOC BIT(0) +You collide with
WQ_FLAG_EXCLUSIVEhere, so I assume i915 will have complete control of the wq in question?
Hmm, we should move bit as well, but yes the flag is atm private to the wake up function for which we use a custom function.
Quoted 10 lines
drivers/gpu/drm/i915/i915_sw_fence.c int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence, · line 220
Quoted 6 lines
drivers/gpu/drm/i915/i915_sw_fence.c +6
-0+ wq =kmalloc (sizeof (*wq), gfp); +if (!wq) { +if (!gfpflags_allow_blocking (gfp)) +return -ENOMEM ; + +i915_sw_fence_wait (signaler);This looks like a strange API, unless I am missing something - it normally sets things up for waiting, unless the allocation fails when it actually does the wait?
Yes. If we blocked on the malloc and failed, we block a little more for the completion. iirc, the idea came from the async task.
Quoted 7 lines
drivers/gpu/drm/i915/i915_sw_fence.c int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence, · line 224
Quoted 5 lines
drivers/gpu/drm/i915/i915_sw_fence.c +4
-0+return 0 ; + } + + pending |=I915_SW_FENCE_FLAG_ALLOC ;Why is this stored in pending? Just because it is around as re-used?
Urm, yes. s/pending/tmp/
-Chris
Signature · 5 lines
Chris Wilson, Intel Open Source Technology Centre _______________________________________________ Intel-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/intel-gfx
Re: [PATCH 07/42] drm/i915: Allow i915_sw_fence_await_sw_fence() to allocate
Quoted 36 lines · Chris Wilson
Quoted 13 lines · Tvrtko Ursulin
Quoted 12 lines · Chris Wilson
In forthcoming patches, we want to be able to dynamically allocate the
wait_queue_tused whilst awaiting. This is more convenient if we extend thei915_sw_fence_await_sw_fence()to perform the allocation for us if we pass in a gfp mask as an alternative than a preallocated struct.Signed-off-by Chris Wilson <[email protected]>Reviewed-by Joonas Lahtinen <[email protected]>--- drivers/gpu/drm/i915/i915_sw_fence.c | 40 ++++++++++++++++++++++++++++++++---- drivers/gpu/drm/i915/i915_sw_fence.h | 8 ++++++++ 2 files changed, 44 insertions(+), 4 deletions(-)Quoted 10 lines
drivers/gpu/drm/i915/i915_sw_fence.c line 15
Quoted 8 lines
drivers/gpu/drm/i915/i915_sw_fence.c +2
-0@@ -13,6 +13,8 @@#include "i915_sw_fence.h" +#define I915_SW_FENCE_FLAG_ALLOC BIT(0) +You collide with
WQ_FLAG_EXCLUSIVEhere, so I assume i915 will have complete control of the wq in question?Hmm, we should move bit as well, but yes the flag is atm private to the wake up function for which we use a custom function.
Quoted 9 lines
drivers/gpu/drm/i915/i915_sw_fence.c int i915_sw_fence_await_sw_fence(struct i915_sw_fence *fence, · line 220
Quoted 6 lines
drivers/gpu/drm/i915/i915_sw_fence.c +6
-0+ wq =kmalloc (sizeof (*wq), gfp); +if (!wq) { +if (!gfpflags_allow_blocking (gfp)) +return -ENOMEM ; + +i915_sw_fence_wait (signaler);This looks like a strange API, unless I am missing something - it normally sets things up for waiting, unless the allocation fails when it actually does the wait?
Quoted 2 lines
Yes. If we blocked on the malloc and failed, we block a little more for the completion. iirc, the idea came from the async task.
Still sounds strange to me. I understand it is a unlikely corner case but sounds like that sort of decision should happen in the caller. I will have to revisit the i915_sw_fence implementation and usage in more detail it seems.
Regards,
Tvrtko
_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH 8/42 drm/i915: Rearrange i915_wait_request() accounting with callers
In reply to Chris WilsonOur low-level wait routine has evolved from our generic wait interface that handled unlocked, RPS boosting, waits with time tracking. If we push our GEM fence tracking to use reservation_objects (required for handling multiple timelines), we lose the ability to pass the required information down to i915_wait_request(). However, if we push the extra functionality from i915_wait_request() to the individual callsites (i915_gem_object_wait_rendering and i915_gem_wait_ioctl) that make use of those extras, we can both simplify our low level wait and prepare for extending the GEM interface for use of reservation_objects.
--- drivers/gpu/drm/i915/i915_drv.h | 7 +- drivers/gpu/drm/i915/i915_gem.c | 301 +++++++++++++++++++++++--------- drivers/gpu/drm/i915/i915_gem_request.c | 119 ++----------- drivers/gpu/drm/i915/i915_gem_request.h | 32 ++-- drivers/gpu/drm/i915/i915_gem_userptr.c | 12 +- drivers/gpu/drm/i915/intel_display.c | 27 +-- drivers/gpu/drm/i915/intel_ringbuffer.c | 14 +- drivers/gpu/drm/i915/intel_ringbuffer.h | 3 +- 8 files changed, 283 insertions(+), 232 deletions(-)
drivers/gpu/drm/i915/i915_drv.h +4 -3
@@ -3301,9 +3301,10 @@ int __must_check i915_gem_wait_for_idle(struct drm_i915_private *dev_priv,int __must_check i915_gem_suspend (struct drm_device *dev);void i915_gem_resume (struct drm_device *dev);int i915_gem_fault (struct vm_area_struct *vma,struct vm_fault *vmf); -int __must_check -i915_gem_object_wait_rendering (struct drm_i915_gem_object *obj, -bool readonly); +int i915_gem_object_wait (struct drm_i915_gem_object *obj, +unsigned int flags, +long timeout, +struct intel_rps_client *rps);int __must_check i915_gem_object_set_to_gtt_domain (struct drm_i915_gem_object *obj,bool write);
drivers/gpu/drm/i915/i915_gem.c +219 -82
443 lines not on this page · raw message
drivers/gpu/drm/i915/i915_gem_request.c +18 -101
179 lines not on this page · raw message
drivers/gpu/drm/i915/i915_gem_request.h +16 -16
@@ -228,13 +228,13 @@ struct intel_rps_client;#define IS_RPS_CLIENT (p) (!IS_ERR(p))#define IS_RPS_USER (p) (!IS_ERR_OR_NULL(p)) -int i915_wait_request (struct drm_i915_gem_request *req, -unsigned int flags, -s64 *timeout, -struct intel_rps_client *rps) +long i915_wait_request (struct drm_i915_gem_request *req, +unsigned int flags, +long timeout) __attribute__((nonnull (1 )));#define I915_WAIT_INTERRUPTIBLE BIT(0)#define I915_WAIT_LOCKED BIT(1)/* struct_mutex held, handle GPU reset */ +#define I915_WAIT_ALL BIT(2)/* used by i915_gem_object_wait() */ static inline u32 intel_engine_get_seqno (struct intel_engine_cs *engine);
@@ -583,14 +583,16 @@ static inline int __must_check i915_gem_active_wait(const struct i915_gem_active *active, struct mutex *mutex) { struct drm_i915_gem_request *request; +long ret; request =i915_gem_active_peek (active, mutex);if (!request)return 0 ; -return i915_wait_request (request, -I915_WAIT_INTERRUPTIBLE |I915_WAIT_LOCKED , -NULL ,NULL ); + ret =i915_wait_request (request, +I915_WAIT_INTERRUPTIBLE |I915_WAIT_LOCKED , +MAX_SCHEDULE_TIMEOUT ); +return ret <0 ? ret :0 ; }/**
@@ -617,20 +619,18 @@ i915_gem_active_wait(const struct i915_gem_active *active, struct mutex *mutex)*/ static inline int i915_gem_active_wait_unlocked (const struct i915_gem_active *active, -unsigned int flags, -s64 *timeout, -struct intel_rps_client *rps) +unsigned int flags) {struct drm_i915_gem_request *request; -int ret =0 ; +long ret =0 ; request =i915_gem_active_get_unlocked (active);if (request) { - ret =i915_wait_request (request, flags, timeout, rps); + ret =i915_wait_request (request, flags,MAX_SCHEDULE_TIMEOUT );i915_gem_request_put (request); } -return ret; +return ret <0 ? ret :0 ; }/**
@@ -647,7 +647,7 @@ i915_gem_active_retire(struct i915_gem_active *active, struct mutex *mutex) {struct drm_i915_gem_request *request; -int ret; +long ret; request =i915_gem_active_raw (active, mutex);if (!request)
@@ -655,8 +655,8 @@ i915_gem_active_retire(struct i915_gem_active *active, ret =i915_wait_request (request,I915_WAIT_INTERRUPTIBLE |I915_WAIT_LOCKED , -NULL ,NULL ); -if (ret) +MAX_SCHEDULE_TIMEOUT ); +if (ret <0 )return ret;list_del_init (&active->link);
drivers/gpu/drm/i915/i915_gem_userptr.c +1 -11
@@ -61,23 +61,13 @@ struct i915_mmu_object {bool attached; }; -static void wait_rendering (struct drm_i915_gem_object *obj) -{ -unsigned long active =__I915_BO_ACTIVE (obj); -int idx; - -for_each_active (active, idx) -i915_gem_active_wait_unlocked (&obj->last_read[idx], -0 ,NULL ,NULL ); -} -static void cancel_userptr (struct work_struct *work) {struct i915_mmu_object *mo =container_of (work,typeof (*mo), work);struct drm_i915_gem_object *obj = mo->obj;struct drm_device *dev = obj->base.dev; -wait_rendering (obj); +i915_gem_object_wait (obj,I915_WAIT_ALL ,MAX_SCHEDULE_TIMEOUT ,NULL );mutex_lock (&dev->struct_mutex);/* Cancel any active worker and force us to re-evaluate gup */
drivers/gpu/drm/i915/intel_display.c +16 -11
@@ -12042,7 +12042,7 @@ static void intel_mmio_flip_work_func(struct work_struct *w)if (work->flip_queued_req)WARN_ON (i915_wait_request (work->flip_queued_req, -0 ,NULL ,NO_WAITBOOST )); +0 ,MAX_SCHEDULE_TIMEOUT ) <0 );/* For framebuffer backed by dmabuf, wait for fence */ resv =i915_gem_object_get_dmabuf_resv (obj);
@@ -14089,19 +14089,21 @@ static int intel_atomic_prepare_commit(struct drm_device *dev,for_each_plane_in_state (state, plane, plane_state, i) {struct intel_plane_state *intel_plane_state =to_intel_plane_state (plane_state); +long timeout;if (!intel_plane_state->wait_req)continue ; - ret =i915_wait_request (intel_plane_state->wait_req, -I915_WAIT_INTERRUPTIBLE , -NULL ,NULL ); -if (ret) { + timeout =i915_wait_request (intel_plane_state->wait_req, +I915_WAIT_INTERRUPTIBLE , +MAX_SCHEDULE_TIMEOUT ); +if (timeout <0 ) {/* Any hang should be swallowed by the wait */ -WARN_ON (ret == -EIO ); +WARN_ON (timeout == -EIO );mutex_lock (&dev->struct_mutex);drm_atomic_helper_cleanup_planes (dev, state);mutex_unlock (&dev->struct_mutex); + ret = timeout;break ; } }
@@ -14303,7 +14305,7 @@ static void intel_atomic_commit_tail(struct drm_atomic_state *state)bool hw_check = intel_state->modeset;unsigned long put_domains[I915_MAX_PIPES ] = {};unsigned crtc_vblank_mask =0 ; -int i, ret; +int i;for_each_plane_in_state (state, plane, plane_state, i) { struct intel_plane_state *intel_plane_state =
@@ -14312,11 +14314,10 @@ static void intel_atomic_commit_tail(struct drm_atomic_state *state)if (!intel_plane_state->wait_req)continue ; - ret =i915_wait_request (intel_plane_state->wait_req, -0 ,NULL ,NULL );/* EIO should be eaten, and we can't get interrupted in the * worker, and blocking commits have waited already. */ -WARN_ON (ret); +WARN_ON (i915_wait_request (intel_plane_state->wait_req, +0 ,MAX_SCHEDULE_TIMEOUT ) <0 ); }drm_atomic_helper_wait_for_dependencies (state);
@@ -14679,7 +14680,11 @@ intel_prepare_plane_fb(struct drm_plane *plane,* can safely continue. */ if (needs_modeset (crtc_state)) - ret =i915_gem_object_wait_rendering (old_obj, true); + ret =i915_gem_object_wait (old_obj, +I915_WAIT_INTERRUPTIBLE | +I915_WAIT_LOCKED , +MAX_SCHEDULE_TIMEOUT , +NULL );if (ret) {/* GPU hangs should have been swallowed by the wait */ WARN_ON (ret == -EIO );
drivers/gpu/drm/i915/intel_ringbuffer.c +8 -6
@@ -2156,7 +2156,9 @@ static int wait_for_space(struct drm_i915_gem_request *req, int bytes) {struct intel_ring *ring = req->ring;struct drm_i915_gem_request *target; -int ret; +long timeout; + +lockdep_assert_held (&req->i915->drm.struct_mutex);intel_ring_update_space (ring);if (ring->space >= bytes)
@@ -2186,11 +2188,11 @@ static int wait_for_space(struct drm_i915_gem_request *req, int bytes)if (WARN_ON (&target->ring_link == &ring->request_list))return -ENOSPC ; - ret =i915_wait_request (target, -I915_WAIT_INTERRUPTIBLE |I915_WAIT_LOCKED , -NULL ,NO_WAITBOOST ); -if (ret) -return ret; + timeout =i915_wait_request (target, +I915_WAIT_INTERRUPTIBLE |I915_WAIT_LOCKED , +MAX_SCHEDULE_TIMEOUT ); +if (timeout <0 ) +return timeout;i915_gem_request_retire_upto (target);
drivers/gpu/drm/i915/intel_ringbuffer.h +1 -2
@@ -530,8 +530,7 @@ static inline int intel_engine_idle(struct intel_engine_cs *engine, unsigned int flags) {/* Wait upon the last request to be completed */ -return i915_gem_active_wait_unlocked (&engine->last_request, - flags,NULL ,NULL ); +return i915_gem_active_wait_unlocked (&engine->last_request, flags); }int intel_init_render_ring_buffer (struct intel_engine_cs *engine);
Signature · 6 lines
2.9.3 _______________________________________________ Intel-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/intel-gfx
Joonas Lahtinen
maintainerRe: [PATCH 08/42] drm/i915: Rearrange i915_wait_request() accounting with callers
Quoted 9 lines · Chris Wilson
Our low-level wait routine has evolved from our generic wait interface that handled unlocked, RPS boosting, waits with time tracking. If we push our GEM fence tracking to use
reservation_objects(required for handling multiple timelines), we lose the ability to pass the required information down toi915_wait_request(). However, if we push the extra functionality fromi915_wait_request()to the individual callsites (i915_gem_object_wait_renderingandi915_gem_wait_ioctl) that make use of those extras, we can both simplify our low level wait and prepare for extending the GEM interface for use ofreservation_objects.
No changelog, no re-review.
Regards, Joonas
Signature · 7 lines
Joonas Lahtinen Open Source Technology Center Intel Corporation _______________________________________________ Intel-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH 9/42 drm/i915: Remove unused i915_gem_active_wait() in favour of _unlocked()
In reply to Chris WilsonSince we only use the more generic unlocked variant, just rename it as the normal i915_gem_active_wait(). The temporary cost is that we need to always acquire the reference in a RCU safe manner, but the benefit is that we will combine the common paths.
--- drivers/gpu/drm/i915/i915_gem_request.h | 34 +++------------------------------ drivers/gpu/drm/i915/intel_ringbuffer.h | 2 +- 2 files changed, 4 insertions(+), 32 deletions(-)
drivers/gpu/drm/i915/i915_gem_request.h +3 -31
@@ -569,40 +569,13 @@ i915_gem_active_is_idle(const struct i915_gem_active *active, }/** -* i915_gem_active_wait - waits until the request is completed -* @active - the active request on which to wait -* -* i915_gem_active_wait() waits until the request is completed before -* returning. Note that it does not guarantee that the request is -* retired first, see i915_gem_active_retire(). -* -* i915_gem_active_wait() returns immediately if the active -* request is already complete. -*/ -static inline int __must_check -i915_gem_active_wait (const struct i915_gem_active *active,struct mutex *mutex) -{ -struct drm_i915_gem_request *request; -long ret; - - request =i915_gem_active_peek (active, mutex); -if (!request) -return 0 ; - - ret =i915_wait_request (request, -I915_WAIT_INTERRUPTIBLE |I915_WAIT_LOCKED , -MAX_SCHEDULE_TIMEOUT ); -return ret <0 ? ret :0 ; -} - -/** -* i915_gem_active_wait_unlocked - waits until the request is completed +* i915_gem_active_wait- waits until the request is completed * @active - the active request on which to wait * @flags - how to wait * @timeout - how long to wait at most * @rps - userspace client to charge for a waitboost * -* i915_gem_active_wait_unlocked() waits until the request is completed before +* i915_gem_active_wait() waits until the request is completed before * returning, without requiring any locks to be held. Note that it does not * retire any requests before returning. *
@@ -618,8 +591,7 @@ i915_gem_active_wait(const struct i915_gem_active *active, struct mutex *mutex)* Returns 0 if successful, or a negative error code. */ static inline int -i915_gem_active_wait_unlocked (const struct i915_gem_active *active, -unsigned int flags) +i915_gem_active_wait (const struct i915_gem_active *active,unsigned int flags) {struct drm_i915_gem_request *request;long ret =0 ;
drivers/gpu/drm/i915/intel_ringbuffer.h +1 -1
@@ -530,7 +530,7 @@ static inline int intel_engine_idle(struct intel_engine_cs *engine, unsigned int flags) {/* Wait upon the last request to be completed */ -return i915_gem_active_wait_unlocked (&engine->last_request, flags); +return i915_gem_active_wait (&engine->last_request, flags); }int intel_init_render_ring_buffer (struct intel_engine_cs *engine);
Signature · 6 lines
2.9.3 _______________________________________________ Intel-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH 10/42 drm/i915: Defer active reference until required
In reply to Chris WilsonWe only need the active reference to keep the object alive after the handle has been deleted (so as to prevent a synchronous gem_close). Why then pay the price of a kref on every execbuf when we can insert that final active ref just in time for the handle deletion?
--- drivers/gpu/drm/i915/i915_drv.h | 28 ++++++++++++++++++++++++++++ drivers/gpu/drm/i915/i915_gem.c | 22 +++++++++++++++++++++- drivers/gpu/drm/i915/i915_gem_batch_pool.c | 2 +- drivers/gpu/drm/i915/i915_gem_context.c | 2 +- drivers/gpu/drm/i915/i915_gem_execbuffer.c | 2 -- drivers/gpu/drm/i915/i915_gem_gtt.c | 7 ++++++- drivers/gpu/drm/i915/i915_gem_render_state.c | 3 ++- drivers/gpu/drm/i915/intel_ringbuffer.c | 15 ++++++++++++--- 8 files changed, 71 insertions(+), 10 deletions(-)
drivers/gpu/drm/i915/i915_drv.h +28 -0
@@ -2232,6 +2232,12 @@ struct drm_i915_gem_object { ((READ_ONCE ((bo)->flags) >>I915_BO_ACTIVE_SHIFT ) &I915_BO_ACTIVE_MASK )/** +* Have we taken a reference for the object for incomplete GPU +* activity? + */ +#define I915_BO_ACTIVE_REF (I915_BO_ACTIVE_SHIFT + I915_NUM_ENGINES) + +/** * This is set if the object has been written to since last bound * to the GTT */
@@ -2399,6 +2405,28 @@ i915_gem_object_has_active_engine(const struct drm_i915_gem_object *obj,return obj->flags &BIT (engine +I915_BO_ACTIVE_SHIFT ); } +static inline bool +i915_gem_object_has_active_reference (const struct drm_i915_gem_object *obj) +{ +return test_bit (I915_BO_ACTIVE_REF , &obj->flags); +} + +static inline void +i915_gem_object_set_active_reference (struct drm_i915_gem_object *obj) +{ +lockdep_assert_held (&obj->base.dev->struct_mutex); +__set_bit (I915_BO_ACTIVE_REF , &obj->flags); +} + +static inline void +i915_gem_object_clear_active_reference (struct drm_i915_gem_object *obj) +{ +lockdep_assert_held (&obj->base.dev->struct_mutex); +__clear_bit (I915_BO_ACTIVE_REF , &obj->flags); +} + +void __i915_gem_object_release_unless_active (struct drm_i915_gem_object *obj); +static inline unsigned int i915_gem_object_get_tiling (struct drm_i915_gem_object *obj) {
drivers/gpu/drm/i915/i915_gem.c +21 -1
@@ -2618,7 +2618,10 @@ i915_gem_object_retire__read(struct i915_gem_active *active,list_move_tail (&obj->global_list, &request->i915->mm.bound_list); -i915_gem_object_put (obj); +if (i915_gem_object_has_active_reference (obj)) { +i915_gem_object_clear_active_reference (obj); +i915_gem_object_put (obj); + } }static bool i915_context_is_banned (const struct i915_gem_context *ctx)
@@ -2889,6 +2892,12 @@ void i915_gem_close_object(struct drm_gem_object *gem, struct drm_file *file)list_for_each_entry_safe (vma, vn, &obj->vma_list, obj_link)if (vma->vm->file == fpriv)i915_vma_close (vma); + +if (i915_gem_object_is_active (obj) && + !i915_gem_object_has_active_reference (obj)) { +i915_gem_object_set_active_reference (obj); +i915_gem_object_get (obj); + }mutex_unlock (&obj->base.dev->struct_mutex); }
@@ -4365,6 +4374,17 @@ void i915_gem_free_object(struct drm_gem_object *gem_obj)intel_runtime_pm_put (dev_priv); } +void __i915_gem_object_release_unless_active (struct drm_i915_gem_object *obj) +{ +lockdep_assert_held (&obj->base.dev->struct_mutex); + +GEM_BUG_ON (i915_gem_object_has_active_reference (obj)); +if (i915_gem_object_is_active (obj)) +i915_gem_object_set_active_reference (obj); +else +i915_gem_object_put (obj); +} +int i915_gem_suspend (struct drm_device *dev) {struct drm_i915_private *dev_priv =to_i915 (dev);
drivers/gpu/drm/i915/i915_gem_batch_pool.c +1 -1
@@ -73,7 +73,7 @@ void i915_gem_batch_pool_fini(struct i915_gem_batch_pool *pool)list_for_each_entry_safe (obj, next, &pool->cache_list[n], batch_pool_link) -i915_gem_object_put (obj); +__i915_gem_object_release_unless_active (obj);INIT_LIST_HEAD (&pool->cache_list[n]); }
drivers/gpu/drm/i915/i915_gem_context.c +1 -1
@@ -155,7 +155,7 @@ void i915_gem_context_free(struct kref *ctx_ref)if (ce->ring)intel_ring_free (ce->ring); -i915_vma_put (ce->state); +__i915_gem_object_release_unless_active (ce->state->obj); }put_pid (ctx->pid);
drivers/gpu/drm/i915/i915_gem_execbuffer.c +0 -2
@@ -1299,8 +1299,6 @@ void i915_vma_move_to_active(struct i915_vma *vma,* add the active reference first and queue for it to be dropped * *last*. */ -if (!i915_gem_object_is_active (obj)) -i915_gem_object_get (obj);i915_gem_object_set_active (obj, idx);i915_gem_active_set (&obj->last_read[idx], req);
drivers/gpu/drm/i915/i915_gem_gtt.c +6 -1
@@ -3712,11 +3712,16 @@ void __iomem *i915_vma_pin_iomap(struct i915_vma *vma)void i915_vma_unpin_and_release (struct i915_vma **p_vma) {struct i915_vma *vma; +struct drm_i915_gem_object *obj; vma =fetch_and_zero (p_vma);if (!vma)return ; + obj = vma->obj; +i915_vma_unpin (vma); -i915_vma_put (vma); +i915_vma_close (vma); + +__i915_gem_object_release_unless_active (obj); }
drivers/gpu/drm/i915/i915_gem_render_state.c +2 -1
@@ -224,7 +224,8 @@ int i915_gem_render_state_init(struct drm_i915_gem_request *req)i915_vma_move_to_active (so.vma, req,0 ); err_unpin:i915_vma_unpin (so.vma); +i915_vma_close (so.vma); err_obj: -i915_gem_object_put (obj); +__i915_gem_object_release_unless_active (obj);return ret; }
drivers/gpu/drm/i915/intel_ringbuffer.c +12 -3
@@ -1763,14 +1763,19 @@ static void cleanup_phys_status_page(struct intel_engine_cs *engine)static void cleanup_status_page (struct intel_engine_cs *engine) {struct i915_vma *vma; +struct drm_i915_gem_object *obj; vma =fetch_and_zero (&engine->status_page.vma);if (!vma)return ; + obj = vma->obj; +i915_vma_unpin (vma); -i915_gem_object_unpin_map (vma->obj); -i915_vma_put (vma); +i915_vma_close (vma); + +i915_gem_object_unpin_map (obj); +__i915_gem_object_release_unless_active (obj); }static int init_status_page (struct intel_engine_cs *engine)
@@ -1968,7 +1973,11 @@ intel_engine_create_ring(struct intel_engine_cs *engine, int size)void intel_ring_free (struct intel_ring *ring) { -i915_vma_put (ring->vma); +struct drm_i915_gem_object *obj = ring->vma->obj; + +i915_vma_close (ring->vma); +__i915_gem_object_release_unless_active (obj); +kfree (ring); }
Signature · 6 lines
2.9.3 _______________________________________________ Intel-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/intel-gfx
Re: [PATCH 10/42] drm/i915: Defer active reference until required
Quoted 4 lines · Chris Wilson
We only need the active reference to keep the object alive after the handle has been deleted (so as to prevent a synchronous
gem_close). Why then pay the price of a kref on every execbuf when we can insert that final active ref just in time for the handle deletion?
I really dislike this. Where there was elegance with obj/vma_put, it is now replaced with out of place looking __i915_gem_object_release_unless_active. I don't see why would higher level layers have to concern themselves with calling something with such a low-level sounding name.
How much does this influence performance and in what cases? If significant, could we try to come up with something similar but more elegant?
Regards,
Tvrtko
Quoted 210 lines
Signed-off-by Chris Wilson <[email protected]>Reviewed-by Joonas Lahtinen <[email protected]>--- drivers/gpu/drm/i915/i915_drv.h | 28 ++++++++++++++++++++++++++++ drivers/gpu/drm/i915/i915_gem.c | 22 +++++++++++++++++++++- drivers/gpu/drm/i915/i915_gem_batch_pool.c | 2 +- drivers/gpu/drm/i915/i915_gem_context.c | 2 +- drivers/gpu/drm/i915/i915_gem_execbuffer.c | 2 -- drivers/gpu/drm/i915/i915_gem_gtt.c | 7 ++++++- drivers/gpu/drm/i915/i915_gem_render_state.c | 3 ++- drivers/gpu/drm/i915/intel_ringbuffer.c | 15 ++++++++++++--- 8 files changed, 71 insertions(+), 10 deletions(-)drivers/gpu/drm/i915/i915_drv.h +28
-0@@ -2232,6 +2232,12 @@ struct drm_i915_gem_object { ((READ_ONCE ((bo)->flags) >>I915_BO_ACTIVE_SHIFT ) &I915_BO_ACTIVE_MASK )/** +* Have we taken a reference for the object for incomplete GPU +* activity? + */ +#define I915_BO_ACTIVE_REF (I915_BO_ACTIVE_SHIFT + I915_NUM_ENGINES) + +/** * This is set if the object has been written to since last bound * to the GTT */ @@ -2399,6 +2405,28 @@ i915_gem_object_has_active_engine(const struct drm_i915_gem_object *obj,return obj->flags &BIT (engine +I915_BO_ACTIVE_SHIFT ); } +static inline bool +i915_gem_object_has_active_reference (const struct drm_i915_gem_object *obj) +{ +return test_bit (I915_BO_ACTIVE_REF , &obj->flags); +} + +static inline void +i915_gem_object_set_active_reference (struct drm_i915_gem_object *obj) +{ +lockdep_assert_held (&obj->base.dev->struct_mutex); +__set_bit (I915_BO_ACTIVE_REF , &obj->flags); +} + +static inline void +i915_gem_object_clear_active_reference (struct drm_i915_gem_object *obj) +{ +lockdep_assert_held (&obj->base.dev->struct_mutex); +__clear_bit (I915_BO_ACTIVE_REF , &obj->flags); +} + +void __i915_gem_object_release_unless_active (struct drm_i915_gem_object *obj); +static inline unsigned int i915_gem_object_get_tiling (struct drm_i915_gem_object *obj) {drivers/gpu/drm/i915/i915_gem.c +21
-1@@ -2618,7 +2618,10 @@ i915_gem_object_retire__read(struct i915_gem_active *active,list_move_tail (&obj->global_list, &request->i915->mm.bound_list); -i915_gem_object_put (obj); +if (i915_gem_object_has_active_reference (obj)) { +i915_gem_object_clear_active_reference (obj); +i915_gem_object_put (obj); + } }static bool i915_context_is_banned (const struct i915_gem_context *ctx)@@ -2889,6 +2892,12 @@ void i915_gem_close_object(struct drm_gem_object *gem, struct drm_file *file)list_for_each_entry_safe (vma, vn, &obj->vma_list, obj_link)if (vma->vm->file == fpriv)i915_vma_close (vma); + +if (i915_gem_object_is_active (obj) && + !i915_gem_object_has_active_reference (obj)) { +i915_gem_object_set_active_reference (obj); +i915_gem_object_get (obj); + }mutex_unlock (&obj->base.dev->struct_mutex); }@@ -4365,6 +4374,17 @@ void i915_gem_free_object(struct drm_gem_object *gem_obj)intel_runtime_pm_put (dev_priv); } +void __i915_gem_object_release_unless_active (struct drm_i915_gem_object *obj) +{ +lockdep_assert_held (&obj->base.dev->struct_mutex); + +GEM_BUG_ON (i915_gem_object_has_active_reference (obj)); +if (i915_gem_object_is_active (obj)) +i915_gem_object_set_active_reference (obj); +else +i915_gem_object_put (obj); +} +int i915_gem_suspend (struct drm_device *dev) {struct drm_i915_private *dev_priv =to_i915 (dev);drivers/gpu/drm/i915/i915_gem_batch_pool.c +1
-1@@ -73,7 +73,7 @@ void i915_gem_batch_pool_fini(struct i915_gem_batch_pool *pool)list_for_each_entry_safe (obj, next, &pool->cache_list[n], batch_pool_link) -i915_gem_object_put (obj); +__i915_gem_object_release_unless_active (obj);INIT_LIST_HEAD (&pool->cache_list[n]); }drivers/gpu/drm/i915/i915_gem_context.c +1
-1@@ -155,7 +155,7 @@ void i915_gem_context_free(struct kref *ctx_ref)if (ce->ring)intel_ring_free (ce->ring); -i915_vma_put (ce->state); +__i915_gem_object_release_unless_active (ce->state->obj); }put_pid (ctx->pid);drivers/gpu/drm/i915/i915_gem_execbuffer.c +0
-2@@ -1299,8 +1299,6 @@ void i915_vma_move_to_active(struct i915_vma *vma,* add the active reference first and queue for it to be dropped * *last*. */ -if (!i915_gem_object_is_active (obj)) -i915_gem_object_get (obj);i915_gem_object_set_active (obj, idx);i915_gem_active_set (&obj->last_read[idx], req);drivers/gpu/drm/i915/i915_gem_gtt.c +6
-1@@ -3712,11 +3712,16 @@ void __iomem *i915_vma_pin_iomap(struct i915_vma *vma)void i915_vma_unpin_and_release (struct i915_vma **p_vma) {struct i915_vma *vma; +struct drm_i915_gem_object *obj; vma =fetch_and_zero (p_vma);if (!vma)return ; + obj = vma->obj; +i915_vma_unpin (vma); -i915_vma_put (vma); +i915_vma_close (vma); + +__i915_gem_object_release_unless_active (obj); }drivers/gpu/drm/i915/i915_gem_render_state.c +2
-1@@ -224,7 +224,8 @@ int i915_gem_render_state_init(struct drm_i915_gem_request *req)i915_vma_move_to_active (so.vma, req,0 ); err_unpin:i915_vma_unpin (so.vma); +i915_vma_close (so.vma); err_obj: -i915_gem_object_put (obj); +__i915_gem_object_release_unless_active (obj);return ret; }drivers/gpu/drm/i915/intel_ringbuffer.c +7
-2@@ -1763,14 +1763,19 @@ static void cleanup_phys_status_page(struct intel_engine_cs *engine)static void cleanup_status_page (struct intel_engine_cs *engine) {struct i915_vma *vma; +struct drm_i915_gem_object *obj; vma =fetch_and_zero (&engine->status_page.vma);if (!vma)return ; + obj = vma->obj; +i915_vma_unpin (vma); -i915_gem_object_unpin_map (vma->obj); -i915_vma_put (vma); +i915_vma_close (vma); + +i915_gem_object_unpin_map (obj); +__i915_gem_object_release_unless_active (obj); }static int init_status_page (struct intel_engine_cs *engine)@@ -1968,7 +1973,11 @@ intel_engine_create_ring(struct intel_engine_cs *engine, int size)void intel_ring_free (struct intel_ring *ring)
drivers/gpu/drm/i915/intel_ringbuffer.c intel_engine_create_ring(struct intel_engine_cs *engine, int size) · line 1983
Quoted 10 lines
drivers/gpu/drm/i915/intel_ringbuffer.c +5
-1{ -i915_vma_put (ring->vma); +struct drm_i915_gem_object *obj = ring->vma->obj; + +i915_vma_close (ring->vma); +__i915_gem_object_release_unless_active (obj); +kfree (ring); }
_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
Re: [PATCH 10/42] drm/i915: Defer active reference until required
Quoted 13 lines · Tvrtko Ursulin
Quoted 4 lines · Chris Wilson
We only need the active reference to keep the object alive after the handle has been deleted (so as to prevent a synchronous
gem_close). Why then pay the price of a kref on every execbuf when we can insert that final active ref just in time for the handle deletion?I really dislike this. Where there was elegance with obj/vma_put, it is now replaced with out of place looking
__i915_gem_object_release_unless_active. I don't see why would higher level layers have to concern themselves with calling something with such a low-level sounding name.
Quoted 3 lines
How much does this influence performance and in what cases? If significant, could we try to come up with something similar but more elegant?
Back in the day, this was one of the most frequent atomic operations we did. And whilst perf overemphasizes the stalls from locked instructions, the sheer numbers of them we do are significant (since we do one at the start and end of every execbuf for every object in typical conditions). Whilst it is less significant in the face of obj->resv undoing all of the gains, it is still a deep paper cut. (At the GL level, consider about 100 objects per batch, several thousand times a second x 2, these ops are low hanging fruit.)
What's needed is a function to take the place of the close_object for internally allocated objects. It is also worth noting that they are either already part of a cache, or are suitable for caching.... -Chris
Quoted 210 lines
Regards,
Tvrtko
Quoted 204 lines
Signed-off-by Chris Wilson <[email protected]>Reviewed-by Joonas Lahtinen <[email protected]>--- drivers/gpu/drm/i915/i915_drv.h | 28 ++++++++++++++++++++++++++++ drivers/gpu/drm/i915/i915_gem.c | 22 +++++++++++++++++++++- drivers/gpu/drm/i915/i915_gem_batch_pool.c | 2 +- drivers/gpu/drm/i915/i915_gem_context.c | 2 +- drivers/gpu/drm/i915/i915_gem_execbuffer.c | 2 -- drivers/gpu/drm/i915/i915_gem_gtt.c | 7 ++++++- drivers/gpu/drm/i915/i915_gem_render_state.c | 3 ++- drivers/gpu/drm/i915/intel_ringbuffer.c | 15 ++++++++++++--- 8 files changed, 71 insertions(+), 10 deletions(-)drivers/gpu/drm/i915/i915_drv.h +28
-0@@ -2232,6 +2232,12 @@ struct drm_i915_gem_object { ((READ_ONCE ((bo)->flags) >>I915_BO_ACTIVE_SHIFT ) &I915_BO_ACTIVE_MASK )/** +* Have we taken a reference for the object for incomplete GPU +* activity? + */ +#define I915_BO_ACTIVE_REF (I915_BO_ACTIVE_SHIFT + I915_NUM_ENGINES) + +/** * This is set if the object has been written to since last bound * to the GTT */ @@ -2399,6 +2405,28 @@ i915_gem_object_has_active_engine(const struct drm_i915_gem_object *obj,return obj->flags &BIT (engine +I915_BO_ACTIVE_SHIFT ); } +static inline bool +i915_gem_object_has_active_reference (const struct drm_i915_gem_object *obj) +{ +return test_bit (I915_BO_ACTIVE_REF , &obj->flags); +} + +static inline void +i915_gem_object_set_active_reference (struct drm_i915_gem_object *obj) +{ +lockdep_assert_held (&obj->base.dev->struct_mutex); +__set_bit (I915_BO_ACTIVE_REF , &obj->flags); +} + +static inline void +i915_gem_object_clear_active_reference (struct drm_i915_gem_object *obj) +{ +lockdep_assert_held (&obj->base.dev->struct_mutex); +__clear_bit (I915_BO_ACTIVE_REF , &obj->flags); +} + +void __i915_gem_object_release_unless_active (struct drm_i915_gem_object *obj); +static inline unsigned int i915_gem_object_get_tiling (struct drm_i915_gem_object *obj) {drivers/gpu/drm/i915/i915_gem.c +21
-1@@ -2618,7 +2618,10 @@ i915_gem_object_retire__read(struct i915_gem_active *active,list_move_tail (&obj->global_list, &request->i915->mm.bound_list); -i915_gem_object_put (obj); +if (i915_gem_object_has_active_reference (obj)) { +i915_gem_object_clear_active_reference (obj); +i915_gem_object_put (obj); + } }static bool i915_context_is_banned (const struct i915_gem_context *ctx)@@ -2889,6 +2892,12 @@ void i915_gem_close_object(struct drm_gem_object *gem, struct drm_file *file)list_for_each_entry_safe (vma, vn, &obj->vma_list, obj_link)if (vma->vm->file == fpriv)i915_vma_close (vma); + +if (i915_gem_object_is_active (obj) && + !i915_gem_object_has_active_reference (obj)) { +i915_gem_object_set_active_reference (obj); +i915_gem_object_get (obj); + }mutex_unlock (&obj->base.dev->struct_mutex); }@@ -4365,6 +4374,17 @@ void i915_gem_free_object(struct drm_gem_object *gem_obj)intel_runtime_pm_put (dev_priv); } +void __i915_gem_object_release_unless_active (struct drm_i915_gem_object *obj) +{ +lockdep_assert_held (&obj->base.dev->struct_mutex); + +GEM_BUG_ON (i915_gem_object_has_active_reference (obj)); +if (i915_gem_object_is_active (obj)) +i915_gem_object_set_active_reference (obj); +else +i915_gem_object_put (obj); +} +int i915_gem_suspend (struct drm_device *dev) {struct drm_i915_private *dev_priv =to_i915 (dev);drivers/gpu/drm/i915/i915_gem_batch_pool.c +1
-1@@ -73,7 +73,7 @@ void i915_gem_batch_pool_fini(struct i915_gem_batch_pool *pool)list_for_each_entry_safe (obj, next, &pool->cache_list[n], batch_pool_link) -i915_gem_object_put (obj); +__i915_gem_object_release_unless_active (obj);INIT_LIST_HEAD (&pool->cache_list[n]); }drivers/gpu/drm/i915/i915_gem_context.c +1
-1@@ -155,7 +155,7 @@ void i915_gem_context_free(struct kref *ctx_ref)if (ce->ring)intel_ring_free (ce->ring); -i915_vma_put (ce->state); +__i915_gem_object_release_unless_active (ce->state->obj); }put_pid (ctx->pid);drivers/gpu/drm/i915/i915_gem_execbuffer.c +0
-2@@ -1299,8 +1299,6 @@ void i915_vma_move_to_active(struct i915_vma *vma,* add the active reference first and queue for it to be dropped * *last*. */ -if (!i915_gem_object_is_active (obj)) -i915_gem_object_get (obj);i915_gem_object_set_active (obj, idx);i915_gem_active_set (&obj->last_read[idx], req);drivers/gpu/drm/i915/i915_gem_gtt.c +6
-1@@ -3712,11 +3712,16 @@ void __iomem *i915_vma_pin_iomap(struct i915_vma *vma)void i915_vma_unpin_and_release (struct i915_vma **p_vma) {struct i915_vma *vma; +struct drm_i915_gem_object *obj; vma =fetch_and_zero (p_vma);if (!vma)return ; + obj = vma->obj; +i915_vma_unpin (vma); -i915_vma_put (vma); +i915_vma_close (vma); + +__i915_gem_object_release_unless_active (obj); }drivers/gpu/drm/i915/i915_gem_render_state.c +2
-1@@ -224,7 +224,8 @@ int i915_gem_render_state_init(struct drm_i915_gem_request *req)i915_vma_move_to_active (so.vma, req,0 ); err_unpin:i915_vma_unpin (so.vma); +i915_vma_close (so.vma); err_obj: -i915_gem_object_put (obj); +__i915_gem_object_release_unless_active (obj);return ret; }drivers/gpu/drm/i915/intel_ringbuffer.c +12
-3@@ -1763,14 +1763,19 @@ static void cleanup_phys_status_page(struct intel_engine_cs *engine)static void cleanup_status_page (struct intel_engine_cs *engine) {struct i915_vma *vma; +struct drm_i915_gem_object *obj; vma =fetch_and_zero (&engine->status_page.vma);if (!vma)return ; + obj = vma->obj; +i915_vma_unpin (vma); -i915_gem_object_unpin_map (vma->obj); -i915_vma_put (vma); +i915_vma_close (vma); + +i915_gem_object_unpin_map (obj); +__i915_gem_object_release_unless_active (obj); }static int init_status_page (struct intel_engine_cs *engine)@@ -1968,7 +1973,11 @@ intel_engine_create_ring(struct intel_engine_cs *engine, int size)void intel_ring_free (struct intel_ring *ring) { -i915_vma_put (ring->vma); +struct drm_i915_gem_object *obj = ring->vma->obj; + +i915_vma_close (ring->vma); +__i915_gem_object_release_unless_active (obj); +kfree (ring); }
Signature · 5 lines
Chris Wilson, Intel Open Source Technology Centre _______________________________________________ Intel-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/intel-gfx
Re: [PATCH 10/42] drm/i915: Defer active reference until required
Quoted 15 lines · Chris Wilson
Quoted 11 lines · Tvrtko Ursulin
Quoted 4 lines · Chris Wilson
We only need the active reference to keep the object alive after the handle has been deleted (so as to prevent a synchronous
gem_close). Why then pay the price of a kref on every execbuf when we can insert that final active ref just in time for the handle deletion?I really dislike this. Where there was elegance with obj/vma_put, it is now replaced with out of place looking
__i915_gem_object_release_unless_active. I don't see why would higher level layers have to concern themselves with calling something with such a low-level sounding name.Quoted 3 lines
How much does this influence performance and in what cases? If significant, could we try to come up with something similar but more elegant?
Quoted 8 lines
Back in the day, this was one of the most frequent atomic operations we did. And whilst perf overemphasizes the stalls from locked instructions, the sheer numbers of them we do are significant (since we do one at the start and end of every execbuf for every object in typical conditions). Whilst it is less significant in the face of
obj->resvundoing all of the gains, it is still a deep paper cut. (At the GL level, consider about 100 objects per batch, several thousand times a second x 2, these ops are low hanging fruit.)
I understand there is a lot of them (same is true for many other operations we do), but how does that translate to some benchmarks?
Quoted 3 lines
What's needed is a function to take the place of the
close_objectfor internally allocated objects. It is also worth noting that they are either already part of a cache, or are suitable for caching....
Ok but those are not x 100 per batch.
Regards,
Tvrtko
_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH 11/42 drm/i915: Introduce an internal allocator for disposable private objects
In reply to Chris WilsonQuite a few of our objects used for internal hardware programming do not benefit from being swappable or from being zero initialised. As such they do not benefit from using a shmemfs backing storage and since they are internal and never directly exposed to the user, we do not need to worry about providing a filp. For these we can use an drm_i915_gem_object wrapper around a sg_table of plain struct page. They are not swap backed and not automatically pinned. If they are reaped by the shrinker, the pages are released and the contents discarded. For the internal use case, this is fine as for example, ringbuffers are pinned from being written by a request to be read by the hardware. Once they are idle, they can be discarded entirely. As such they are a good match for execlist ringbuffers and a small variety of other internal objects.
In the first iteration, this is limited to the scratch batch buffers we use (for command parsing and state initialisation).
--- drivers/gpu/drm/i915/Makefile | 1 + drivers/gpu/drm/i915/i915_drv.h | 5 + drivers/gpu/drm/i915/i915_gem_batch_pool.c | 28 ++--- drivers/gpu/drm/i915/i915_gem_internal.c | 161 +++++++++++++++++++++++++++ drivers/gpu/drm/i915/i915_gem_render_state.c | 2 +- drivers/gpu/drm/i915/intel_engine_cs.c | 2 +- drivers/gpu/drm/i915/intel_ringbuffer.c | 14 ++- 7 files changed, 189 insertions(+), 24 deletions(-) create mode 100644 drivers/gpu/drm/i915/i915_gem_internal.c
drivers/gpu/drm/i915/Makefile +1 -0
@@ -35,6 +35,7 @@ i915-y += i915_cmd_parser.o \ i915_gem_execbuffer.o \ i915_gem_fence.o \ i915_gem_gtt.o \ + i915_gem_internal.o \ i915_gem.o \ i915_gem_render_state.o \ i915_gem_request.o \
drivers/gpu/drm/i915/i915_drv.h +5 -0
@@ -3538,6 +3538,11 @@ i915_gem_object_create_stolen_for_preallocated(struct drm_device *dev,u32 gtt_offset, u32 size); +/* i915_gem_internal.c */ +struct drm_i915_gem_object * +i915_gem_object_create_internal (struct drm_device *dev, +unsigned int size); +/* i915_gem_shrinker.c */ unsigned long i915_gem_shrink(struct drm_i915_private *dev_priv, unsigned long target,
drivers/gpu/drm/i915/i915_gem_batch_pool.c +9 -19
@@ -97,9 +97,9 @@ i915_gem_batch_pool_get(struct i915_gem_batch_pool *pool, size_t size) {struct drm_i915_gem_object *obj =NULL ; -struct drm_i915_gem_object *tmp, *next; +struct drm_i915_gem_object *tmp;struct list_head *list; -int n; +int n, ret;lockdep_assert_held (&pool->engine->i915->drm.struct_mutex);
@@ -112,19 +112,12 @@ i915_gem_batch_pool_get(struct i915_gem_batch_pool *pool, n =ARRAY_SIZE (pool->cache_list) -1 ; list = &pool->cache_list[n]; -list_for_each_entry_safe (tmp, next, list, batch_pool_link) { +list_for_each_entry (tmp, list, batch_pool_link) {/* The batches are strictly LRU ordered */ if (!i915_gem_active_is_idle (&tmp->last_read[pool->engine->id], &tmp->base.dev->struct_mutex))break ; -/* While we're looping, do some clean up */ -if (tmp->madv == __I915_MADV_PURGED) { -list_del (&tmp->batch_pool_link); -i915_gem_object_put (tmp); -continue ; - } -if (tmp->base.size >= size) { obj = tmp;break ;
@@ -132,19 +125,16 @@ i915_gem_batch_pool_get(struct i915_gem_batch_pool *pool, }if (obj ==NULL ) { -int ret; - - obj =i915_gem_object_create (&pool->engine->i915->drm, size); + obj =i915_gem_object_create_internal (&pool->engine->i915->drm, + size);if (IS_ERR (obj))return obj; - - ret =i915_gem_object_get_pages (obj); -if (ret) -return ERR_PTR (ret); - - obj->madv =I915_MADV_DONTNEED ; } + ret =i915_gem_object_get_pages (obj); +if (ret) +return ERR_PTR (ret); +list_move_tail (&obj->batch_pool_link, list);i915_gem_object_pin_pages (obj);return obj;
drivers/gpu/drm/i915/i915_gem_internal.c new file +161 -0
161 lines not on this page · raw message
drivers/gpu/drm/i915/i915_gem_render_state.c +1 -1
@@ -187,7 +187,7 @@ int i915_gem_render_state_init(struct drm_i915_gem_request *req)if (so.rodata->batch_items *4 >4096 )return -EINVAL ; - obj =i915_gem_object_create (&req->i915->drm,4096 ); + obj =i915_gem_object_create_internal (&req->i915->drm,4096 );if (IS_ERR (obj))return PTR_ERR (obj);
drivers/gpu/drm/i915/intel_engine_cs.c +1 -1
@@ -251,7 +251,7 @@ int intel_engine_create_scratch(struct intel_engine_cs *engine, int size) obj =i915_gem_object_create_stolen (&engine->i915->drm, size);if (!obj) - obj =i915_gem_object_create (&engine->i915->drm, size); + obj =i915_gem_object_create_internal (&engine->i915->drm, size);if (IS_ERR (obj)) {DRM_ERROR ("Failed to allocate scratch page\n" );return PTR_ERR (obj);
drivers/gpu/drm/i915/intel_ringbuffer.c +11 -3
@@ -1783,9 +1783,10 @@ static int init_status_page(struct intel_engine_cs *engine)struct drm_i915_gem_object *obj;struct i915_vma *vma;unsigned int flags; +void *vaddr;int ret; - obj =i915_gem_object_create (&engine->i915->drm,4096 ); + obj =i915_gem_object_create_internal (&engine->i915->drm,4096 );if (IS_ERR (obj)) {DRM_ERROR ("Failed to allocate status page\n" );return PTR_ERR (obj);
@@ -1818,15 +1819,22 @@ static int init_status_page(struct intel_engine_cs *engine)if (ret) goto err; + vaddr =i915_gem_object_pin_map (obj,I915_MAP_WB ); +if (IS_ERR (vaddr)) { + ret =PTR_ERR (vaddr); + goto err_unpin; + } + engine->status_page.vma = vma; engine->status_page.ggtt_offset =i915_ggtt_offset (vma); - engine->status_page.page_addr = -i915_gem_object_pin_map (obj,I915_MAP_WB ); + engine->status_page.page_addr =memset (vaddr,0 ,4096 );DRM_DEBUG_DRIVER ("%s hws offset: 0x%08x\n" , engine->name,i915_ggtt_offset (vma));return 0 ; +err_unpin: +i915_vma_unpin (vma); err:i915_gem_object_put (obj);return ret;
Signature · 6 lines
2.9.3 _______________________________________________ Intel-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/intel-gfx
Re: [PATCH 11/42] drm/i915: Introduce an internal allocator for disposable private objects
Quoted 14 lines · Chris Wilson
Quite a few of our objects used for internal hardware programming do not benefit from being swappable or from being zero initialised. As such they do not benefit from using a shmemfs backing storage and since they are internal and never directly exposed to the user, we do not need to worry about providing a filp. For these we can use an
drm_i915_gem_objectwrapper around asg_tableof plain struct page. They are not swap backed and not automatically pinned. If they are reaped by the shrinker, the pages are released and the contents discarded. For the internal use case, this is fine as for example, ringbuffers are pinned from being written by a request to be read by the hardware. Once they are idle, they can be discarded entirely. As such they are a good match for execlist ringbuffers and a small variety of other internal objects.
Quoted 3 lines
In the first iteration, this is limited to the scratch batch buffers we use (for command parsing and state initialisation).
No changelog, assuming otherwise intact but the typo fixed;
Regards, Joonas
Signature · 7 lines
Joonas Lahtinen Open Source Technology Center Intel Corporation _______________________________________________ Intel-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/intel-gfx
Re: [PATCH 11/42] drm/i915: Introduce an internal allocator for disposable private objects
In reply to Chris WilsonQuoted 46 lines · Chris Wilson
Quite a few of our objects used for internal hardware programming do not benefit from being swappable or from being zero initialised. As such they do not benefit from using a shmemfs backing storage and since they are internal and never directly exposed to the user, we do not need to worry about providing a filp. For these we can use an
drm_i915_gem_objectwrapper around asg_tableof plain struct page. They are not swap backed and not automatically pinned. If they are reaped by the shrinker, the pages are released and the contents discarded. For the internal use case, this is fine as for example, ringbuffers are pinned from being written by a request to be read by the hardware. Once they are idle, they can be discarded entirely. As such they are a good match for execlist ringbuffers and a small variety of other internal objects.In the first iteration, this is limited to the scratch batch buffers we use (for command parsing and state initialisation).
Signed-off-by Chris Wilson <[email protected]>--- drivers/gpu/drm/i915/Makefile | 1 + drivers/gpu/drm/i915/i915_drv.h | 5 + drivers/gpu/drm/i915/i915_gem_batch_pool.c | 28 ++--- drivers/gpu/drm/i915/i915_gem_internal.c | 161 +++++++++++++++++++++++++++ drivers/gpu/drm/i915/i915_gem_render_state.c | 2 +- drivers/gpu/drm/i915/intel_engine_cs.c | 2 +- drivers/gpu/drm/i915/intel_ringbuffer.c | 14 ++- 7 files changed, 189 insertions(+), 24 deletions(-)create mode 100644
drivers/gpu/drm/i915/i915_gem_internal.cdrivers/gpu/drm/i915/Makefile +1
-0@@ -35,6 +35,7 @@ i915-y += i915_cmd_parser.o \ i915_gem_execbuffer.o \ i915_gem_fence.o \ i915_gem_gtt.o \ + i915_gem_internal.o \ i915_gem.o \ i915_gem_render_state.o \ i915_gem_request.o \drivers/gpu/drm/i915/i915_drv.h +0
-0
drivers/gpu/drm/i915/i915_drv.h i915_gem_object_create_stolen_for_preallocated(struct drm_device *dev, · line 3545
Quoted 8 lines
drivers/gpu/drm/i915/i915_drv.h +5
-0@@ -3538,6 +3538,11 @@ i915_gem_object_create_stolen_for_preallocated(struct drm_device *dev,u32 gtt_offset, u32 size); +/* i915_gem_internal.c */ +struct drm_i915_gem_object * +i915_gem_object_create_internal (struct drm_device *dev, +unsigned int size); +
Wasn't size_t our convention for GEM objects?
Quoted 124 lines
drivers/gpu/drm/i915/i915_drv.h +0
-0/* i915_gem_shrinker.c */ unsigned long i915_gem_shrink(struct drm_i915_private *dev_priv, unsigned long target,drivers/gpu/drm/i915/i915_gem_batch_pool.c +9
-19@@ -97,9 +97,9 @@ i915_gem_batch_pool_get(struct i915_gem_batch_pool *pool, size_t size) {struct drm_i915_gem_object *obj =NULL ; -struct drm_i915_gem_object *tmp, *next; +struct drm_i915_gem_object *tmp;struct list_head *list; -int n; +int n, ret;lockdep_assert_held (&pool->engine->i915->drm.struct_mutex);@@ -112,19 +112,12 @@ i915_gem_batch_pool_get(struct i915_gem_batch_pool *pool, n =ARRAY_SIZE (pool->cache_list) -1 ; list = &pool->cache_list[n]; -list_for_each_entry_safe (tmp, next, list, batch_pool_link) { +list_for_each_entry (tmp, list, batch_pool_link) {/* The batches are strictly LRU ordered */ if (!i915_gem_active_is_idle (&tmp->last_read[pool->engine->id], &tmp->base.dev->struct_mutex))break ; -/* While we're looping, do some clean up */ -if (tmp->madv == __I915_MADV_PURGED) { -list_del (&tmp->batch_pool_link); -i915_gem_object_put (tmp); -continue ; - } -if (tmp->base.size >= size) { obj = tmp;break ;@@ -132,19 +125,16 @@ i915_gem_batch_pool_get(struct i915_gem_batch_pool *pool, }if (obj ==NULL ) { -int ret; - - obj =i915_gem_object_create (&pool->engine->i915->drm, size); + obj =i915_gem_object_create_internal (&pool->engine->i915->drm, + size);if (IS_ERR (obj))return obj; - - ret =i915_gem_object_get_pages (obj); -if (ret) -return ERR_PTR (ret); - - obj->madv =I915_MADV_DONTNEED ; } + ret =i915_gem_object_get_pages (obj); +if (ret) +return ERR_PTR (ret); +list_move_tail (&obj->batch_pool_link, list);i915_gem_object_pin_pages (obj);return obj;drivers/gpu/drm/i915/i915_gem_internal.c new file +52
-0@@ -0,0 +1,161 @@ +/* +* Copyright © 2014-2016 Intel Corporation +* +* Permission is hereby granted, free of charge, to any person obtaining a +* copy of this software and associated documentation files (the "Software"), +* to deal in the Software without restriction, including without limitation +* the rights to use, copy, modify, merge, publish, distribute, sublicense, +* and/or sell copies of the Software, and to permit persons to whom the +* Software is furnished to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice (including the next +* paragraph) shall be included in all copies or substantial portions of the +* Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +* IN THE SOFTWARE. +* +*/ + +#include <drm/drmP.h> +#include <drm/i915_drm.h> +#include "i915_drv.h" + +static void internal_free_pages (struct sg_table *st) +{ +struct sg_page_iter sg_iter; + +for_each_sg_page (st->sgl, &sg_iter, st->nents,0 ) +put_page (sg_page_iter_page (&sg_iter)); + +sg_free_table (st); +kfree (st); +} + +static int i915_gem_object_get_pages_internal (struct drm_i915_gem_object *obj) +{ +const unsigned int npages = obj->base.size /PAGE_SIZE ; +struct sg_table *st; +struct scatterlist *sg; +unsigned long last_pfn =0 ;/* suppress gcc warning */ +gfp_t gfp; +int i; + + st =kmalloc (sizeof (*st),GFP_KERNEL ); +if (!st) +return -ENOMEM ; +
drivers/gpu/drm/i915/i915_gem_internal.c line 62
Quoted 10 lines
drivers/gpu/drm/i915/i915_gem_internal.c new file +10
-0+if (sg_alloc_table (st, npages,GFP_KERNEL )) { +kfree (st); +return -ENOMEM ; + } + + sg = st->sgl; + st->nents =0 ; + + gfp =GFP_KERNEL | __GFP_HIGHMEM | __GFP_RECLAIMABLE; +if (IS_CRESTLINE (obj->base.dev) ||IS_BROADWATER (obj->base.dev)) {
/O\ Nooooo... dev_priv please! :)
Quoted 24 lines
drivers/gpu/drm/i915/i915_gem_internal.c new file +24
-0+/* 965gm cannot relocate objects above 4GiB. */ + gfp &= ~__GFP_HIGHMEM; + gfp |= __GFP_DMA32; + } + +for (i =0 ; i < npages; i++) { +struct page *page; + + page =alloc_page (gfp); +if (!page) + goto err; + +#ifdef CONFIG_SWIOTLB +if (swiotlb_nr_tbl ()) { + st->nents++; +sg_set_page (sg, page,PAGE_SIZE ,0 ); + sg =sg_next (sg); +continue ; + } +#endif +if (!i ||page_to_pfn (page) != last_pfn +1 ) { +if (i) + sg =sg_next (sg); + st->nents++;
drivers/gpu/drm/i915/i915_gem_internal.c line 96
Quoted 10 lines
drivers/gpu/drm/i915/i915_gem_internal.c new file +10
-0+sg_set_page (sg, page,PAGE_SIZE ,0 ); + }else { + sg->length +=PAGE_SIZE ; + } + last_pfn =page_to_pfn (page); + } +#ifdef CONFIG_SWIOTLB + if (!swiotlb_nr_tbl()) +#endif +sg_mark_end (sg);
Looks like the loop above could be moved into a helper and shared with i915_gem_object_get_pages_gtt. Maybe just a page-alloc and page-alloc-error callbacks would be required.
Quoted 39 lines
drivers/gpu/drm/i915/i915_gem_internal.c new file +39
-0+ obj->pages = st; + +if (i915_gem_gtt_prepare_object (obj)) { + obj->pages =NULL ; + goto err; + } + +/* Mark the pages as dontneed whilst they are still pinned. As soon +* as they are unpinned they are allowed to be reaped by the shrinker, +* and the caller is expected to repopulate - the contents of this +* object are only valid whilst active and pinned. +*/ + obj->madv =I915_MADV_DONTNEED ; +return 0 ; + +err: +sg_mark_end (sg); +internal_free_pages (st); +return -ENOMEM ; +} + +static void i915_gem_object_put_pages_internal (struct drm_i915_gem_object *obj) +{ +internal_free_pages (obj->pages); + + obj->dirty =0 ; + obj->madv =I915_MADV_WILLNEED ; +} + +static const struct drm_i915_gem_object_ops i915_gem_object_internal_ops = { + .flags =I915_GEM_OBJECT_HAS_STRUCT_PAGE , + .get_pages = i915_gem_object_get_pages_internal, + .put_pages = i915_gem_object_put_pages_internal, +}; + +/** +* Creates a new object that wraps some internal memory for private use. +* This object is not backed by swappable storage, and as such its contents +* are volatile and only valid whilst pinned. If the object is reaped by the
drivers/gpu/drm/i915/i915_gem_internal.c line 145
Quoted 10 lines
drivers/gpu/drm/i915/i915_gem_internal.c new file +10
-0+* shrinker, its pages and data will be discarded. Equally, it is not a full +* GEM object and so not valid for access from userspace. This makes it useful +* for hardware interfaces like ringbuffers (which are pinned from the time +* the request is written to the time the hardware stops accessing it), but +* not for contexts (which need to be preserved when not active for later +* reuse). Note that it is not cleared upon allocation. +*/ +struct drm_i915_gem_object * +i915_gem_object_create_internal (struct drm_device *dev, +unsigned int size)
size_t again.
drivers/gpu/drm/i915/i915_gem_internal.c line 158
Quoted 13 lines
drivers/gpu/drm/i915/i915_gem_internal.c new file +13
-0+{ +struct drm_i915_gem_object *obj; + + obj =i915_gem_object_alloc (dev); +if (!obj) +return ERR_PTR (-ENOMEM ); + +drm_gem_private_object_init (dev, &obj->base, size); +i915_gem_object_init (obj, &i915_gem_object_internal_ops); + + obj->base.write_domain =I915_GEM_DOMAIN_CPU ; + obj->base.read_domains =I915_GEM_DOMAIN_CPU ; + obj->cache_level =HAS_LLC (dev) ?I915_CACHE_LLC :I915_CACHE_NONE ;
At least to_i915 would be helpful.
Quoted 46 lines
drivers/gpu/drm/i915/i915_gem_internal.c new file +3
-0+ +return obj; +}drivers/gpu/drm/i915/i915_gem_render_state.c +1
-1@@ -187,7 +187,7 @@ int i915_gem_render_state_init(struct drm_i915_gem_request *req)if (so.rodata->batch_items *4 >4096 )return -EINVAL ; - obj =i915_gem_object_create (&req->i915->drm,4096 ); + obj =i915_gem_object_create_internal (&req->i915->drm,4096 );if (IS_ERR (obj))return PTR_ERR (obj);drivers/gpu/drm/i915/intel_engine_cs.c +1
-1@@ -251,7 +251,7 @@ int intel_engine_create_scratch(struct intel_engine_cs *engine, int size) obj =i915_gem_object_create_stolen (&engine->i915->drm, size);if (!obj) - obj =i915_gem_object_create (&engine->i915->drm, size); + obj =i915_gem_object_create_internal (&engine->i915->drm, size);if (IS_ERR (obj)) {DRM_ERROR ("Failed to allocate scratch page\n" );return PTR_ERR (obj);drivers/gpu/drm/i915/intel_ringbuffer.c +2
-1@@ -1783,9 +1783,10 @@ static int init_status_page(struct intel_engine_cs *engine)struct drm_i915_gem_object *obj;struct i915_vma *vma;unsigned int flags; +void *vaddr;int ret; - obj =i915_gem_object_create (&engine->i915->drm,4096 ); + obj =i915_gem_object_create_internal (&engine->i915->drm,4096 );if (IS_ERR (obj)) {DRM_ERROR ("Failed to allocate status page\n" );return PTR_ERR (obj);
drivers/gpu/drm/i915/intel_ringbuffer.c static int init_status_page(struct intel_engine_cs *engine) · line 1827
Quoted 9 lines
drivers/gpu/drm/i915/intel_ringbuffer.c +6
-0@@ -1818,15 +1819,22 @@ static int init_status_page(struct intel_engine_cs *engine)if (ret) goto err; + vaddr =i915_gem_object_pin_map (obj,I915_MAP_WB ); +if (IS_ERR (vaddr)) { + ret =PTR_ERR (vaddr); + goto err_unpin; + } +
Unrelated fix, best if you could afford to extract it.
drivers/gpu/drm/i915/intel_ringbuffer.c static int init_status_page(struct intel_engine_cs *engine) · line 1831
Quoted 6 lines
drivers/gpu/drm/i915/intel_ringbuffer.c +1
-2engine->status_page.vma = vma; engine->status_page.ggtt_offset =i915_ggtt_offset (vma); - engine->status_page.page_addr = -i915_gem_object_pin_map (obj,I915_MAP_WB ); + engine->status_page.page_addr =memset (vaddr,0 ,4096 );
Very unusual, at least for me. :)
drivers/gpu/drm/i915/intel_ringbuffer.c static int init_status_page(struct intel_engine_cs *engine) · line 1840
Quoted 9 lines
drivers/gpu/drm/i915/intel_ringbuffer.c +2
-0DRM_DEBUG_DRIVER ("%s hws offset: 0x%08x\n" , engine->name,i915_ggtt_offset (vma));return 0 ; +err_unpin: +i915_vma_unpin (vma); err:i915_gem_object_put (obj);return ret;
Regards,
Tvrtko
_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
Re: [PATCH 11/42] drm/i915: Introduce an internal allocator for disposable private objects
Quoted 49 lines · Tvrtko Ursulin
Quoted 47 lines · Chris Wilson
Quite a few of our objects used for internal hardware programming do not benefit from being swappable or from being zero initialised. As such they do not benefit from using a shmemfs backing storage and since they are internal and never directly exposed to the user, we do not need to worry about providing a filp. For these we can use an
drm_i915_gem_objectwrapper around asg_tableof plain struct page. They are not swap backed and not automatically pinned. If they are reaped by the shrinker, the pages are released and the contents discarded. For the internal use case, this is fine as for example, ringbuffers are pinned from being written by a request to be read by the hardware. Once they are idle, they can be discarded entirely. As such they are a good match for execlist ringbuffers and a small variety of other internal objects.In the first iteration, this is limited to the scratch batch buffers we use (for command parsing and state initialisation).
Signed-off-by Chris Wilson <[email protected]>--- drivers/gpu/drm/i915/Makefile | 1 + drivers/gpu/drm/i915/i915_drv.h | 5 + drivers/gpu/drm/i915/i915_gem_batch_pool.c | 28 ++--- drivers/gpu/drm/i915/i915_gem_internal.c | 161 +++++++++++++++++++++++++++ drivers/gpu/drm/i915/i915_gem_render_state.c | 2 +- drivers/gpu/drm/i915/intel_engine_cs.c | 2 +- drivers/gpu/drm/i915/intel_ringbuffer.c | 14 ++- 7 files changed, 189 insertions(+), 24 deletions(-) create mode 100644 drivers/gpu/drm/i915/i915_gem_internal.cdrivers/gpu/drm/i915/Makefile +1
-0@@ -35,6 +35,7 @@ i915-y += i915_cmd_parser.o \ i915_gem_execbuffer.o \ i915_gem_fence.o \ i915_gem_gtt.o \ + i915_gem_internal.o \ i915_gem.o \ i915_gem_render_state.o \ i915_gem_request.o \drivers/gpu/drm/i915/i915_drv.h +0
-0@@ -3538,6 +3538,11 @@ i915_gem_object_create_stolen_for_preallocated(struct drm_device *dev, u32 gtt_offset,
Quoted 8 lines
drivers/gpu/drm/i915/i915_drv.h i915_gem_object_create_stolen_for_preallocated(struct drm_device *dev, · line 3545
Quoted 6 lines
drivers/gpu/drm/i915/i915_drv.h +5
-0u32 size); +/* i915_gem_internal.c */ +struct drm_i915_gem_object * +i915_gem_object_create_internal (struct drm_device *dev, +unsigned int size); +Wasn't
size_tour convention for GEM objects?
Not our convention, no. Using size_t has caused too many bugs, that we started to erradicate it (in our usual piecemeal approach).
Quoted 30 lines
Quoted 30 lines
drivers/gpu/drm/i915/i915_drv.h +30
-0+/* 965gm cannot relocate objects above 4GiB. */ + gfp &= ~__GFP_HIGHMEM; + gfp |= __GFP_DMA32; + } + +for (i =0 ; i < npages; i++) { +struct page *page; + + page =alloc_page (gfp); +if (!page) + goto err; + +#ifdef CONFIG_SWIOTLB +if (swiotlb_nr_tbl ()) { + st->nents++; +sg_set_page (sg, page,PAGE_SIZE ,0 ); + sg =sg_next (sg); +continue ; + } +#endif +if (!i ||page_to_pfn (page) != last_pfn +1 ) { +if (i) + sg =sg_next (sg); + st->nents++; +sg_set_page (sg, page,PAGE_SIZE ,0 ); + }else { + sg->length +=PAGE_SIZE ; + } + last_pfn =page_to_pfn (page); + }
Quoted 8 lines
drivers/gpu/drm/i915/i915_drv.h
Quoted 4 lines
drivers/gpu/drm/i915/i915_drv.h +4
-0+#ifdef CONFIG_SWIOTLB + if (!swiotlb_nr_tbl()) +#endif +sg_mark_end (sg);Looks like the loop above could be moved into a helper and shared with
i915_gem_object_get_pages_gtt. Maybe just a page-alloc and page-alloc-error callbacks would be required.
So just the entire thing as a callback... I would have thought you might suggest trying high order allocations and falling back to low order. -Chirs
Signature · 5 lines
Chris Wilson, Intel Open Source Technology Centre _______________________________________________ Intel-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/intel-gfx
Re: [PATCH 11/42] drm/i915: Introduce an internal allocator for disposable private objects
Quoted 56 lines · Chris Wilson
Quoted 47 lines · Tvrtko Ursulin
Quoted 46 lines · Chris Wilson
Quite a few of our objects used for internal hardware programming do not benefit from being swappable or from being zero initialised. As such they do not benefit from using a shmemfs backing storage and since they are internal and never directly exposed to the user, we do not need to worry about providing a filp. For these we can use an
drm_i915_gem_objectwrapper around asg_tableof plain struct page. They are not swap backed and not automatically pinned. If they are reaped by the shrinker, the pages are released and the contents discarded. For the internal use case, this is fine as for example, ringbuffers are pinned from being written by a request to be read by the hardware. Once they are idle, they can be discarded entirely. As such they are a good match for execlist ringbuffers and a small variety of other internal objects.In the first iteration, this is limited to the scratch batch buffers we use (for command parsing and state initialisation).
Signed-off-by Chris Wilson <[email protected]>--- drivers/gpu/drm/i915/Makefile | 1 + drivers/gpu/drm/i915/i915_drv.h | 5 + drivers/gpu/drm/i915/i915_gem_batch_pool.c | 28 ++--- drivers/gpu/drm/i915/i915_gem_internal.c | 161 +++++++++++++++++++++++++++ drivers/gpu/drm/i915/i915_gem_render_state.c | 2 +- drivers/gpu/drm/i915/intel_engine_cs.c | 2 +- drivers/gpu/drm/i915/intel_ringbuffer.c | 14 ++- 7 files changed, 189 insertions(+), 24 deletions(-)create mode 100644
drivers/gpu/drm/i915/i915_gem_internal.cdrivers/gpu/drm/i915/Makefile +1
-0@@ -35,6 +35,7 @@ i915-y += i915_cmd_parser.o \ i915_gem_execbuffer.o \ i915_gem_fence.o \ i915_gem_gtt.o \ + i915_gem_internal.o \ i915_gem.o \ i915_gem_render_state.o \ i915_gem_request.o \drivers/gpu/drm/i915/i915_drv.h +0
-0@@ -3538,6 +3538,11 @@ i915_gem_object_create_stolen_for_preallocated(struct drm_device *dev,Quoted 8 lines
drivers/gpu/drm/i915/i915_drv.h i915_gem_object_create_stolen_for_preallocated(struct drm_device *dev, · line 3545
Quoted 7 lines
drivers/gpu/drm/i915/i915_drv.h +5
-0u32 gtt_offset, u32 size); +/* i915_gem_internal.c */ +struct drm_i915_gem_object * +i915_gem_object_create_internal (struct drm_device *dev, +unsigned int size); +Wasn't
size_tour convention for GEM objects?
Quoted 2 lines
Not our convention, no. Using
size_thas caused too many bugs, that we started to erradicate it (in our usual piecemeal approach).
Oh wow, I missed that decision. Last thing I remember was that someone was trying to convert it all to size_t. Fine by me.
Quoted 37 lines
Quoted 29 lines
Quoted 29 lines
drivers/gpu/drm/i915/i915_drv.h +29
-0+/* 965gm cannot relocate objects above 4GiB. */ + gfp &= ~__GFP_HIGHMEM; + gfp |= __GFP_DMA32; + } + +for (i =0 ; i < npages; i++) { +struct page *page; + + page =alloc_page (gfp); +if (!page) + goto err; + +#ifdef CONFIG_SWIOTLB +if (swiotlb_nr_tbl ()) { + st->nents++; +sg_set_page (sg, page,PAGE_SIZE ,0 ); + sg =sg_next (sg); +continue ; + } +#endif +if (!i ||page_to_pfn (page) != last_pfn +1 ) { +if (i) + sg =sg_next (sg); + st->nents++; +sg_set_page (sg, page,PAGE_SIZE ,0 ); + }else { + sg->length +=PAGE_SIZE ; + } + last_pfn =page_to_pfn (page);Quoted 8 lines
drivers/gpu/drm/i915/i915_drv.h
Quoted 5 lines
drivers/gpu/drm/i915/i915_drv.h +5
-0+ } +#ifdef CONFIG_SWIOTLB + if (!swiotlb_nr_tbl()) +#endif +sg_mark_end (sg);Looks like the loop above could be moved into a helper and shared with
i915_gem_object_get_pages_gtt. Maybe just a page-alloc and page-alloc-error callbacks would be required.
Quoted 2 lines
So just the entire thing as a callback... I would have thought you might suggest trying high order allocations and falling back to low order.
I am not sure higher order attempts would be worth it. What I was thinking was to implement the loop above generically in one place, and then two (or even three with userptr) callers would call that with their own alloc-page and alloc-page-on-error callbacks. So that there is a single implementation of the coalescing logic etc.
Regards,
Tvrtko
_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
Re: [PATCH 11/42] drm/i915: Introduce an internal allocator for disposable private objects
Quoted 58 lines · Tvrtko Ursulin
Quoted 56 lines · Chris Wilson
Quoted 55 lines · Tvrtko Ursulin
Quoted 46 lines · Chris Wilson
Quite a few of our objects used for internal hardware programming do not benefit from being swappable or from being zero initialised. As such they do not benefit from using a shmemfs backing storage and since they are internal and never directly exposed to the user, we do not need to worry about providing a filp. For these we can use an
drm_i915_gem_objectwrapper around asg_tableof plain struct page. They are not swap backed and not automatically pinned. If they are reaped by the shrinker, the pages are released and the contents discarded. For the internal use case, this is fine as for example, ringbuffers are pinned from being written by a request to be read by the hardware. Once they are idle, they can be discarded entirely. As such they are a good match for execlist ringbuffers and a small variety of other internal objects.In the first iteration, this is limited to the scratch batch buffers we use (for command parsing and state initialisation).
Signed-off-by Chris Wilson <[email protected]>--- drivers/gpu/drm/i915/Makefile | 1 + drivers/gpu/drm/i915/i915_drv.h | 5 + drivers/gpu/drm/i915/i915_gem_batch_pool.c | 28 ++--- drivers/gpu/drm/i915/i915_gem_internal.c | 161 +++++++++++++++++++++++++++ drivers/gpu/drm/i915/i915_gem_render_state.c | 2 +- drivers/gpu/drm/i915/intel_engine_cs.c | 2 +- drivers/gpu/drm/i915/intel_ringbuffer.c | 14 ++- 7 files changed, 189 insertions(+), 24 deletions(-) create mode 100644 drivers/gpu/drm/i915/i915_gem_internal.cdrivers/gpu/drm/i915/Makefile +1
-0@@ -35,6 +35,7 @@ i915-y += i915_cmd_parser.o \ i915_gem_execbuffer.o \ i915_gem_fence.o \ i915_gem_gtt.o \ + i915_gem_internal.o \ i915_gem.o \ i915_gem_render_state.o \ i915_gem_request.o \drivers/gpu/drm/i915/i915_drv.h +0
-0drivers/gpu/drm/i915/i915_drv.h i915_gem_object_create_stolen_for_preallocated(struct drm_device *dev, · line 3544
Quoted 7 lines
drivers/gpu/drm/i915/i915_drv.h +5
-0@@ -3538,6 +3538,11 @@ i915_gem_object_create_stolen_for_preallocated(struct drm_device *dev,u32 gtt_offset, u32 size); +/* i915_gem_internal.c */ +struct drm_i915_gem_object * +i915_gem_object_create_internal (struct drm_device *dev, +unsigned int size); +Wasn't
size_tour convention for GEM objects?
Quoted 5 lines
Quoted 2 lines
Not our convention, no. Using
size_thas caused too many bugs, that we started to erradicate it (in our usual piecemeal approach).Oh wow, I missed that decision. Last thing I remember was that someone was trying to convert it all to
size_t. Fine by me.
Who? They probably didn't have to spend the time fixing size_t variables being wrong on 32bit.
Quoted 37 lines
Quoted 37 lines
Quoted 37 lines
Quoted 24 lines
drivers/gpu/drm/i915/i915_drv.h +24
-0+/* 965gm cannot relocate objects above 4GiB. */ + gfp &= ~__GFP_HIGHMEM; + gfp |= __GFP_DMA32; + } + +for (i =0 ; i < npages; i++) { +struct page *page; + + page =alloc_page (gfp); +if (!page) + goto err; + +#ifdef CONFIG_SWIOTLB +if (swiotlb_nr_tbl ()) { + st->nents++; +sg_set_page (sg, page,PAGE_SIZE ,0 ); + sg =sg_next (sg); +continue ; + } +#endif +if (!i ||page_to_pfn (page) != last_pfn +1 ) { +if (i) + sg =sg_next (sg); + st->nents++;drivers/gpu/drm/i915/i915_drv.h i915_gem_object_create_stolen_for_preallocated(struct drm_device *dev,
Quoted 10 lines
drivers/gpu/drm/i915/i915_drv.h +10
-0+sg_set_page (sg, page,PAGE_SIZE ,0 ); + }else { + sg->length +=PAGE_SIZE ; + } + last_pfn =page_to_pfn (page); + } +#ifdef CONFIG_SWIOTLB + if (!swiotlb_nr_tbl()) +#endif +sg_mark_end (sg);Looks like the loop above could be moved into a helper and shared with
i915_gem_object_get_pages_gtt. Maybe just a page-alloc and page-alloc-error callbacks would be required.
Quoted 4 lines
Quoted 2 lines
So just the entire thing as a callback... I would have thought you might suggest trying high order allocations and falling back to low order.
I am not sure higher order attempts would be worth it.
Looks like it is. :)
-Chris
Signature · 5 lines
Chris Wilson, Intel Open Source Technology Centre _______________________________________________ Intel-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH v2 1/1 drm/i915: Introduce an internal allocator for disposable private objects
In reply to Tvrtko UrsulinQuite a few of our objects used for internal hardware programming do not benefit from being swappable or from being zero initialised. As such they do not benefit from using a shmemfs backing storage and since they are internal and never directly exposed to the user, we do not need to worry about providing a filp. For these we can use an drm_i915_gem_object wrapper around a sg_table of plain struct page. They are not swap backed and not automatically pinned. If they are reaped by the shrinker, the pages are released and the contents discarded. For the internal use case, this is fine as for example, ringbuffers are pinned from being written by a request to be read by the hardware. Once they are idle, they can be discarded entirely. As such they are a good match for execlist ringbuffers and a small variety of other internal objects.
In the first iteration, this is limited to the scratch batch buffers we use (for command parsing and state initialisation).
v2: Allocate physically contiguous pages, where possible.
--- drivers/gpu/drm/i915/Makefile | 1 + drivers/gpu/drm/i915/i915_drv.h | 5 + drivers/gpu/drm/i915/i915_gem_batch_pool.c | 28 ++--- drivers/gpu/drm/i915/i915_gem_internal.c | 162 +++++++++++++++++++++++++++ drivers/gpu/drm/i915/i915_gem_render_state.c | 2 +- drivers/gpu/drm/i915/intel_engine_cs.c | 2 +- drivers/gpu/drm/i915/intel_ringbuffer.c | 14 ++- 7 files changed, 190 insertions(+), 24 deletions(-) create mode 100644 drivers/gpu/drm/i915/i915_gem_internal.c
drivers/gpu/drm/i915/Makefile +1 -0
@@ -35,6 +35,7 @@ i915-y += i915_cmd_parser.o \ i915_gem_execbuffer.o \ i915_gem_fence.o \ i915_gem_gtt.o \ + i915_gem_internal.o \ i915_gem.o \ i915_gem_render_state.o \ i915_gem_request.o \
drivers/gpu/drm/i915/i915_drv.h +5 -0
@@ -3538,6 +3538,11 @@ i915_gem_object_create_stolen_for_preallocated(struct drm_device *dev,u32 gtt_offset, u32 size); +/* i915_gem_internal.c */ +struct drm_i915_gem_object * +i915_gem_object_create_internal (struct drm_device *dev, +unsigned int size); +/* i915_gem_shrinker.c */ unsigned long i915_gem_shrink(struct drm_i915_private *dev_priv, unsigned long target,
drivers/gpu/drm/i915/i915_gem_batch_pool.c +9 -19
@@ -97,9 +97,9 @@ i915_gem_batch_pool_get(struct i915_gem_batch_pool *pool, size_t size) {struct drm_i915_gem_object *obj =NULL ; -struct drm_i915_gem_object *tmp, *next; +struct drm_i915_gem_object *tmp;struct list_head *list; -int n; +int n, ret;lockdep_assert_held (&pool->engine->i915->drm.struct_mutex);
@@ -112,19 +112,12 @@ i915_gem_batch_pool_get(struct i915_gem_batch_pool *pool, n =ARRAY_SIZE (pool->cache_list) -1 ; list = &pool->cache_list[n]; -list_for_each_entry_safe (tmp, next, list, batch_pool_link) { +list_for_each_entry (tmp, list, batch_pool_link) {/* The batches are strictly LRU ordered */ if (!i915_gem_active_is_idle (&tmp->last_read[pool->engine->id], &tmp->base.dev->struct_mutex))break ; -/* While we're looping, do some clean up */ -if (tmp->madv == __I915_MADV_PURGED) { -list_del (&tmp->batch_pool_link); -i915_gem_object_put (tmp); -continue ; - } -if (tmp->base.size >= size) { obj = tmp;break ;
@@ -132,19 +125,16 @@ i915_gem_batch_pool_get(struct i915_gem_batch_pool *pool, }if (obj ==NULL ) { -int ret; - - obj =i915_gem_object_create (&pool->engine->i915->drm, size); + obj =i915_gem_object_create_internal (&pool->engine->i915->drm, + size);if (IS_ERR (obj))return obj; - - ret =i915_gem_object_get_pages (obj); -if (ret) -return ERR_PTR (ret); - - obj->madv =I915_MADV_DONTNEED ; } + ret =i915_gem_object_get_pages (obj); +if (ret) +return ERR_PTR (ret); +list_move_tail (&obj->batch_pool_link, list);i915_gem_object_pin_pages (obj);return obj;
drivers/gpu/drm/i915/i915_gem_internal.c new file +162 -0
162 lines not on this page · raw message
drivers/gpu/drm/i915/i915_gem_render_state.c +1 -1
@@ -187,7 +187,7 @@ int i915_gem_render_state_init(struct drm_i915_gem_request *req)if (so.rodata->batch_items *4 >4096 )return -EINVAL ; - obj =i915_gem_object_create (&req->i915->drm,4096 ); + obj =i915_gem_object_create_internal (&req->i915->drm,4096 );if (IS_ERR (obj))return PTR_ERR (obj);
drivers/gpu/drm/i915/intel_engine_cs.c +1 -1
@@ -251,7 +251,7 @@ int intel_engine_create_scratch(struct intel_engine_cs *engine, int size) obj =i915_gem_object_create_stolen (&engine->i915->drm, size);if (!obj) - obj =i915_gem_object_create (&engine->i915->drm, size); + obj =i915_gem_object_create_internal (&engine->i915->drm, size);if (IS_ERR (obj)) {DRM_ERROR ("Failed to allocate scratch page\n" );return PTR_ERR (obj);
drivers/gpu/drm/i915/intel_ringbuffer.c +11 -3
@@ -1783,9 +1783,10 @@ static int init_status_page(struct intel_engine_cs *engine)struct drm_i915_gem_object *obj;struct i915_vma *vma;unsigned int flags; +void *vaddr;int ret; - obj =i915_gem_object_create (&engine->i915->drm,4096 ); + obj =i915_gem_object_create_internal (&engine->i915->drm,4096 );if (IS_ERR (obj)) {DRM_ERROR ("Failed to allocate status page\n" );return PTR_ERR (obj);
@@ -1818,15 +1819,22 @@ static int init_status_page(struct intel_engine_cs *engine)if (ret) goto err; + vaddr =i915_gem_object_pin_map (obj,I915_MAP_WB ); +if (IS_ERR (vaddr)) { + ret =PTR_ERR (vaddr); + goto err_unpin; + } + engine->status_page.vma = vma; engine->status_page.ggtt_offset =i915_ggtt_offset (vma); - engine->status_page.page_addr = -i915_gem_object_pin_map (obj,I915_MAP_WB ); + engine->status_page.page_addr =memset (vaddr,0 ,4096 );DRM_DEBUG_DRIVER ("%s hws offset: 0x%08x\n" , engine->name,i915_ggtt_offset (vma));return 0 ; +err_unpin: +i915_vma_unpin (vma); err:i915_gem_object_put (obj);return ret;
Signature · 6 lines
2.9.3 _______________________________________________ Intel-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/intel-gfx
Re: [PATCH v2] drm/i915: Introduce an internal allocator for disposable private objects
Quoted 14 lines · Chris Wilson
Quite a few of our objects used for internal hardware programming do not benefit from being swappable or from being zero initialised. As such they do not benefit from using a shmemfs backing storage and since they are internal and never directly exposed to the user, we do not need to worry about providing a filp. For these we can use an
drm_i915_gem_objectwrapper around asg_tableof plain struct page. They are not swap backed and not automatically pinned. If they are reaped by the shrinker, the pages are released and the contents discarded. For the internal use case, this is fine as for example, ringbuffers are pinned from being written by a request to be read by the hardware. Once they are idle, they can be discarded entirely. As such they are a good match for execlist ringbuffers and a small variety of other internal objects.
Quoted 4 lines
In the first iteration, this is limited to the scratch batch buffers we use (for command parsing and state initialisation).
v2: Allocate physically contiguous pages, where possible.
Does not hurt, but what does it gain us, exactly?
Quoted 3 lines
Signed-off-by Chris Wilson <[email protected]>Cc Tvrtko Ursulin <[email protected]>
Regards, Joonas
Signature · 7 lines
Joonas Lahtinen Open Source Technology Center Intel Corporation _______________________________________________ Intel-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/intel-gfx
Re: [PATCH v2] drm/i915: Introduce an internal allocator for disposable private objects
In reply to Chris WilsonQuoted 14 lines · Chris Wilson
Quite a few of our objects used for internal hardware programming do not benefit from being swappable or from being zero initialised. As such they do not benefit from using a shmemfs backing storage and since they are internal and never directly exposed to the user, we do not need to worry about providing a filp. For these we can use an
drm_i915_gem_objectwrapper around asg_tableof plain struct page. They are not swap backed and not automatically pinned. If they are reaped by the shrinker, the pages are released and the contents discarded. For the internal use case, this is fine as for example, ringbuffers are pinned from being written by a request to be read by the hardware. Once they are idle, they can be discarded entirely. As such they are a good match for execlist ringbuffers and a small variety of other internal objects.
Quoted 4 lines
In the first iteration, this is limited to the scratch batch buffers we use (for command parsing and state initialisation).
v2: Allocate physically contiguous pages, where possible.
Since the allocator will be used constantly at runtime, my recollection is that higher order allocations can easily become next to impossible, so I am wondering why? Also, on your last reply you did not write why you think this is interesting to try?
Quoted 136 lines
Signed-off-by Chris Wilson <[email protected]>Cc Tvrtko Ursulin <[email protected]>--- drivers/gpu/drm/i915/Makefile | 1 + drivers/gpu/drm/i915/i915_drv.h | 5 + drivers/gpu/drm/i915/i915_gem_batch_pool.c | 28 ++--- drivers/gpu/drm/i915/i915_gem_internal.c | 162 +++++++++++++++++++++++++++ drivers/gpu/drm/i915/i915_gem_render_state.c | 2 +- drivers/gpu/drm/i915/intel_engine_cs.c | 2 +- drivers/gpu/drm/i915/intel_ringbuffer.c | 14 ++- 7 files changed, 190 insertions(+), 24 deletions(-)create mode 100644
drivers/gpu/drm/i915/i915_gem_internal.cdrivers/gpu/drm/i915/Makefile +1
-0@@ -35,6 +35,7 @@ i915-y += i915_cmd_parser.o \ i915_gem_execbuffer.o \ i915_gem_fence.o \ i915_gem_gtt.o \ + i915_gem_internal.o \ i915_gem.o \ i915_gem_render_state.o \ i915_gem_request.o \drivers/gpu/drm/i915/i915_drv.h +5
-0@@ -3538,6 +3538,11 @@ i915_gem_object_create_stolen_for_preallocated(struct drm_device *dev,u32 gtt_offset, u32 size); +/* i915_gem_internal.c */ +struct drm_i915_gem_object * +i915_gem_object_create_internal (struct drm_device *dev, +unsigned int size); +/* i915_gem_shrinker.c */ unsigned long i915_gem_shrink(struct drm_i915_private *dev_priv, unsigned long target,drivers/gpu/drm/i915/i915_gem_batch_pool.c +9
-19@@ -97,9 +97,9 @@ i915_gem_batch_pool_get(struct i915_gem_batch_pool *pool, size_t size) {struct drm_i915_gem_object *obj =NULL ; -struct drm_i915_gem_object *tmp, *next; +struct drm_i915_gem_object *tmp;struct list_head *list; -int n; +int n, ret;lockdep_assert_held (&pool->engine->i915->drm.struct_mutex);@@ -112,19 +112,12 @@ i915_gem_batch_pool_get(struct i915_gem_batch_pool *pool, n =ARRAY_SIZE (pool->cache_list) -1 ; list = &pool->cache_list[n]; -list_for_each_entry_safe (tmp, next, list, batch_pool_link) { +list_for_each_entry (tmp, list, batch_pool_link) {/* The batches are strictly LRU ordered */ if (!i915_gem_active_is_idle (&tmp->last_read[pool->engine->id], &tmp->base.dev->struct_mutex))break ; -/* While we're looping, do some clean up */ -if (tmp->madv == __I915_MADV_PURGED) { -list_del (&tmp->batch_pool_link); -i915_gem_object_put (tmp); -continue ; - } -if (tmp->base.size >= size) { obj = tmp;break ;@@ -132,19 +125,16 @@ i915_gem_batch_pool_get(struct i915_gem_batch_pool *pool, }if (obj ==NULL ) { -int ret; - - obj =i915_gem_object_create (&pool->engine->i915->drm, size); + obj =i915_gem_object_create_internal (&pool->engine->i915->drm, + size);if (IS_ERR (obj))return obj; - - ret =i915_gem_object_get_pages (obj); -if (ret) -return ERR_PTR (ret); - - obj->madv =I915_MADV_DONTNEED ; } + ret =i915_gem_object_get_pages (obj); +if (ret) +return ERR_PTR (ret); +list_move_tail (&obj->batch_pool_link, list);i915_gem_object_pin_pages (obj);return obj;drivers/gpu/drm/i915/i915_gem_internal.c new file +26
-0@@ -0,0 +1,162 @@ +/* +* Copyright © 2014-2016 Intel Corporation +* +* Permission is hereby granted, free of charge, to any person obtaining a +* copy of this software and associated documentation files (the "Software"), +* to deal in the Software without restriction, including without limitation +* the rights to use, copy, modify, merge, publish, distribute, sublicense, +* and/or sell copies of the Software, and to permit persons to whom the +* Software is furnished to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice (including the next +* paragraph) shall be included in all copies or substantial portions of the +* Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +* IN THE SOFTWARE. +* +*/ + +#include <drm/drmP.h> +#include <drm/i915_drm.h>
drivers/gpu/drm/i915/i915_gem_internal.c line 36
Quoted 10 lines
drivers/gpu/drm/i915/i915_gem_internal.c new file +10
-0+#include "i915_drv.h" + +#define QUIET (__GFP_NORETRY | __GFP_NOWARN) + +static void internal_free_pages (struct sg_table *st) +{ +struct scatterlist *sg; + +for (sg = st->sgl; sg; sg =__sg_next (sg)) +__free_pages (sg_page (sg),get_order (sg->length));
Fragile since wont work together with coalescing, which I am sad to see not implemented. For some reason it makes me feel real good when it is there.
Quoted 23 lines
drivers/gpu/drm/i915/i915_gem_internal.c new file +23
-0+ +sg_free_table (st); +kfree (st); +} + +static int i915_gem_object_get_pages_internal (struct drm_i915_gem_object *obj) +{ +unsigned int npages = obj->base.size /PAGE_SIZE ; +struct sg_table *st; +struct scatterlist *sg; +int max_order; +gfp_t gfp; + + st =kmalloc (sizeof (*st),GFP_KERNEL ); +if (!st) +return -ENOMEM ; + +if (sg_alloc_table (st, npages,GFP_KERNEL )) { +kfree (st); +return -ENOMEM ; + } + + sg = st->sgl;
drivers/gpu/drm/i915/i915_gem_internal.c line 69
Quoted 10 lines
drivers/gpu/drm/i915/i915_gem_internal.c new file +10
-0+ st->nents =0 ; + + max_order =MAX_ORDER ; +#ifdef CONFIG_SWIOTLB +if (swiotlb_nr_tbl ()) + max_order =min (max_order,ilog2 (IO_TLB_SEGSIZE )); +#endif + + gfp =GFP_KERNEL | __GFP_HIGHMEM | __GFP_RECLAIMABLE; +if (IS_CRESTLINE (obj->base.dev) ||IS_BROADWATER (obj->base.dev)) {
You don't want to avoid more usages of dev?
Quoted 6 lines
drivers/gpu/drm/i915/i915_gem_internal.c new file +6
-0+/* 965gm cannot relocate objects above 4GiB. */ + gfp &= ~__GFP_HIGHMEM; + gfp |= __GFP_DMA32; + } + + do {
drivers/gpu/drm/i915/i915_gem_internal.c line 85
Quoted 10 lines
drivers/gpu/drm/i915/i915_gem_internal.c new file +10
-0+int order =min (fls (npages) -1 , max_order); +struct page *page; + +do { + page =alloc_pages (gfp | (order ?QUIET :0 ), order); +if (page) +break ; +if (!order--) + goto err; + }while (1 );
Feels like it could hammer hard on a fragmented system, I have big concerns about this.
Quoted 120 lines
drivers/gpu/drm/i915/i915_gem_internal.c new file +77
-0+ +sg_set_page (sg, page,PAGE_SIZE << order,0 ); + st->nents++; + + npages -=1 << order; +if (!npages) { +sg_mark_end (sg); +break ; + } + + sg =__sg_next (sg); + }while (1 ); + obj->pages = st; + +if (i915_gem_gtt_prepare_object (obj)) { + obj->pages =NULL ; + goto err; + } + +/* Mark the pages as dontneed whilst they are still pinned. As soon +* as they are unpinned they are allowed to be reaped by the shrinker, +* and the caller is expected to repopulate - the contents of this +* object are only valid whilst active and pinned. +*/ + obj->madv =I915_MADV_DONTNEED ; +return 0 ; + +err: +sg_mark_end (sg); +internal_free_pages (st); +return -ENOMEM ; +} + +static void i915_gem_object_put_pages_internal (struct drm_i915_gem_object *obj) +{ +internal_free_pages (obj->pages); + + obj->dirty =0 ; + obj->madv =I915_MADV_WILLNEED ; +} + +static const struct drm_i915_gem_object_ops i915_gem_object_internal_ops = { + .flags =I915_GEM_OBJECT_HAS_STRUCT_PAGE , + .get_pages = i915_gem_object_get_pages_internal, + .put_pages = i915_gem_object_put_pages_internal, +}; + +/** +* Creates a new object that wraps some internal memory for private use. +* This object is not backed by swappable storage, and as such its contents +* are volatile and only valid whilst pinned. If the object is reaped by the +* shrinker, its pages and data will be discarded. Equally, it is not a full +* GEM object and so not valid for access from userspace. This makes it useful +* for hardware interfaces like ringbuffers (which are pinned from the time +* the request is written to the time the hardware stops accessing it), but +* not for contexts (which need to be preserved when not active for later +* reuse). Note that it is not cleared upon allocation. +*/ +struct drm_i915_gem_object * +i915_gem_object_create_internal (struct drm_device *dev, +unsigned int size) +{ +struct drm_i915_gem_object *obj; + + obj =i915_gem_object_alloc (dev); +if (!obj) +return ERR_PTR (-ENOMEM ); + +drm_gem_private_object_init (dev, &obj->base, size); +i915_gem_object_init (obj, &i915_gem_object_internal_ops); + + obj->base.write_domain =I915_GEM_DOMAIN_CPU ; + obj->base.read_domains =I915_GEM_DOMAIN_CPU ; + obj->cache_level =HAS_LLC (dev) ?I915_CACHE_LLC :I915_CACHE_NONE ; + +return obj; +}drivers/gpu/drm/i915/i915_gem_render_state.c +1
-1@@ -187,7 +187,7 @@ int i915_gem_render_state_init(struct drm_i915_gem_request *req)if (so.rodata->batch_items *4 >4096 )return -EINVAL ; - obj =i915_gem_object_create (&req->i915->drm,4096 ); + obj =i915_gem_object_create_internal (&req->i915->drm,4096 );if (IS_ERR (obj))return PTR_ERR (obj);drivers/gpu/drm/i915/intel_engine_cs.c +1
-1@@ -251,7 +251,7 @@ int intel_engine_create_scratch(struct intel_engine_cs *engine, int size) obj =i915_gem_object_create_stolen (&engine->i915->drm, size);if (!obj) - obj =i915_gem_object_create (&engine->i915->drm, size); + obj =i915_gem_object_create_internal (&engine->i915->drm, size);if (IS_ERR (obj)) {DRM_ERROR ("Failed to allocate scratch page\n" );return PTR_ERR (obj);drivers/gpu/drm/i915/intel_ringbuffer.c +2
-1@@ -1783,9 +1783,10 @@ static int init_status_page(struct intel_engine_cs *engine)struct drm_i915_gem_object *obj;struct i915_vma *vma;unsigned int flags; +void *vaddr;int ret; - obj =i915_gem_object_create (&engine->i915->drm,4096 ); + obj =i915_gem_object_create_internal (&engine->i915->drm,4096 );if (IS_ERR (obj)) {DRM_ERROR ("Failed to allocate status page\n" );return PTR_ERR (obj);
drivers/gpu/drm/i915/intel_ringbuffer.c static int init_status_page(struct intel_engine_cs *engine) · line 1826
Quoted 8 lines
drivers/gpu/drm/i915/intel_ringbuffer.c +5
-0@@ -1818,15 +1819,22 @@ static int init_status_page(struct intel_engine_cs *engine)if (ret) goto err; + vaddr =i915_gem_object_pin_map (obj,I915_MAP_WB ); +if (IS_ERR (vaddr)) { + ret =PTR_ERR (vaddr); + goto err_unpin; + }
And you don't want to split the bugfix out?
Quoted 6 lines
drivers/gpu/drm/i915/intel_ringbuffer.c +2
-2+ engine->status_page.vma = vma; engine->status_page.ggtt_offset =i915_ggtt_offset (vma); - engine->status_page.page_addr = -i915_gem_object_pin_map (obj,I915_MAP_WB ); + engine->status_page.page_addr =memset (vaddr,0 ,4096 );
drivers/gpu/drm/i915/intel_ringbuffer.c static int init_status_page(struct intel_engine_cs *engine) · line 1840
Quoted 10 lines
drivers/gpu/drm/i915/intel_ringbuffer.c +2
-0DRM_DEBUG_DRIVER ("%s hws offset: 0x%08x\n" , engine->name,i915_ggtt_offset (vma));return 0 ; +err_unpin: +i915_vma_unpin (vma); err:i915_gem_object_put (obj);return ret;
Regards,
Tvrtko
_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
Re: [PATCH v2] drm/i915: Introduce an internal allocator for disposable private objects
Quoted 160 lines · Tvrtko Ursulin
Quoted 14 lines · Chris Wilson
Quite a few of our objects used for internal hardware programming do not benefit from being swappable or from being zero initialised. As such they do not benefit from using a shmemfs backing storage and since they are internal and never directly exposed to the user, we do not need to worry about providing a filp. For these we can use an
drm_i915_gem_objectwrapper around asg_tableof plain struct page. They are not swap backed and not automatically pinned. If they are reaped by the shrinker, the pages are released and the contents discarded. For the internal use case, this is fine as for example, ringbuffers are pinned from being written by a request to be read by the hardware. Once they are idle, they can be discarded entirely. As such they are a good match for execlist ringbuffers and a small variety of other internal objects.Quoted 4 lines
In the first iteration, this is limited to the scratch batch buffers we use (for command parsing and state initialisation).
v2: Allocate physically contiguous pages, where possible.
Since the allocator will be used constantly at runtime, my recollection is that higher order allocations can easily become next to impossible, so I am wondering why? Also, on your last reply you did not write why you think this is interesting to try?
Quoted 135 lines
Signed-off-by Chris Wilson <[email protected]>Cc Tvrtko Ursulin <[email protected]>--- drivers/gpu/drm/i915/Makefile | 1 + drivers/gpu/drm/i915/i915_drv.h | 5 + drivers/gpu/drm/i915/i915_gem_batch_pool.c | 28 ++--- drivers/gpu/drm/i915/i915_gem_internal.c | 162 +++++++++++++++++++++++++++ drivers/gpu/drm/i915/i915_gem_render_state.c | 2 +- drivers/gpu/drm/i915/intel_engine_cs.c | 2 +- drivers/gpu/drm/i915/intel_ringbuffer.c | 14 ++- 7 files changed, 190 insertions(+), 24 deletions(-) create mode 100644 drivers/gpu/drm/i915/i915_gem_internal.cdrivers/gpu/drm/i915/Makefile +1
-0@@ -35,6 +35,7 @@ i915-y += i915_cmd_parser.o \ i915_gem_execbuffer.o \ i915_gem_fence.o \ i915_gem_gtt.o \ + i915_gem_internal.o \ i915_gem.o \ i915_gem_render_state.o \ i915_gem_request.o \drivers/gpu/drm/i915/i915_drv.h +5
-0@@ -3538,6 +3538,11 @@ i915_gem_object_create_stolen_for_preallocated(struct drm_device *dev,u32 gtt_offset, u32 size); +/* i915_gem_internal.c */ +struct drm_i915_gem_object * +i915_gem_object_create_internal (struct drm_device *dev, +unsigned int size); +/* i915_gem_shrinker.c */ unsigned long i915_gem_shrink(struct drm_i915_private *dev_priv, unsigned long target,drivers/gpu/drm/i915/i915_gem_batch_pool.c +9
-19@@ -97,9 +97,9 @@ i915_gem_batch_pool_get(struct i915_gem_batch_pool *pool, size_t size) {struct drm_i915_gem_object *obj =NULL ; -struct drm_i915_gem_object *tmp, *next; +struct drm_i915_gem_object *tmp;struct list_head *list; -int n; +int n, ret;lockdep_assert_held (&pool->engine->i915->drm.struct_mutex);@@ -112,19 +112,12 @@ i915_gem_batch_pool_get(struct i915_gem_batch_pool *pool, n =ARRAY_SIZE (pool->cache_list) -1 ; list = &pool->cache_list[n]; -list_for_each_entry_safe (tmp, next, list, batch_pool_link) { +list_for_each_entry (tmp, list, batch_pool_link) {/* The batches are strictly LRU ordered */ if (!i915_gem_active_is_idle (&tmp->last_read[pool->engine->id], &tmp->base.dev->struct_mutex))break ; -/* While we're looping, do some clean up */ -if (tmp->madv == __I915_MADV_PURGED) { -list_del (&tmp->batch_pool_link); -i915_gem_object_put (tmp); -continue ; - } -if (tmp->base.size >= size) { obj = tmp;break ;@@ -132,19 +125,16 @@ i915_gem_batch_pool_get(struct i915_gem_batch_pool *pool, }if (obj ==NULL ) { -int ret; - - obj =i915_gem_object_create (&pool->engine->i915->drm, size); + obj =i915_gem_object_create_internal (&pool->engine->i915->drm, + size);if (IS_ERR (obj))return obj; - - ret =i915_gem_object_get_pages (obj); -if (ret) -return ERR_PTR (ret); - - obj->madv =I915_MADV_DONTNEED ; } + ret =i915_gem_object_get_pages (obj); +if (ret) +return ERR_PTR (ret); +list_move_tail (&obj->batch_pool_link, list);i915_gem_object_pin_pages (obj);return obj;drivers/gpu/drm/i915/i915_gem_internal.c new file +32
-0@@ -0,0 +1,162 @@ +/* +* Copyright © 2014-2016 Intel Corporation +* +* Permission is hereby granted, free of charge, to any person obtaining a +* copy of this software and associated documentation files (the "Software"), +* to deal in the Software without restriction, including without limitation +* the rights to use, copy, modify, merge, publish, distribute, sublicense, +* and/or sell copies of the Software, and to permit persons to whom the +* Software is furnished to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice (including the next +* paragraph) shall be included in all copies or substantial portions of the +* Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +* IN THE SOFTWARE. +* +*/ + +#include <drm/drmP.h> +#include <drm/i915_drm.h> +#include "i915_drv.h" + +#define QUIET (__GFP_NORETRY | __GFP_NOWARN) + +static void internal_free_pages (struct sg_table *st) +{
Quoted 8 lines
drivers/gpu/drm/i915/i915_gem_internal.c line 36
Quoted 4 lines
drivers/gpu/drm/i915/i915_gem_internal.c new file +4
-0+struct scatterlist *sg; + +for (sg = st->sgl; sg; sg =__sg_next (sg)) +__free_pages (sg_page (sg),get_order (sg->length));Fragile since wont work together with coalescing, which I am sad to see not implemented. For some reason it makes me feel real good when it is there.
We do the coalescing.
Quoted 10 lines
drivers/gpu/drm/i915/i915_gem_internal.c line 85
Quoted 7 lines
drivers/gpu/drm/i915/i915_gem_internal.c new file +7
-0+do { + page =alloc_pages (gfp | (order ?QUIET :0 ), order); +if (page) +break ; +if (!order--) + goto err; + }while (1 );Feels like it could hammer hard on a fragmented system, I have big concerns about this.
You did notice that these were marked as temporary allocations (because they are reclaimed under memory pressure), and on an already fragemented system we do the right thing anyway?
-Chris
Signature · 5 lines
Chris Wilson, Intel Open Source Technology Centre _______________________________________________ Intel-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/intel-gfx
Re: [PATCH v2] drm/i915: Introduce an internal allocator for disposable private objects
Quoted 167 lines · Chris Wilson
Quoted 158 lines · Tvrtko Ursulin
Quoted 14 lines · Chris Wilson
Quite a few of our objects used for internal hardware programming do not benefit from being swappable or from being zero initialised. As such they do not benefit from using a shmemfs backing storage and since they are internal and never directly exposed to the user, we do not need to worry about providing a filp. For these we can use an
drm_i915_gem_objectwrapper around asg_tableof plain struct page. They are not swap backed and not automatically pinned. If they are reaped by the shrinker, the pages are released and the contents discarded. For the internal use case, this is fine as for example, ringbuffers are pinned from being written by a request to be read by the hardware. Once they are idle, they can be discarded entirely. As such they are a good match for execlist ringbuffers and a small variety of other internal objects.Quoted 4 lines
In the first iteration, this is limited to the scratch batch buffers we use (for command parsing and state initialisation).
v2: Allocate physically contiguous pages, where possible.
Since the allocator will be used constantly at runtime, my recollection is that higher order allocations can easily become next to impossible, so I am wondering why? Also, on your last reply you did not write why you think this is interesting to try?
Quoted 134 lines
Signed-off-by Chris Wilson <[email protected]>Cc Tvrtko Ursulin <[email protected]>--- drivers/gpu/drm/i915/Makefile | 1 + drivers/gpu/drm/i915/i915_drv.h | 5 + drivers/gpu/drm/i915/i915_gem_batch_pool.c | 28 ++--- drivers/gpu/drm/i915/i915_gem_internal.c | 162 +++++++++++++++++++++++++++ drivers/gpu/drm/i915/i915_gem_render_state.c | 2 +- drivers/gpu/drm/i915/intel_engine_cs.c | 2 +- drivers/gpu/drm/i915/intel_ringbuffer.c | 14 ++- 7 files changed, 190 insertions(+), 24 deletions(-)create mode 100644
drivers/gpu/drm/i915/i915_gem_internal.cdrivers/gpu/drm/i915/Makefile +1
-0@@ -35,6 +35,7 @@ i915-y += i915_cmd_parser.o \ i915_gem_execbuffer.o \ i915_gem_fence.o \ i915_gem_gtt.o \ + i915_gem_internal.o \ i915_gem.o \ i915_gem_render_state.o \ i915_gem_request.o \drivers/gpu/drm/i915/i915_drv.h +5
-0@@ -3538,6 +3538,11 @@ i915_gem_object_create_stolen_for_preallocated(struct drm_device *dev,u32 gtt_offset, u32 size); +/* i915_gem_internal.c */ +struct drm_i915_gem_object * +i915_gem_object_create_internal (struct drm_device *dev, +unsigned int size); +/* i915_gem_shrinker.c */ unsigned long i915_gem_shrink(struct drm_i915_private *dev_priv, unsigned long target,drivers/gpu/drm/i915/i915_gem_batch_pool.c +9
-19@@ -97,9 +97,9 @@ i915_gem_batch_pool_get(struct i915_gem_batch_pool *pool, size_t size) {struct drm_i915_gem_object *obj =NULL ; -struct drm_i915_gem_object *tmp, *next; +struct drm_i915_gem_object *tmp;struct list_head *list; -int n; +int n, ret;lockdep_assert_held (&pool->engine->i915->drm.struct_mutex);@@ -112,19 +112,12 @@ i915_gem_batch_pool_get(struct i915_gem_batch_pool *pool, n =ARRAY_SIZE (pool->cache_list) -1 ; list = &pool->cache_list[n]; -list_for_each_entry_safe (tmp, next, list, batch_pool_link) { +list_for_each_entry (tmp, list, batch_pool_link) {/* The batches are strictly LRU ordered */ if (!i915_gem_active_is_idle (&tmp->last_read[pool->engine->id], &tmp->base.dev->struct_mutex))break ; -/* While we're looping, do some clean up */ -if (tmp->madv == __I915_MADV_PURGED) { -list_del (&tmp->batch_pool_link); -i915_gem_object_put (tmp); -continue ; - } -if (tmp->base.size >= size) { obj = tmp;break ;@@ -132,19 +125,16 @@ i915_gem_batch_pool_get(struct i915_gem_batch_pool *pool, }if (obj ==NULL ) { -int ret; - - obj =i915_gem_object_create (&pool->engine->i915->drm, size); + obj =i915_gem_object_create_internal (&pool->engine->i915->drm, + size);if (IS_ERR (obj))return obj; - - ret =i915_gem_object_get_pages (obj); -if (ret) -return ERR_PTR (ret); - - obj->madv =I915_MADV_DONTNEED ; } + ret =i915_gem_object_get_pages (obj); +if (ret) +return ERR_PTR (ret); +list_move_tail (&obj->batch_pool_link, list);i915_gem_object_pin_pages (obj);return obj;drivers/gpu/drm/i915/i915_gem_internal.c new file +31
-0@@ -0,0 +1,162 @@ +/* +* Copyright © 2014-2016 Intel Corporation +* +* Permission is hereby granted, free of charge, to any person obtaining a +* copy of this software and associated documentation files (the "Software"), +* to deal in the Software without restriction, including without limitation +* the rights to use, copy, modify, merge, publish, distribute, sublicense, +* and/or sell copies of the Software, and to permit persons to whom the +* Software is furnished to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice (including the next +* paragraph) shall be included in all copies or substantial portions of the +* Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +* IN THE SOFTWARE. +* +*/ + +#include <drm/drmP.h> +#include <drm/i915_drm.h> +#include "i915_drv.h" + +#define QUIET (__GFP_NORETRY | __GFP_NOWARN) + +static void internal_free_pages (struct sg_table *st)Quoted 8 lines
drivers/gpu/drm/i915/i915_gem_internal.c line 36
Quoted 5 lines
drivers/gpu/drm/i915/i915_gem_internal.c new file +5
-0+{ +struct scatterlist *sg; + +for (sg = st->sgl; sg; sg =__sg_next (sg)) +__free_pages (sg_page (sg),get_order (sg->length));Fragile since wont work together with coalescing, which I am sad to see not implemented. For some reason it makes me feel real good when it is there.
Quoted 1 line
We do the coalescing.
Where? It is not in this patch? One allocation == one st->nents++. If the next allocation is contiguous there is no coalescing.
Quoted 9 lines
Quoted 9 lines
drivers/gpu/drm/i915/i915_gem_internal.c line 85
Quoted 7 lines
drivers/gpu/drm/i915/i915_gem_internal.c new file +7
-0+do { + page =alloc_pages (gfp | (order ?QUIET :0 ), order); +if (page) +break ; +if (!order--) + goto err; + }while (1 );Feels like it could hammer hard on a fragmented system, I have big concerns about this.
Quoted 3 lines
You did notice that these were marked as temporary allocations (because they are reclaimed under memory pressure), and on an already fragemented system we do the right thing anyway?
You try smaller and then for the next allocation start from the initial order which did not work previously.
Regards,
Tvrtko
_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH 12/42 drm/i915: Reuse the active golden render state batch
In reply to Chris WilsonThe golden render state is constant, but we recreate the batch setting it up for every new context. If we keep that batch in a volatile cache we can safely reuse it whenever we need to initialise a new context. We mark the pages as purgeable and use the shrinker to recover pages from the batch whenever we face memory pressues, recreating that batch afresh on the next new context.
--- drivers/gpu/drm/i915/i915_gem_render_state.c | 184 +++++++++++++++++---------- drivers/gpu/drm/i915/i915_gem_render_state.h | 4 +- drivers/gpu/drm/i915/intel_engine_cs.c | 5 + drivers/gpu/drm/i915/intel_lrc.c | 2 +- drivers/gpu/drm/i915/intel_ringbuffer.c | 2 +- drivers/gpu/drm/i915/intel_ringbuffer.h | 3 + 6 files changed, 129 insertions(+), 71 deletions(-)
drivers/gpu/drm/i915/i915_gem_render_state.c +116 -68
@@ -28,17 +28,19 @@#include "i915_drv.h" #include "intel_renderstate.h" -struct render_state { +struct intel_render_state {const struct intel_renderstate_rodata *rodata;struct i915_vma *vma; -u32 aux_batch_size; -u32 aux_batch_offset; +u32 batch_offset; +u32 batch_size; +u32 aux_offset; +u32 aux_size; };static const struct intel_renderstate_rodata * -render_state_get_rodata (const struct drm_i915_gem_request *req) +render_state_get_rodata (const struct intel_engine_cs *engine) { -switch (INTEL_GEN (req->i915)) { +switch (INTEL_GEN (engine->i915)) {case 6 :return &gen6_null_state;case 7 :
@@ -63,29 +65,27 @@ render_state_get_rodata(const struct drm_i915_gem_request *req)*/ #define OUT_BATCH (batch, i, val) \ do { \ - if (WARN_ON((i) >= PAGE_SIZE / sizeof(u32))) { \ - ret = -ENOSPC; \ - goto err_out; \ - } \ + if ((i) >= PAGE_SIZE / sizeof(u32)) \ + goto err; \ (batch)[(i)++] = (val); \ } while(0) -static int render_state_setup (struct render_state *so) +static int render_state_setup (struct intel_render_state *so, +struct drm_i915_private *i915) { -struct drm_device *dev = so->vma->vm->dev;const struct intel_renderstate_rodata *rodata = so->rodata; -const bool has_64bit_reloc =INTEL_GEN (dev) >=8 ; +const bool has_64bit_reloc =INTEL_GEN (i915) >=8 ; +struct drm_i915_gem_object *obj = so->vma->obj;unsigned int i =0 , reloc_index =0 ; -struct page *page; +unsigned int needs_clflush;u32 *d;int ret; - ret =i915_gem_object_set_to_cpu_domain (so->vma->obj, true); + ret =i915_gem_obj_prepare_shmem_write (obj, &needs_clflush);if (ret)return ret; - page =i915_gem_object_get_dirty_page (so->vma->obj,0 ); - d =kmap (page); + d =kmap_atomic (i915_gem_object_get_dirty_page (obj,0 ));while (i < rodata->batch_items) {u32 s = rodata->batch[i];
@@ -95,10 +95,8 @@ static int render_state_setup(struct render_state *so) s =lower_32_bits (r);if (has_64bit_reloc) {if (i +1 >= rodata->batch_items || - rodata->batch[i +1 ] !=0 ) { - ret = -EINVAL ; - goto err_out; - } + rodata->batch[i +1 ] !=0 ) + goto err; d[i++] = s; s =upper_32_bits (r);
@@ -110,12 +108,20 @@ static int render_state_setup(struct render_state *so) d[i++] = s; } +if (rodata->reloc[reloc_index] !=-1 ) { +DRM_ERROR ("only %d relocs resolved\n" , reloc_index); + goto err; + } + + so->batch_offset = so->vma->node.start; + so->batch_size = rodata->batch_items *sizeof (u32); +while (i %CACHELINE_DWORDS )OUT_BATCH (d, i,MI_NOOP ); - so->aux_batch_offset = i *sizeof (u32); + so->aux_offset = i *sizeof (u32); -if (HAS_POOLED_EU (dev)) { +if (HAS_POOLED_EU (i915)) {/* * We always program 3x6 pool config but depending upon which * subslice is disabled HW drops down to appropriate config
@@ -143,89 +149,131 @@ static int render_state_setup(struct render_state *so) }OUT_BATCH (d, i,MI_BATCH_BUFFER_END ); - so->aux_batch_size = (i *sizeof (u32)) - so->aux_batch_offset; - + so->aux_size = i *sizeof (u32) - so->aux_offset; + so->aux_offset += so->batch_offset;/* * Since we are sending length, we need to strictly conform to * all requirements. For Gen2 this must be a multiple of 8. */ - so->aux_batch_size =ALIGN (so->aux_batch_size,8 ); - -kunmap (page); - - ret =i915_gem_object_set_to_gtt_domain (so->vma->obj, false); -if (ret) -return ret; - -if (rodata->reloc[reloc_index] !=-1 ) { -DRM_ERROR ("only %d relocs resolved\n" , reloc_index); -return -EINVAL ; - } + so->aux_size =ALIGN (so->aux_size,8 ); -return 0 ; +if (needs_clflush) +drm_clflush_virt_range (d, i *sizeof (u32)); +kunmap_atomic (d); -err_out: -kunmap (page); + ret =i915_gem_object_set_to_gtt_domain (obj, false); +out: +i915_gem_obj_finish_shmem_access (obj);return ret; + +err: +kunmap_atomic (d); + ret = -EINVAL ; + goto out; }#undef OUT_BATCH -int i915_gem_render_state_init (struct drm_i915_gem_request *req) +int i915_gem_render_state_init (struct intel_engine_cs *engine) { -struct render_state so; +struct intel_render_state *so; +const struct intel_renderstate_rodata *rodata;struct drm_i915_gem_object *obj;int ret; -if (WARN_ON (req->engine->id !=RCS )) -return -ENOENT ; +if (engine->id !=RCS ) +return 0 ; - so.rodata =render_state_get_rodata (req); -if (!so.rodata) + rodata =render_state_get_rodata (engine); +if (!rodata)return 0 ; -if (so.rodata->batch_items *4 >4096 ) +if (rodata->batch_items *4 >4096 )return -EINVAL ; - obj =i915_gem_object_create_internal (&req->i915->drm,4096 ); -if (IS_ERR (obj)) -return PTR_ERR (obj); + so =kmalloc (sizeof (*so),GFP_KERNEL ); +if (!so) +return -ENOMEM ; - so.vma =i915_vma_create (obj, &req->i915->ggtt.base,NULL ); -if (IS_ERR (so.vma)) { - ret =PTR_ERR (so.vma); - goto err_obj; + obj =i915_gem_object_create_internal (&engine->i915->drm,4096 ); +if (IS_ERR (obj)) { + ret =PTR_ERR (obj); + goto err_free; } - ret =i915_vma_pin (so.vma,0 ,0 ,PIN_GLOBAL ); -if (ret) + so->vma =i915_vma_create (obj, &engine->i915->ggtt.base,NULL ); +if (IS_ERR (so->vma)) { + ret =PTR_ERR (so->vma); goto err_obj; + } + + so->rodata = rodata; + engine->render_state = so; +return 0 ; - ret =render_state_setup (&so); +err_obj: +i915_gem_object_put (obj); +err_free: +kfree (so); +return ret; +} + +int i915_gem_render_state_emit (struct drm_i915_gem_request *req) +{ +struct intel_render_state *so; +int ret; + + so = req->engine->render_state; +if (!so) +return 0 ; + +/* Recreate the page after shrinking */ +if (!so->vma->obj->pages) + so->batch_offset =-1 ; + + ret =i915_vma_pin (so->vma,0 ,0 ,PIN_GLOBAL |PIN_HIGH );if (ret) - goto err_unpin; +return ret; - ret = req->engine->emit_bb_start (req, so.vma->node.start, - so.rodata->batch_items *4 , +if (so->vma->node.start != so->batch_offset) { + ret =render_state_setup (so, req->i915); +if (ret) + goto err_unpin; + } + + ret = req->engine->emit_bb_start (req, + so->batch_offset, so->batch_size,I915_DISPATCH_SECURE );if (ret) goto err_unpin; -if (so.aux_batch_size >8 ) { +if (so->aux_size >8 ) { ret = req->engine->emit_bb_start (req, - (so.vma->node.start + - so.aux_batch_offset), - so.aux_batch_size, + so->aux_offset, so->aux_size,I915_DISPATCH_SECURE );if (ret) goto err_unpin; } -i915_vma_move_to_active (so.vma, req,0 ); +i915_vma_move_to_active (so->vma, req,0 ); err_unpin: -i915_vma_unpin (so.vma); -i915_vma_close (so.vma); -err_obj: -__i915_gem_object_release_unless_active (obj); +i915_vma_unpin (so->vma);return ret; } + +void i915_gem_render_state_fini (struct intel_engine_cs *engine) +{ +struct intel_render_state *so; +struct drm_i915_gem_object *obj; + + so =fetch_and_zero (&engine->render_state); +if (!so) +return ; + + obj = so->vma->obj; + +i915_vma_close (so->vma); +__i915_gem_object_release_unless_active (obj); + +kfree (so); +}
drivers/gpu/drm/i915/i915_gem_render_state.h +3 -1
@@ -26,6 +26,8 @@struct drm_i915_gem_request ; -int i915_gem_render_state_init (struct drm_i915_gem_request *req); +int i915_gem_render_state_init (struct intel_engine_cs *engine); +int i915_gem_render_state_emit (struct drm_i915_gem_request *req); +void i915_gem_render_state_fini (struct intel_engine_cs *engine);#endif /* _I915_GEM_RENDER_STATE_H_ */
drivers/gpu/drm/i915/intel_engine_cs.c +5 -0
@@ -301,6 +301,10 @@ int intel_engine_init_common(struct intel_engine_cs *engine)if (ret)return ret; + ret =i915_gem_render_state_init (engine); +if (ret) +return ret; +return 0 ; }
@@ -315,6 +319,7 @@ void intel_engine_cleanup_common(struct intel_engine_cs *engine) {intel_engine_cleanup_scratch (engine); +i915_gem_render_state_fini (engine);intel_engine_fini_breadcrumbs (engine);intel_engine_cleanup_cmd_parser (engine);i915_gem_batch_pool_fini (&engine->batch_pool);
drivers/gpu/drm/i915/intel_lrc.c +1 -1
@@ -1637,7 +1637,7 @@ static int gen8_init_rcs_context(struct drm_i915_gem_request *req)if (ret)DRM_ERROR ("MOCS failed to program: expect performance issues.\n" ); -return i915_gem_render_state_init (req); +return i915_gem_render_state_emit (req); }/**
drivers/gpu/drm/i915/intel_ringbuffer.c +1 -1
@@ -648,7 +648,7 @@ static int intel_rcs_ctx_init(struct drm_i915_gem_request *req)if (ret !=0 )return ret; - ret =i915_gem_render_state_init (req); + ret =i915_gem_render_state_emit (req);if (ret)return ret;
drivers/gpu/drm/i915/intel_ringbuffer.h +3 -0
@@ -157,6 +157,7 @@ struct i915_ctx_workarounds { };struct drm_i915_gem_request ; +struct intel_render_state ;struct intel_engine_cs {struct drm_i915_private *i915;
@@ -184,6 +185,8 @@ struct intel_engine_cs {unsigned int irq_shift;struct intel_ring *buffer; +struct intel_render_state *render_state; +/* Rather than have every client wait upon all user interrupts, * with the herd waking after every interrupt and each doing the * heavyweight seqno dance, we delegate the task (of being the
Signature · 6 lines
2.9.3 _______________________________________________ Intel-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH 13/42 drm/i915: Markup GEM API with lockdep asserts
In reply to Chris WilsonAdd lockdep_assert_held(struct_mutex) to the API preamble of the internal GEM interfaces.
--- drivers/gpu/drm/i915/i915_gem.c | 21 +++++++++++++++++++++ drivers/gpu/drm/i915/i915_gem_evict.c | 5 ++++- drivers/gpu/drm/i915/i915_gem_gtt.c | 2 ++ drivers/gpu/drm/i915/i915_gem_render_state.c | 2 ++ drivers/gpu/drm/i915/i915_gem_request.c | 6 ++++++ 5 files changed, 35 insertions(+), 1 deletion(-)
drivers/gpu/drm/i915/i915_gem.c +21 -0
@@ -104,6 +104,8 @@ i915_gem_wait_for_error(struct i915_gpu_error *error) {int ret; +might_sleep (); +if (!i915_reset_in_progress (error))return 0 ;
@@ -2295,6 +2297,8 @@ i915_gem_object_put_pages(struct drm_i915_gem_object *obj) {const struct drm_i915_gem_object_ops *ops = obj->ops; +lockdep_assert_held (&obj->base.dev->struct_mutex); +if (obj->pages ==NULL )return 0 ;
@@ -2466,6 +2470,8 @@ i915_gem_object_get_pages(struct drm_i915_gem_object *obj)const struct drm_i915_gem_object_ops *ops = obj->ops;int ret; +lockdep_assert_held (&obj->base.dev->struct_mutex); +if (obj->pages)return 0 ;
@@ -2746,6 +2752,8 @@ void i915_gem_reset(struct drm_i915_private *dev_priv) {struct intel_engine_cs *engine; +lockdep_assert_held (&dev_priv->drm.struct_mutex); +i915_gem_retire_requests (dev_priv);for_each_engine (engine, dev_priv)
@@ -2974,6 +2982,8 @@ int i915_vma_unbind(struct i915_vma *vma)unsigned long active;int ret; +lockdep_assert_held (&obj->base.dev->struct_mutex); +/* First wait upon any activity as retiring the request may * have side-effects such as unpinning or even unbinding this vma. */
@@ -3366,6 +3376,7 @@ i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)int ret;lockdep_assert_held (&obj->base.dev->struct_mutex); + ret = i915_gem_object_wait(obj, I915_WAIT_INTERRUPTIBLE | I915_WAIT_LOCKED |
@@ -3444,6 +3455,8 @@ int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,struct i915_vma *vma;int ret =0 ; +lockdep_assert_held (&obj->base.dev->struct_mutex); +if (obj->cache_level == cache_level) goto out;
@@ -3654,6 +3667,8 @@ i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,u32 old_read_domains, old_write_domain;int ret; +lockdep_assert_held (&obj->base.dev->struct_mutex); +/* Mark the pin_display early so that we account for the * display coherency whilst setting up the cache domains. */
@@ -3720,6 +3735,8 @@ err_unpin_display:void i915_gem_object_unpin_from_display_plane (struct i915_vma *vma) { +lockdep_assert_held (&vma->vm->dev->struct_mutex); +if (WARN_ON (vma->obj->pin_display ==0 ))return ;
@@ -3749,6 +3766,7 @@ i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)int ret;lockdep_assert_held (&obj->base.dev->struct_mutex); + ret = i915_gem_object_wait(obj, I915_WAIT_INTERRUPTIBLE | I915_WAIT_LOCKED |
@@ -3904,6 +3922,7 @@ int __i915_vma_do_pin(struct i915_vma *vma,unsigned int bound = vma->flags;int ret; +lockdep_assert_held (&vma->vm->dev->struct_mutex);GEM_BUG_ON ((flags & (PIN_GLOBAL |PIN_USER )) ==0 );GEM_BUG_ON ((flags &PIN_GLOBAL ) && !i915_vma_is_ggtt (vma));
@@ -3944,6 +3963,8 @@ i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj,struct i915_vma *vma;int ret; +lockdep_assert_held (&obj->base.dev->struct_mutex); + vma =i915_gem_obj_lookup_or_create_vma (obj, vm, view);if (IS_ERR (vma))return vma;
drivers/gpu/drm/i915/i915_gem_evict.c +4 -1
@@ -102,6 +102,7 @@ i915_gem_evict_something(struct i915_address_space *vm,struct i915_vma *vma, *next;int ret; +lockdep_assert_held (&vm->dev->struct_mutex);trace_i915_gem_evict (vm, min_size, alignment, flags);/*
@@ -212,6 +213,8 @@ i915_gem_evict_for_vma(struct i915_vma *target) {struct drm_mm_node *node, *next; +lockdep_assert_held (&target->vm->dev->struct_mutex); +list_for_each_entry_safe (node, next, &target->vm->mm.head_node.node_list, node_list) {
@@ -265,7 +268,7 @@ int i915_gem_evict_vm(struct i915_address_space *vm, bool do_idle)struct i915_vma *vma, *next;int ret; -WARN_ON (!mutex_is_locked (&vm->dev->struct_mutex)); +lockdep_assert_held (&vm->dev->struct_mutex);trace_i915_gem_evict_vm (vm);if (do_idle) {
drivers/gpu/drm/i915/i915_gem_gtt.c +2 -0
@@ -3427,6 +3427,7 @@ i915_vma_create(struct drm_i915_gem_object *obj,struct i915_address_space *vm, const struct i915_ggtt_view *view) { + lockdep_assert_held(&obj->base.dev->struct_mutex);GEM_BUG_ON (view && !i915_is_ggtt (vm));GEM_BUG_ON (i915_gem_obj_to_vma (obj, vm, view));
@@ -3454,6 +3455,7 @@ i915_gem_obj_lookup_or_create_vma(struct drm_i915_gem_object *obj, {struct i915_vma *vma; +lockdep_assert_held (&obj->base.dev->struct_mutex);GEM_BUG_ON (view && !i915_is_ggtt (vm)); vma =i915_gem_obj_to_vma (obj, vm, view);
drivers/gpu/drm/i915/i915_gem_render_state.c +2 -0
@@ -223,6 +223,8 @@ int i915_gem_render_state_emit(struct drm_i915_gem_request *req)struct intel_render_state *so;int ret; +lockdep_assert_held (&req->i915->drm.struct_mutex); + so = req->engine->render_state;if (!so)return 0 ;
drivers/gpu/drm/i915/i915_gem_request.c +6 -0
@@ -143,6 +143,9 @@ static void i915_gem_request_retire(struct drm_i915_gem_request *request) {struct i915_gem_active *active, *next; +lockdep_assert_held (&request->i915->drm.struct_mutex); +GEM_BUG_ON (!i915_gem_request_completed (request)); +trace_i915_gem_request_retire (request);list_del_init (&request->link);
@@ -267,6 +270,8 @@ int i915_gem_set_seqno(struct drm_device *dev, u32 seqno)struct drm_i915_private *dev_priv =to_i915 (dev);int ret; +lockdep_assert_held (&dev_priv->drm.struct_mutex); +if (seqno ==0 )return -EINVAL ;
@@ -603,6 +608,7 @@ void __i915_add_request(struct drm_i915_gem_request *request, bool flush_caches)u32 reserved_tail;int ret; +lockdep_assert_held (&request->i915->drm.struct_mutex);trace_i915_gem_request_add (request);/*
Signature · 6 lines
2.9.3 _______________________________________________ Intel-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH 14/42 drm/i915: Use a radixtree for random access to the object's backing storage
In reply to Chris WilsonA while ago we switched from a contiguous array of pages into an sglist, for that was both more convenient for mapping to hardware and avoided the requirement for a vmalloc array of pages on every object. However, certain GEM API calls (like pwrite, pread as well as performing relocations) do desire access to individual struct pages. A quick hack was to introduce a cache of the last access such that finding the following page was quick - this works so long as the caller desired sequential access. Walking backwards, or multiple callers, still hits a slow linear search for each page. One solution is to store each successful lookup in a radix tree.
--- drivers/gpu/drm/i915/i915_drv.h | 57 ++++-------- drivers/gpu/drm/i915/i915_gem.c | 149 ++++++++++++++++++++++++++++---- drivers/gpu/drm/i915/i915_gem_stolen.c | 4 +- drivers/gpu/drm/i915/i915_gem_userptr.c | 4 +- 4 files changed, 154 insertions(+), 60 deletions(-)
drivers/gpu/drm/i915/i915_drv.h +18 -39
@@ -2278,9 +2278,12 @@ struct drm_i915_gem_object {struct sg_table *pages;int pages_pin_count; -struct get_page { -struct scatterlist *sg; -int last; +struct i915_gem_object_page_iter { +struct scatterlist *sg_pos; +unsigned long sg_idx; + +struct radix_tree_root radix; +struct mutex lock; } get_page;void *mapping;
@@ -3168,45 +3171,21 @@ static inline int __sg_page_count(struct scatterlist *sg)return sg->length >>PAGE_SHIFT ; } -struct page * -i915_gem_object_get_dirty_page (struct drm_i915_gem_object *obj,int n); - -static inline dma_addr_t -i915_gem_object_get_dma_address (struct drm_i915_gem_object *obj,int n) -{ -if (n < obj->get_page.last) { - obj->get_page.sg = obj->pages->sgl; - obj->get_page.last =0 ; - } - -while (obj->get_page.last +__sg_page_count (obj->get_page.sg) <= n) { - obj->get_page.last +=__sg_page_count (obj->get_page.sg++); -if (unlikely (sg_is_chain (obj->get_page.sg))) - obj->get_page.sg =sg_chain_ptr (obj->get_page.sg); - } - -return sg_dma_address (obj->get_page.sg) + ((n - obj->get_page.last) <<PAGE_SHIFT ); -} - -static inline struct page * -i915_gem_object_get_page (struct drm_i915_gem_object *obj,int n) -{ -if (WARN_ON (n >= obj->base.size >>PAGE_SHIFT )) -return NULL ; +struct scatterlist * +i915_gem_object_get_sg (struct drm_i915_gem_object *obj, +unsigned long n,unsigned int *offset); -if (n < obj->get_page.last) { - obj->get_page.sg = obj->pages->sgl; - obj->get_page.last =0 ; - } +struct page * +i915_gem_object_get_page (struct drm_i915_gem_object *obj, +unsigned long n); -while (obj->get_page.last +__sg_page_count (obj->get_page.sg) <= n) { - obj->get_page.last +=__sg_page_count (obj->get_page.sg++); -if (unlikely (sg_is_chain (obj->get_page.sg))) - obj->get_page.sg =sg_chain_ptr (obj->get_page.sg); - } +struct page * +i915_gem_object_get_dirty_page (struct drm_i915_gem_object *obj, +unsigned long n); -return nth_page (sg_page (obj->get_page.sg), n - obj->get_page.last); -} +dma_addr_t +i915_gem_object_get_dma_address (struct drm_i915_gem_object *obj, +unsigned long n);static inline void i915_gem_object_pin_pages (struct drm_i915_gem_object *obj) {
drivers/gpu/drm/i915/i915_gem.c +132 -17
182 lines not on this page · raw message
drivers/gpu/drm/i915/i915_gem_stolen.c +2 -2
@@ -594,8 +594,8 @@ _i915_gem_object_create_stolen(struct drm_device *dev,if (obj->pages ==NULL ) goto cleanup; - obj->get_page.sg = obj->pages->sgl; - obj->get_page.last =0 ; + obj->get_page.sg_pos = obj->pages->sgl; + obj->get_page.sg_idx =0 ;i915_gem_object_pin_pages (obj); obj->stolen = stolen;
drivers/gpu/drm/i915/i915_gem_userptr.c +2 -2
@@ -526,8 +526,8 @@ __i915_gem_userptr_get_pages_worker(struct work_struct *_work)if (ret ==0 ) {list_add_tail (&obj->global_list, &to_i915 (dev)->mm.unbound_list); - obj->get_page.sg = obj->pages->sgl; - obj->get_page.last =0 ; + obj->get_page.sg_pos = obj->pages->sgl; + obj->get_page.sg_idx =0 ; pinned =0 ; } }
Signature · 6 lines
2.9.3 _______________________________________________ Intel-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/intel-gfx
Joonas Lahtinen
maintainerRe: [PATCH 14/42] drm/i915: Use a radixtree for random access to the object's backing storage
Patches 14-25 to a separate series. Will continue their review there.
Regards, Joonas
Signature · 7 lines
Joonas Lahtinen Open Source Technology Center Intel Corporation _______________________________________________ Intel-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/intel-gfx
Re: [PATCH 14/42] drm/i915: Use a radixtree for random access to the object's backing storage
Quoted 1 line · Joonas Lahtinen
Patches 14-25 to a separate series. Will continue their review there.
Nope. For reasons as I explained when I pulled them in, they are required to prevent a critical use-after-free when moving the active reference handling onto the vma retirement.
-Chris
Signature · 5 lines
Chris Wilson, Intel Open Source Technology Centre _______________________________________________ Intel-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/intel-gfx
Joonas Lahtinen
maintainerRe: [PATCH 14/42] drm/i915: Use a radixtree for random access to the object's backing storage
Quoted 7 lines · Chris Wilson
Quoted 2 lines · Joonas Lahtinen
Patches 14-25 to a separate series. Will continue their review there.
Nope. For reasons as I explained when I pulled them in, they are required to prevent a critical use-after-free when moving the active reference handling onto the vma retirement.
Ack.
As discussed in IRC, asked John to give them a look too.
Will proceed with the remaining ones on Monday.
Regards, Joonas
Signature · 7 lines
Joonas Lahtinen Open Source Technology Center Intel Corporation _______________________________________________ Intel-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/intel-gfx
Re: [PATCH 14/42] drm/i915: Use a radixtree for random access to the object's backing storage
In reply to Chris WilsonQuoted 11 lines · Chris Wilson
A while ago we switched from a contiguous array of pages into an sglist, for that was both more convenient for mapping to hardware and avoided the requirement for a vmalloc array of pages on every object. However, certain GEM API calls (like pwrite, pread as well as performing relocations) do desire access to individual struct pages. A quick hack was to introduce a cache of the last access such that finding the following page was quick - this works so long as the caller desired sequential access. Walking backwards, or multiple callers, still hits a slow linear search for each page. One solution is to store each successful lookup in a radix tree.
Quoted 7 lines
Signed-off-by Chris Wilson <[email protected]>--- drivers/gpu/drm/i915/i915_drv.h | 57 ++++-------- drivers/gpu/drm/i915/i915_gem.c | 149 ++++++++++++++++++++++++++++---- drivers/gpu/drm/i915/i915_gem_stolen.c | 4 +- drivers/gpu/drm/i915/i915_gem_userptr.c | 4 +- 4 files changed, 154 insertions(+), 60 deletions(-)
Am curious to know if you have any performance data to show how much of a benefit this is? Are there any real world apps that would notice or just IGT benchmarks?
_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
Re: [PATCH 14/42] drm/i915: Use a radixtree for random access to the object's backing storage
In reply to Chris WilsonQuoted 194 lines · Chris Wilson
A while ago we switched from a contiguous array of pages into an sglist, for that was both more convenient for mapping to hardware and avoided the requirement for a vmalloc array of pages on every object. However, certain GEM API calls (like pwrite, pread as well as performing relocations) do desire access to individual struct pages. A quick hack was to introduce a cache of the last access such that finding the following page was quick - this works so long as the caller desired sequential access. Walking backwards, or multiple callers, still hits a slow linear search for each page. One solution is to store each successful lookup in a radix tree.
Signed-off-by Chris Wilson <[email protected]>--- drivers/gpu/drm/i915/i915_drv.h | 57 ++++-------- drivers/gpu/drm/i915/i915_gem.c | 149 ++++++++++++++++++++++++++++---- drivers/gpu/drm/i915/i915_gem_stolen.c | 4 +- drivers/gpu/drm/i915/i915_gem_userptr.c | 4 +- 4 files changed, 154 insertions(+), 60 deletions(-)drivers/gpu/drm/i915/i915_drv.h +18
-39@@ -2278,9 +2278,12 @@ struct drm_i915_gem_object {struct sg_table *pages;int pages_pin_count; -struct get_page { -struct scatterlist *sg; -int last; +struct i915_gem_object_page_iter { +struct scatterlist *sg_pos; +unsigned long sg_idx; + +struct radix_tree_root radix; +struct mutex lock; } get_page;void *mapping;@@ -3168,45 +3171,21 @@ static inline int __sg_page_count(struct scatterlist *sg)return sg->length >>PAGE_SHIFT ; } -struct page * -i915_gem_object_get_dirty_page (struct drm_i915_gem_object *obj,int n); - -static inline dma_addr_t -i915_gem_object_get_dma_address (struct drm_i915_gem_object *obj,int n) -{ -if (n < obj->get_page.last) { - obj->get_page.sg = obj->pages->sgl; - obj->get_page.last =0 ; - } - -while (obj->get_page.last +__sg_page_count (obj->get_page.sg) <= n) { - obj->get_page.last +=__sg_page_count (obj->get_page.sg++); -if (unlikely (sg_is_chain (obj->get_page.sg))) - obj->get_page.sg =sg_chain_ptr (obj->get_page.sg); - } - -return sg_dma_address (obj->get_page.sg) + ((n - obj->get_page.last) <<PAGE_SHIFT ); -} - -static inline struct page * -i915_gem_object_get_page (struct drm_i915_gem_object *obj,int n) -{ -if (WARN_ON (n >= obj->base.size >>PAGE_SHIFT )) -return NULL ; +struct scatterlist * +i915_gem_object_get_sg (struct drm_i915_gem_object *obj, +unsigned long n,unsigned int *offset); -if (n < obj->get_page.last) { - obj->get_page.sg = obj->pages->sgl; - obj->get_page.last =0 ; - } +struct page * +i915_gem_object_get_page (struct drm_i915_gem_object *obj, +unsigned long n); -while (obj->get_page.last +__sg_page_count (obj->get_page.sg) <= n) { - obj->get_page.last +=__sg_page_count (obj->get_page.sg++); -if (unlikely (sg_is_chain (obj->get_page.sg))) - obj->get_page.sg =sg_chain_ptr (obj->get_page.sg); - } +struct page * +i915_gem_object_get_dirty_page (struct drm_i915_gem_object *obj, +unsigned long n); -return nth_page (sg_page (obj->get_page.sg), n - obj->get_page.last); -} +dma_addr_t +i915_gem_object_get_dma_address (struct drm_i915_gem_object *obj, +unsigned long n);static inline void i915_gem_object_pin_pages (struct drm_i915_gem_object *obj) {drivers/gpu/drm/i915/i915_gem.c +37
-17@@ -2292,6 +2292,15 @@ i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)kfree (obj->pages); } +static void __i915_gem_object_reset_page_iter (struct drm_i915_gem_object *obj) +{ +struct radix_tree_iter iter; +void **slot; + +radix_tree_for_each_slot (slot, &obj->get_page.radix, &iter,0 ) +radix_tree_delete (&obj->get_page.radix, iter.index); +} +int i915_gem_object_put_pages (struct drm_i915_gem_object *obj) {@@ -2324,6 +2333,8 @@ i915_gem_object_put_pages(struct drm_i915_gem_object *obj) obj->mapping =NULL ; } +__i915_gem_object_reset_page_iter (obj); + ops->put_pages (obj); obj->pages =NULL ;@@ -2488,8 +2499,8 @@ i915_gem_object_get_pages(struct drm_i915_gem_object *obj)list_add_tail (&obj->global_list, &dev_priv->mm.unbound_list); - obj->get_page.sg = obj->pages->sgl; - obj->get_page.last =0 ; + obj->get_page.sg_pos = obj->pages->sgl; + obj->get_page.sg_idx =0 ;return 0 ; }@@ -4242,6 +4253,8 @@ void i915_gem_object_init(struct drm_i915_gem_object *obj, obj->frontbuffer_ggtt_origin =ORIGIN_GTT ; obj->madv =I915_MADV_WILLNEED ; +INIT_RADIX_TREE (&obj->get_page.radix,GFP_KERNEL ); +mutex_init (&obj->get_page.lock);i915_gem_info_add_obj (to_i915 (obj->base.dev), obj->base.size); }@@ -4904,21 +4917,6 @@ void i915_gem_track_fb(struct drm_i915_gem_object *old, } } -/* Like i915_gem_object_get_page(), but mark the returned page dirty */ -struct page * -i915_gem_object_get_dirty_page (struct drm_i915_gem_object *obj,int n) -{ -struct page *page; - -/* Only default objects have per-page dirty tracking */ -if (WARN_ON (!i915_gem_object_has_struct_page (obj))) -return NULL ; - - page =i915_gem_object_get_page (obj, n); -set_page_dirty (page); -return page; -} -/* Allocate a new GEM object and fill it with the supplied data */ struct drm_i915_gem_object * i915_gem_object_create_from_data(struct drm_device *dev,@@ -4959,3 +4957,120 @@ fail:i915_gem_object_put (obj);return ERR_PTR (ret); } + +static struct scatterlist * +__i915_gem_object_get_sg (struct drm_i915_gem_object *obj, +unsigned long n,unsigned int *offset) +{ +struct scatterlist *sg = obj->pages->sgl; +int idx =0 ; + +while (idx +__sg_page_count (sg) <= n) { + idx +=__sg_page_count (sg++); +if (unlikely (sg_is_chain (sg))) + sg =sg_chain_ptr (sg); + } + + *offset = n - idx; +return sg; +} + +struct scatterlist * +i915_gem_object_get_sg (struct drm_i915_gem_object *obj, +unsigned long n, +unsigned int *offset)
drivers/gpu/drm/i915/i915_gem.c fail: · line 4991
Quoted 10 lines
drivers/gpu/drm/i915/i915_gem.c +10
-0+{ +struct i915_gem_object_page_iter *iter = &obj->get_page; +struct scatterlist *sg; + +GEM_BUG_ON (n >= obj->base.size >>PAGE_SHIFT ); +GEM_BUG_ON (obj->pages_pin_count ==0 ); + +if (n <READ_ONCE (iter->sg_idx)) + goto lookup; +
Ok, so on lookup of "n" you build the radix tree for all sg entries from zero to n. Therefore for a lookup below the current index, there must be a radix tree entry, correct?
Quoted 20 lines
drivers/gpu/drm/i915/i915_gem.c +20
-0+mutex_lock (&iter->lock); +if (n >= iter->sg_idx && + n < iter->sg_idx +__sg_page_count (iter->sg_pos)) { + sg = iter->sg_pos; + *offset = n - iter->sg_idx; +mutex_unlock (&iter->lock); +return sg; + } + +while (iter->sg_idx <= n) { +unsigned long exception; +unsigned int count, i; + +radix_tree_insert (&iter->radix, + iter->sg_idx, + iter->sg_pos); + + exception = +RADIX_TREE_EXCEPTIONAL_ENTRY | + iter->sg_idx <<RADIX_TREE_EXCEPTIONAL_SHIFT ;
drivers/gpu/drm/i915/i915_gem.c fail: · line 5021
Quoted 10 lines
drivers/gpu/drm/i915/i915_gem.c +10
-0+ count =__sg_page_count (iter->sg_pos); +for (i =1 ; i < count; i++) +radix_tree_insert (&iter->radix, + iter->sg_idx + i, + (void *)exception); + + iter->sg_idx += count; + iter->sg_pos =__sg_next (iter->sg_pos); + } +mutex_unlock (&iter->lock);
Why not avoid falling through the lookup and return the previous sg from here?
drivers/gpu/drm/i915/i915_gem.c fail: · line 5030
Quoted 9 lines
drivers/gpu/drm/i915/i915_gem.c +9
-0+ +lookup: +rcu_read_lock (); + sg =radix_tree_lookup (&iter->radix, n); +rcu_read_unlock (); + +if (unlikely (!sg)) +return __i915_gem_object_get_sg (obj, n, offset); +
Considering the first observation from above, when does it then happen that there is no radix tree entry at this point?
Or in other words, maybe some comments should be added to this patch - there are none and it is not that trivial.
Quoted 66 lines
drivers/gpu/drm/i915/i915_gem.c +46
-0+ *offset =0 ; +if (unlikely (radix_tree_exception (sg))) { +unsigned long base = + (unsigned long )sg >>RADIX_TREE_EXCEPTIONAL_SHIFT ; + sg =radix_tree_lookup (&iter->radix, base); + *offset = n - base; + } +return sg; +} + +struct page * +i915_gem_object_get_page (struct drm_i915_gem_object *obj,unsigned long n) +{ +struct scatterlist *sg; +unsigned int offset; + +GEM_BUG_ON (!i915_gem_object_has_struct_page (obj)); + + sg =i915_gem_object_get_sg (obj, n, &offset); +return nth_page (sg_page (sg), offset); +} + +/* Like i915_gem_object_get_page(), but mark the returned page dirty */ +struct page * +i915_gem_object_get_dirty_page (struct drm_i915_gem_object *obj, +unsigned long n) +{ +struct page *page; + + page =i915_gem_object_get_page (obj, n); +if (!obj->dirty) +set_page_dirty (page); + +return page; +} + +dma_addr_t +i915_gem_object_get_dma_address (struct drm_i915_gem_object *obj, +unsigned long n) +{ +struct scatterlist *sg; +unsigned int offset; + + sg =i915_gem_object_get_sg (obj, n, &offset); +return sg_dma_address (sg) + (offset <<PAGE_SHIFT ); +}drivers/gpu/drm/i915/i915_gem_stolen.c +2
-2@@ -594,8 +594,8 @@ _i915_gem_object_create_stolen(struct drm_device *dev,if (obj->pages ==NULL ) goto cleanup; - obj->get_page.sg = obj->pages->sgl; - obj->get_page.last =0 ; + obj->get_page.sg_pos = obj->pages->sgl; + obj->get_page.sg_idx =0 ;i915_gem_object_pin_pages (obj); obj->stolen = stolen;drivers/gpu/drm/i915/i915_gem_userptr.c +0
-0
drivers/gpu/drm/i915/i915_gem_userptr.c __i915_gem_userptr_get_pages_worker(struct work_struct *_work) · line 533
Quoted 10 lines
drivers/gpu/drm/i915/i915_gem_userptr.c +2
-2@@ -526,8 +526,8 @@ __i915_gem_userptr_get_pages_worker(struct work_struct *_work)if (ret ==0 ) {list_add_tail (&obj->global_list, &to_i915 (dev)->mm.unbound_list); - obj->get_page.sg = obj->pages->sgl; - obj->get_page.last =0 ; + obj->get_page.sg_pos = obj->pages->sgl; + obj->get_page.sg_idx =0 ; pinned =0 ; } }
Regards,
Tvrtko
_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
Re: [PATCH 14/42] drm/i915: Use a radixtree for random access to the object's backing storage
Quoted 198 lines · Tvrtko Ursulin
Quoted 196 lines · Chris Wilson
A while ago we switched from a contiguous array of pages into an sglist, for that was both more convenient for mapping to hardware and avoided the requirement for a vmalloc array of pages on every object. However, certain GEM API calls (like pwrite, pread as well as performing relocations) do desire access to individual struct pages. A quick hack was to introduce a cache of the last access such that finding the following page was quick - this works so long as the caller desired sequential access. Walking backwards, or multiple callers, still hits a slow linear search for each page. One solution is to store each successful lookup in a radix tree.
Signed-off-by Chris Wilson <[email protected]>--- drivers/gpu/drm/i915/i915_drv.h | 57 ++++-------- drivers/gpu/drm/i915/i915_gem.c | 149 ++++++++++++++++++++++++++++---- drivers/gpu/drm/i915/i915_gem_stolen.c | 4 +- drivers/gpu/drm/i915/i915_gem_userptr.c | 4 +- 4 files changed, 154 insertions(+), 60 deletions(-)drivers/gpu/drm/i915/i915_drv.h +6
-3@@ -2278,9 +2278,12 @@ struct drm_i915_gem_object {struct sg_table *pages;int pages_pin_count; -struct get_page { -struct scatterlist *sg; -int last; +struct i915_gem_object_page_iter { +struct scatterlist *sg_pos; +unsigned long sg_idx; + +struct radix_tree_root radix; +struct mutex lock; } get_page;void *mapping; @@ -3168,45 +3171,21 @@ static inline int __sg_page_count(structscatterlist *sg)return sg->length >> PAGE_SHIFT;}
-struct page *drivers/gpu/drm/i915/i915_drv.h +0
-11-i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj, int n); - -static inline dma_addr_t -i915_gem_object_get_dma_address (struct drm_i915_gem_object *obj,int n) -{ -if (n < obj->get_page.last) { - obj->get_page.sg = obj->pages->sgl; - obj->get_page.last =0 ; - } - - while (obj->get_page.last + __sg_page_count(obj->get_page.sg) <=n) {
drivers/gpu/drm/i915/i915_drv.h +0
-6- obj->get_page.last +=__sg_page_count (obj->get_page.sg++); -if (unlikely (sg_is_chain (obj->get_page.sg))) - obj->get_page.sg =sg_chain_ptr (obj->get_page.sg); - } - - return sg_dma_address(obj->get_page.sg) + ((n -
obj->get_page.last) <<PAGE_SHIFT);drivers/gpu/drm/i915/i915_drv.h +6
-10-} - -static inline struct page * -i915_gem_object_get_page (struct drm_i915_gem_object *obj,int n) -{ -if (WARN_ON (n >= obj->base.size >>PAGE_SHIFT )) -return NULL ; +struct scatterlist * +i915_gem_object_get_sg (struct drm_i915_gem_object *obj, +unsigned long n,unsigned int *offset); - if (n < obj->get_page.last) { - obj->get_page.sg = obj->pages->sgl; - obj->get_page.last =0 ; - } +struct page * +i915_gem_object_get_page (struct drm_i915_gem_object *obj, +unsigned long n); - while (obj->get_page.last + __sg_page_count(obj->get_page.sg)<= n) {
drivers/gpu/drm/i915/i915_drv.h +3
-4- obj->get_page.last +=__sg_page_count (obj->get_page.sg++); -if (unlikely (sg_is_chain (obj->get_page.sg))) - obj->get_page.sg =sg_chain_ptr (obj->get_page.sg); - } +struct page * +i915_gem_object_get_dirty_page (struct drm_i915_gem_object *obj, +unsigned long n); - return nth_page(sg_page(obj->get_page.sg), n -
obj->get_page.last);drivers/gpu/drm/i915/i915_drv.h +3
-1-} +dma_addr_t +i915_gem_object_get_dma_address (struct drm_i915_gem_object *obj, +unsigned long n); static inline void i915_gem_object_pin_pages(struct
drm_i915_gem_object*obj)
{drivers/gpu/drm/i915/i915_gem.c +9
-0@@ -2292,6 +2292,15 @@ i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj) kfree(obj->pages); } +static void __i915_gem_object_reset_page_iter(structdrm_i915_gem_object *obj) +{ +struct radix_tree_iter iter; +void **slot; + +radix_tree_for_each_slot (slot, &obj->get_page.radix, &iter,0 ) +radix_tree_delete (&obj->get_page.radix, iter.index); +} +int i915_gem_object_put_pages (struct drm_i915_gem_object *obj) {@@ -2324,6 +2333,8 @@ i915_gem_object_put_pages(struct drm_i915_gem_object *obj) obj->mapping = NULL; } +__i915_gem_object_reset_page_iter (obj); + ops->put_pages (obj); obj->pages =NULL ; @@ -2488,8 +2499,8 @@ i915_gem_object_get_pages(struct
drm_i915_gem_object*obj)list_add_tail(&obj->global_list, &dev_priv->mm.unbound_list);-
obj->get_page.sg=obj->pages->sgl;drivers/gpu/drm/i915/i915_gem.c +4
-1- obj->get_page.last =0 ; + obj->get_page.sg_pos = obj->pages->sgl; + obj->get_page.sg_idx =0 ;return 0 ; }@@ -4242,6 +4253,8 @@ void i915_gem_object_init(struct drm_i915_gem_object *obj, obj->frontbuffer_ggtt_origin =ORIGIN_GTT ; obj->madv =I915_MADV_WILLNEED ; +INIT_RADIX_TREE (&obj->get_page.radix,GFP_KERNEL ); +mutex_init (&obj->get_page.lock);i915_gem_info_add_obj (to_i915 (obj->base.dev), obj->base.size); }@@ -4904,21 +4917,6 @@ void i915_gem_track_fb(struct drm_i915_gem_object *old, } } -/* Like i915_gem_object_get_page(), but mark the returned pagedirty */
drivers/gpu/drm/i915/i915_gem.c +26
-14-struct page * -i915_gem_object_get_dirty_page (struct drm_i915_gem_object *obj,int n) -{ -struct page *page; - -/* Only default objects have per-page dirty tracking */ -if (WARN_ON (!i915_gem_object_has_struct_page (obj))) -return NULL ; - - page =i915_gem_object_get_page (obj, n); -set_page_dirty (page); -return page; -} -/* Allocate a new GEM object and fill it with the supplied data */ struct drm_i915_gem_object * i915_gem_object_create_from_data(struct drm_device *dev,@@ -4959,3 +4957,120 @@ fail:i915_gem_object_put (obj);return ERR_PTR (ret); } + +static struct scatterlist * +__i915_gem_object_get_sg (struct drm_i915_gem_object *obj, +unsigned long n,unsigned int *offset) +{ +struct scatterlist *sg = obj->pages->sgl; +int idx =0 ; + +while (idx +__sg_page_count (sg) <= n) { + idx +=__sg_page_count (sg++); +if (unlikely (sg_is_chain (sg))) + sg =sg_chain_ptr (sg); + } + + *offset = n - idx; +return sg; +} + +struct scatterlist * +i915_gem_object_get_sg (struct drm_i915_gem_object *obj, +unsigned long n, +unsigned int *offset) +{ +struct i915_gem_object_page_iter *iter = &obj->get_page; +struct scatterlist *sg; +
Quoted 8 lines
drivers/gpu/drm/i915/i915_gem.c fail: · line 4991
Quoted 6 lines
drivers/gpu/drm/i915/i915_gem.c +6
-0+GEM_BUG_ON (n >= obj->base.size >>PAGE_SHIFT ); +GEM_BUG_ON (obj->pages_pin_count ==0 ); + +if (n <READ_ONCE (iter->sg_idx)) + goto lookup; +Ok, so on lookup of "n" you build the radix tree for all sg entries from zero to n. Therefore for a lookup below the current index, there must be a radix tree entry, correct?
That is my understanding.
Quoted 25 lines
Quoted 25 lines
drivers/gpu/drm/i915/i915_gem.c +24
-0+mutex_lock (&iter->lock); +if (n >= iter->sg_idx && + n < iter->sg_idx +__sg_page_count (iter->sg_pos)) { + sg = iter->sg_pos; + *offset = n - iter->sg_idx; +mutex_unlock (&iter->lock); +return sg; + } + +while (iter->sg_idx <= n) { +unsigned long exception; +unsigned int count, i; + +radix_tree_insert (&iter->radix, + iter->sg_idx, + iter->sg_pos); + + exception = +RADIX_TREE_EXCEPTIONAL_ENTRY | + iter->sg_idx <<RADIX_TREE_EXCEPTIONAL_SHIFT ; + count =__sg_page_count (iter->sg_pos); +for (i =1 ; i < count; i++) + radix_tree_insert(&iter->radix, + iter->sg_idx + i,
Quoted 8 lines
drivers/gpu/drm/i915/i915_gem.c fail: · line 5021
Quoted 6 lines
drivers/gpu/drm/i915/i915_gem.c +6
-0+ (void *)exception); + + iter->sg_idx += count; + iter->sg_pos =__sg_next (iter->sg_pos); + } +mutex_unlock (&iter->lock);Why not avoid falling through the lookup and return the previous sg from here?
I thought I worked through this before and decided there was a good reason. However, looking again a few days later, I can't obviously see why. As you note below, this is non-trivial stuff and would definitely benefit from some explanatory comments.
Quoted 95 lines
drivers/gpu/drm/i915/i915_gem.c fail: · line 5030
Quoted 10 lines
drivers/gpu/drm/i915/i915_gem.c +9
-0+ +lookup: +rcu_read_lock (); + sg =radix_tree_lookup (&iter->radix, n); +rcu_read_unlock (); + +if (unlikely (!sg)) +return __i915_gem_object_get_sg (obj, n, offset); +Considering the first observation from above, when does it then happen that there is no radix tree entry at this point?
Or in other words, maybe some comments should be added to this patch - there are none and it is not that trivial.
Quoted 76 lines
drivers/gpu/drm/i915/i915_gem.c +12
-0+ *offset =0 ; +if (unlikely (radix_tree_exception (sg))) { +unsigned long base = + (unsigned long )sg >>RADIX_TREE_EXCEPTIONAL_SHIFT ; + sg =radix_tree_lookup (&iter->radix, base); + *offset = n - base; + } +return sg; +} + +struct page * +i915_gem_object_get_page(struct drm_i915_gem_object *obj, unsignedlong n)
drivers/gpu/drm/i915/i915_gem.c +34
-0+{ +struct scatterlist *sg; +unsigned int offset; + +GEM_BUG_ON (!i915_gem_object_has_struct_page (obj)); + + sg =i915_gem_object_get_sg (obj, n, &offset); +return nth_page (sg_page (sg), offset); +} + +/* Like i915_gem_object_get_page(), but mark the returned page dirty */ +struct page * +i915_gem_object_get_dirty_page (struct drm_i915_gem_object *obj, +unsigned long n) +{ +struct page *page; + + page =i915_gem_object_get_page (obj, n); +if (!obj->dirty) +set_page_dirty (page); + +return page; +} + +dma_addr_t +i915_gem_object_get_dma_address (struct drm_i915_gem_object *obj, +unsigned long n) +{ +struct scatterlist *sg; +unsigned int offset; + + sg =i915_gem_object_get_sg (obj, n, &offset); +return sg_dma_address (sg) + (offset <<PAGE_SHIFT ); +}drivers/gpu/drm/i915/i915_gem_stolen.c +2
-1@@ -594,8 +594,8 @@ _i915_gem_object_create_stolen(struct drm_device *dev, if (obj->pages == NULL) goto cleanup; - obj->get_page.sg = obj->pages->sgl; - obj->get_page.last =0 ; + obj->get_page.sg_pos = obj->pages->sgl; + obj->get_page.sg_idx =0 ;i915_gem_object_pin_pages (obj); obj->stolen = stolen;drivers/gpu/drm/i915/i915_gem_userptr.c +2
-2@@ -526,8 +526,8 @@ __i915_gem_userptr_get_pages_worker(struct work_struct *_work)if (ret ==0 ) {list_add_tail (&obj->global_list, &to_i915 (dev)->mm.unbound_list); - obj->get_page.sg = obj->pages->sgl; - obj->get_page.last =0 ; + obj->get_page.sg_pos = obj->pages->sgl; + obj->get_page.sg_idx =0 ; pinned =0 ; }Quoted 1 line
}Regards,
Quoted 5 lines
Tvrtko
_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH 15/42 drm/i915: Use radixtree to jump start intel_partial_pages()
In reply to Chris WilsonWe can use the radixtree index of the obj->pages to find the start position of the desired partial range.
--- drivers/gpu/drm/i915/i915_gem_gtt.c | 38 +++++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 14 deletions(-)
drivers/gpu/drm/i915/i915_gem_gtt.c +24 -14
@@ -3562,35 +3562,45 @@ intel_partial_pages(const struct i915_ggtt_view *view, struct drm_i915_gem_object *obj) {struct sg_table *st; -struct scatterlist *sg; -struct sg_page_iter obj_sg_iter; +struct scatterlist *sg, *iter; +unsigned int count = view->params.partial.size; +unsigned int offset;int ret = -ENOMEM ; st =kmalloc (sizeof (*st),GFP_KERNEL );if (!st) goto err_st_alloc; - ret =sg_alloc_table (st, view->params.partial.size,GFP_KERNEL ); + ret =sg_alloc_table (st, count,GFP_KERNEL );if (ret) goto err_sg_alloc; + iter =i915_gem_object_get_sg (obj, + view->params.partial.offset, + &offset); + sg = st->sgl; st->nents =0 ; -for_each_sg_page (obj->pages->sgl, &obj_sg_iter, obj->pages->nents, - view->params.partial.offset) - { -if (st->nents >= view->params.partial.size) -break ; +do { +unsigned int len = +min (iter->length, (count - offset) <<PAGE_SHIFT ); -sg_set_page (sg,NULL ,PAGE_SIZE ,0 ); -sg_dma_address (sg) =sg_page_iter_dma_address (&obj_sg_iter); -sg_dma_len (sg) =PAGE_SIZE ; +sg_set_page (sg,NULL , len,0 ); +sg_dma_address (sg) = +sg_dma_address (iter) + (offset <<PAGE_SHIFT ); +sg_dma_len (sg) = len; - sg =sg_next (sg); st->nents++; - } + count -= len >>PAGE_SHIFT ; +if (count ==0 ) { +sg_mark_end (sg); +return st; + } -return st; + sg =__sg_next (sg); + iter =__sg_next (iter); + offset =0 ; + }while (1 ); err_sg_alloc:kfree (st);
Signature · 6 lines
2.9.3 _______________________________________________ Intel-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/intel-gfx
Re: [PATCH 15/42] drm/i915: Use radixtree to jump start intel_partial_pages()
Quoted 62 lines · Chris Wilson
We can use the radixtree index of the
obj->pagesto find the start position of the desired partial range.Signed-off-by Chris Wilson <[email protected]>--- drivers/gpu/drm/i915/i915_gem_gtt.c | 38 +++++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 14 deletions(-)drivers/gpu/drm/i915/i915_gem_gtt.c +19
-13@@ -3562,35 +3562,45 @@ intel_partial_pages(const struct i915_ggtt_view *view, struct drm_i915_gem_object *obj) {struct sg_table *st; -struct scatterlist *sg; -struct sg_page_iter obj_sg_iter; +struct scatterlist *sg, *iter; +unsigned int count = view->params.partial.size; +unsigned int offset;int ret = -ENOMEM ; st =kmalloc (sizeof (*st),GFP_KERNEL );if (!st) goto err_st_alloc; - ret =sg_alloc_table (st, view->params.partial.size,GFP_KERNEL ); + ret =sg_alloc_table (st, count,GFP_KERNEL );if (ret) goto err_sg_alloc; + iter =i915_gem_object_get_sg (obj, + view->params.partial.offset, + &offset); + sg = st->sgl; st->nents =0 ; -for_each_sg_page (obj->pages->sgl, &obj_sg_iter, obj->pages->nents, - view->params.partial.offset) - { -if (st->nents >= view->params.partial.size) -break ; + do { + unsigned int len = + min(iter->length, (count - offset) << PAGE_SHIFT); -sg_set_page (sg,NULL ,PAGE_SIZE ,0 ); -sg_dma_address (sg) =sg_page_iter_dma_address (&obj_sg_iter); -sg_dma_len (sg) =PAGE_SIZE ; + sg_set_page(sg, NULL, len, 0); + sg_dma_address(sg) = + sg_dma_address(iter) + (offset << PAGE_SHIFT); + sg_dma_len(sg) = len; - sg =sg_next (sg); st->nents++; - } + count -= len >> PAGE_SHIFT; + if (count == 0) { + sg_mark_end(sg); + return st;
drivers/gpu/drm/i915/i915_gem_gtt.c intel_partial_pages(const struct i915_ggtt_view *view, · line 3606
Quoted 10 lines
drivers/gpu/drm/i915/i915_gem_gtt.c +5
-1+ } -return st; + sg =__sg_next (sg); + iter =__sg_next (iter); + offset =0 ; + }while (1 ); err_sg_alloc:kfree (st);
_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH 16/42 drm/i915: Refactor object page API
In reply to Chris WilsonThe plan is to make obtaining the backing storage for the object avoid struct_mutex (i.e. use its own locking). The first step is to update the API so that normal users only call pin/unpin whilst working on the backing storage.
--- drivers/gpu/drm/i915/i915_cmd_parser.c | 2 +- drivers/gpu/drm/i915/i915_debugfs.c | 17 +-- drivers/gpu/drm/i915/i915_drv.h | 89 ++++++++---- drivers/gpu/drm/i915/i915_gem.c | 207 +++++++++++++-------------- drivers/gpu/drm/i915/i915_gem_batch_pool.c | 3 +- drivers/gpu/drm/i915/i915_gem_dmabuf.c | 14 +- drivers/gpu/drm/i915/i915_gem_execbuffer.c | 2 +- drivers/gpu/drm/i915/i915_gem_fence.c | 4 +- drivers/gpu/drm/i915/i915_gem_gtt.c | 10 +- drivers/gpu/drm/i915/i915_gem_internal.c | 19 +-- drivers/gpu/drm/i915/i915_gem_render_state.c | 2 +- drivers/gpu/drm/i915/i915_gem_shrinker.c | 10 +- drivers/gpu/drm/i915/i915_gem_stolen.c | 24 ++-- drivers/gpu/drm/i915/i915_gem_tiling.c | 8 +- drivers/gpu/drm/i915/i915_gem_userptr.c | 30 ++-- drivers/gpu/drm/i915/i915_gpu_error.c | 4 +- drivers/gpu/drm/i915/intel_lrc.c | 6 +- 17 files changed, 234 insertions(+), 217 deletions(-)
drivers/gpu/drm/i915/i915_cmd_parser.c +1 -1
@@ -1290,7 +1290,7 @@ int intel_engine_cmd_parser(struct intel_engine_cs *engine, }if (ret ==0 && needs_clflush_after) -drm_clflush_virt_range (shadow_batch_obj->mapping, batch_len); +drm_clflush_virt_range (shadow_batch_obj->mm.mapping, batch_len);i915_gem_object_unpin_map (shadow_batch_obj);return ret;
drivers/gpu/drm/i915/i915_debugfs.c +9 -8
@@ -112,7 +112,7 @@ static char get_global_flag(struct drm_i915_gem_object *obj)static char get_pin_mapped_flag (struct drm_i915_gem_object *obj) { -return obj->mapping ?'M' :' ' ; +return obj->mm.mapping ?'M' :' ' ; }static u64 i915_gem_obj_total_ggtt_size (struct drm_i915_gem_object *obj)
@@ -158,8 +158,8 @@ describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj)i915_gem_active_get_seqno (&obj->last_write, &obj->base.dev->struct_mutex),i915_cache_level_str (dev_priv, obj->cache_level), - obj->dirty ?" dirty" :"" , - obj->madv == I915_MADV_DONTNEED ? " purgeable" : ""); + obj->mm.dirty ?" dirty" :"" , + obj->mm.madv == I915_MADV_DONTNEED ? " purgeable" : "");if (obj->base.name)seq_printf (m," (name: %d)" , obj->base.name);list_for_each_entry (vma, &obj->vma_list, obj_link) {
@@ -411,12 +411,12 @@ static int i915_gem_object_info(struct seq_file *m, void *data) size += obj->base.size; ++count; -if (obj->madv ==I915_MADV_DONTNEED ) { +if (obj->mm.madv ==I915_MADV_DONTNEED ) { purgeable_size += obj->base.size; ++purgeable_count; } -if (obj->mapping) { +if (obj->mm.mapping) { mapped_count++; mapped_size += obj->base.size; }
@@ -433,12 +433,12 @@ static int i915_gem_object_info(struct seq_file *m, void *data) ++dpy_count; } -if (obj->madv ==I915_MADV_DONTNEED ) { +if (obj->mm.madv ==I915_MADV_DONTNEED ) { purgeable_size += obj->base.size; ++purgeable_count; } -if (obj->mapping) { +if (obj->mm.mapping) { mapped_count++; mapped_size += obj->base.size; }
@@ -2018,7 +2018,7 @@ static void i915_dump_lrc_obj(struct seq_file *m,seq_printf (m,"\tBound in GGTT at 0x%08x\n" ,i915_ggtt_offset (vma)); -if (i915_gem_object_get_pages (vma->obj)) { +if (i915_gem_object_pin_pages (vma->obj)) {seq_puts (m,"\tFailed to get pages for context object\n\n" );return ; }
@@ -2037,6 +2037,7 @@ static void i915_dump_lrc_obj(struct seq_file *m,kunmap_atomic (reg_state); } +i915_gem_object_unpin_pages (vma->obj);seq_putc (m,'\n' ); }
drivers/gpu/drm/i915/i915_drv.h +60 -29
124 lines not on this page · raw message
drivers/gpu/drm/i915/i915_gem.c +96 -111
578 lines not on this page · raw message
drivers/gpu/drm/i915/i915_gem_batch_pool.c +1 -2
@@ -131,11 +131,10 @@ i915_gem_batch_pool_get(struct i915_gem_batch_pool *pool,return obj; } - ret =i915_gem_object_get_pages (obj); + ret =i915_gem_object_pin_pages (obj);if (ret)return ERR_PTR (ret);list_move_tail (&obj->batch_pool_link, list); -i915_gem_object_pin_pages (obj);return obj; }
drivers/gpu/drm/i915/i915_gem_dmabuf.c +6 -8
@@ -48,12 +48,10 @@ static struct sg_table *i915_gem_map_dma_buf(struct dma_buf_attachment *attachmeif (ret) goto err; - ret =i915_gem_object_get_pages (obj); + ret =i915_gem_object_pin_pages (obj);if (ret) goto err_unlock; -i915_gem_object_pin_pages (obj); -/* Copy sg so that we make an independent mapping */ st =kmalloc (sizeof (struct sg_table ),GFP_KERNEL );if (st ==NULL ) {
@@ -61,13 +59,13 @@ static struct sg_table *i915_gem_map_dma_buf(struct dma_buf_attachment *attachme goto err_unpin; } - ret =sg_alloc_table (st, obj->pages->nents,GFP_KERNEL ); + ret =sg_alloc_table (st, obj->mm.pages->nents,GFP_KERNEL );if (ret) goto err_free; - src = obj->pages->sgl; + src = obj->mm.pages->sgl; dst = st->sgl; -for (i =0 ; i < obj->pages->nents; i++) { +for (i =0 ; i < obj->mm.pages->nents; i++) {sg_set_page (dst,sg_page (src), src->length,0 ); dst =sg_next (dst); src =sg_next (src);
@@ -299,14 +297,14 @@ static int i915_gem_object_get_pages_dmabuf(struct drm_i915_gem_object *obj)if (IS_ERR (sg))return PTR_ERR (sg); - obj->pages = sg; + obj->mm.pages = sg;return 0 ; }static void i915_gem_object_put_pages_dmabuf (struct drm_i915_gem_object *obj) {dma_buf_unmap_attachment (obj->base.import_attach, - obj->pages,DMA_BIDIRECTIONAL ); + obj->mm.pages,DMA_BIDIRECTIONAL ); }static const struct drm_i915_gem_object_ops i915_gem_object_dmabuf_ops = {
drivers/gpu/drm/i915/i915_gem_execbuffer.c +1 -1
@@ -1290,7 +1290,7 @@ void i915_vma_move_to_active(struct i915_vma *vma,GEM_BUG_ON (!drm_mm_node_allocated (&vma->node)); - obj->dirty =1 ;/* be paranoid */ + obj->mm.dirty = true;/* be paranoid */ /* Add a reference if we're newly entering the active list. * The order in which we add operations to the retirement queue is
drivers/gpu/drm/i915/i915_gem_fence.c +2 -2
@@ -648,7 +648,7 @@ i915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object *obj)return ; i =0 ; -for_each_sgt_page (page, sgt_iter, obj->pages) { +for_each_sgt_page (page, sgt_iter, obj->mm.pages) {char new_bit_17 =page_to_phys (page) >>17 ;if ((new_bit_17 &0x1 ) != (test_bit (i, obj->bit_17) !=0 )) {
@@ -687,7 +687,7 @@ i915_gem_object_save_bit_17_swizzle(struct drm_i915_gem_object *obj) i =0 ; -for_each_sgt_page (page, sgt_iter, obj->pages) { +for_each_sgt_page (page, sgt_iter, obj->mm.pages) {if (page_to_phys (page) & (1 <<17 ))__set_bit (i, obj->bit_17);else
drivers/gpu/drm/i915/i915_gem_gtt.c +5 -5
@@ -175,7 +175,7 @@ static int ppgtt_bind_vma(struct i915_vma *vma, {u32 pte_flags =0 ; - vma->pages = vma->obj->pages; + vma->pages = vma->obj->mm.pages;/* Currently applicable only to VLV */ if (vma->obj->gt_ro)
@@ -2295,7 +2295,7 @@ void i915_gem_suspend_gtt_mappings(struct drm_device *dev)int i915_gem_gtt_prepare_object (struct drm_i915_gem_object *obj) {if (!dma_map_sg (&obj->base.dev->pdev->dev, - obj->pages->sgl, obj->pages->nents, + obj->mm.pages->sgl, obj->mm.pages->nents,PCI_DMA_BIDIRECTIONAL ))return -ENOSPC ;
@@ -2685,7 +2685,7 @@ void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj) } } -dma_unmap_sg (kdev, obj->pages->sgl, obj->pages->nents, +dma_unmap_sg (kdev, obj->mm.pages->sgl, obj->mm.pages->nents,PCI_DMA_BIDIRECTIONAL ); }
@@ -3526,7 +3526,7 @@ intel_rotate_fb_obj_pages(const struct intel_rotation_info *rot_info,/* Populate source page list from the object. */ i =0 ; -for_each_sgt_dma (dma_addr, sgt_iter, obj->pages) +for_each_sgt_dma (dma_addr, sgt_iter, obj->mm.pages) page_addr_list[i++] = dma_addr;GEM_BUG_ON (i != n_pages);
@@ -3617,7 +3617,7 @@ i915_get_ggtt_vma_pages(struct i915_vma *vma)return 0 ;if (vma->ggtt_view.type ==I915_GGTT_VIEW_NORMAL ) - vma->pages = vma->obj->pages; + vma->pages = vma->obj->mm.pages;else if (vma->ggtt_view.type ==I915_GGTT_VIEW_ROTATED ) vma->pages =intel_rotate_fb_obj_pages (&vma->ggtt_view.params.rotated, vma->obj);
drivers/gpu/drm/i915/i915_gem_internal.c +10 -9
@@ -28,10 +28,11 @@static void internal_free_pages (struct sg_table *st) { -struct sg_page_iter sg_iter; +struct sgt_iter iter; +struct page *page; -for_each_sg_page (st->sgl, &sg_iter, st->nents,0 ) -put_page (sg_page_iter_page (&sg_iter)); +for_each_sgt_page (page, iter, st) +put_page (page);sg_free_table (st);kfree (st);
@@ -94,10 +95,10 @@ static int i915_gem_object_get_pages_internal(struct drm_i915_gem_object *obj) if (!swiotlb_nr_tbl())#endif sg_mark_end (sg); - obj->pages = st; + obj->mm.pages = st;if (i915_gem_gtt_prepare_object (obj)) { - obj->pages =NULL ; + obj->mm.pages =NULL ; goto err; }
@@ -106,7 +107,7 @@ static int i915_gem_object_get_pages_internal(struct drm_i915_gem_object *obj)* and the caller is expected to repopulate - the contents of this * object are only valid whilst active and pinned. */ - obj->madv =I915_MADV_DONTNEED ; + obj->mm.madv =I915_MADV_DONTNEED ;return 0 ; err:
@@ -117,10 +118,10 @@ err:static void i915_gem_object_put_pages_internal (struct drm_i915_gem_object *obj) { -internal_free_pages (obj->pages); +internal_free_pages (obj->mm.pages); - obj->dirty =0 ; - obj->madv =I915_MADV_WILLNEED ; + obj->mm.dirty = false; + obj->mm.madv =I915_MADV_WILLNEED ; }static const struct drm_i915_gem_object_ops i915_gem_object_internal_ops = {
drivers/gpu/drm/i915/i915_gem_render_state.c +1 -1
@@ -230,7 +230,7 @@ int i915_gem_render_state_emit(struct drm_i915_gem_request *req)return 0 ;/* Recreate the page after shrinking */ -if (!so->vma->obj->pages) +if (!so->vma->obj->mm.pages) so->batch_offset =-1 ; ret =i915_vma_pin (so->vma,0 ,0 ,PIN_GLOBAL |PIN_HIGH );
drivers/gpu/drm/i915/i915_gem_shrinker.c +5 -5
@@ -78,7 +78,7 @@ static bool can_release_pages(struct drm_i915_gem_object *obj)* to the GPU, simply unbinding from the GPU is not going to succeed * in releasing our pin count on the pages themselves. */ -if (obj->pages_pin_count > obj->bind_count) +if (obj->mm.pages_pin_count > obj->bind_count)return false;if (any_vma_pinned (obj))
@@ -88,7 +88,7 @@ static bool can_release_pages(struct drm_i915_gem_object *obj)* discard the contents (because the user has marked them as being * purgeable) or if we can move their contents out to swap. */ -return swap_available () || obj->madv ==I915_MADV_DONTNEED ; +return swap_available () || obj->mm.madv ==I915_MADV_DONTNEED ; }/**
@@ -175,11 +175,11 @@ i915_gem_shrink(struct drm_i915_private *dev_priv,list_move_tail (&obj->global_list, &still_in_list);if (flags &I915_SHRINK_PURGEABLE && - obj->madv !=I915_MADV_DONTNEED ) + obj->mm.madv !=I915_MADV_DONTNEED )continue ;if (flags &I915_SHRINK_VMAPS && - !is_vmalloc_addr (obj->mapping)) + !is_vmalloc_addr (obj->mm.mapping))continue ; if ((flags & I915_SHRINK_ACTIVE) == 0 &&
@@ -193,7 +193,7 @@ i915_gem_shrink(struct drm_i915_private *dev_priv,/* For the unbound phase, this should be a no-op! */ i915_gem_object_unbind (obj); -if (i915_gem_object_put_pages (obj) ==0 ) +if (__i915_gem_object_put_pages (obj) ==0 ) count += obj->base.size >>PAGE_SHIFT ;i915_gem_object_put (obj);
drivers/gpu/drm/i915/i915_gem_stolen.c +13 -11
@@ -554,16 +554,17 @@ static int i915_gem_object_get_pages_stolen(struct drm_i915_gem_object *obj)static void i915_gem_object_put_pages_stolen (struct drm_i915_gem_object *obj) {/* Should only be called during free */ -sg_free_table (obj->pages); -kfree (obj->pages); +sg_free_table (obj->mm.pages); +kfree (obj->mm.pages); } -static void i915_gem_object_release_stolen (struct drm_i915_gem_object *obj) {struct drm_i915_private *dev_priv =to_i915 (obj->base.dev); +__i915_gem_object_unpin_pages (obj); +if (obj->stolen) {i915_gem_stolen_remove_node (dev_priv, obj->stolen);kfree (obj->stolen);
@@ -589,15 +590,16 @@ _i915_gem_object_create_stolen(struct drm_device *dev,drm_gem_private_object_init (dev, &obj->base, stolen->size);i915_gem_object_init (obj, &i915_gem_object_stolen_ops); - obj->pages =i915_pages_create_for_stolen (dev, - stolen->start, stolen->size); -if (obj->pages ==NULL ) + obj->mm.pages =i915_pages_create_for_stolen (dev, + stolen->start, + stolen->size); +if (!obj->mm.pages) goto cleanup; - obj->get_page.sg_pos = obj->pages->sgl; - obj->get_page.sg_idx =0 ; + obj->mm.get_page.sg_pos = obj->mm.pages->sgl; + obj->mm.get_page.sg_idx =0 ; -i915_gem_object_pin_pages (obj); +__i915_gem_object_pin_pages (obj); obj->stolen = stolen; obj->base.read_domains =I915_GEM_DOMAIN_CPU |I915_GEM_DOMAIN_GTT ;
@@ -717,14 +719,14 @@ i915_gem_object_create_stolen_for_preallocated(struct drm_device *dev, goto err; } - vma->pages = obj->pages; + vma->pages = obj->mm.pages; vma->flags |=I915_VMA_GLOBAL_BIND ;__i915_vma_set_map_and_fenceable (vma);list_move_tail (&vma->vm_link, &ggtt->base.inactive_list); obj->bind_count++;list_add_tail (&obj->global_list, &dev_priv->mm.bound_list); -i915_gem_object_pin_pages (obj); +__i915_gem_object_pin_pages (obj);return obj;
drivers/gpu/drm/i915/i915_gem_tiling.c +4 -4
@@ -260,13 +260,13 @@ i915_gem_set_tiling(struct drm_device *dev, void *data,if (!err) {struct i915_vma *vma; -if (obj->pages && - obj->madv ==I915_MADV_WILLNEED && +if (obj->mm.pages && + obj->mm.madv ==I915_MADV_WILLNEED && dev_priv->quirks &QUIRK_PIN_SWIZZLED_PAGES ) {if (args->tiling_mode ==I915_TILING_NONE ) -i915_gem_object_unpin_pages (obj); +__i915_gem_object_unpin_pages (obj);if (!i915_gem_object_is_tiled (obj)) -i915_gem_object_pin_pages (obj); +__i915_gem_object_pin_pages (obj); }list_for_each_entry (vma, &obj->vma_list, obj_link) {
drivers/gpu/drm/i915/i915_gem_userptr.c +15 -15
@@ -73,10 +73,10 @@ static void cancel_userptr(struct work_struct *work)/* Cancel any active worker and force us to re-evaluate gup */ obj->userptr.work =NULL ; -if (obj->pages !=NULL ) { +if (obj->mm.pages) {/* We are inside a kthread context and can't be interrupted */ WARN_ON (i915_gem_object_unbind (obj)); -WARN_ON (i915_gem_object_put_pages (obj)); +WARN_ON (__i915_gem_object_put_pages (obj)); }i915_gem_object_put (obj);
@@ -432,15 +432,15 @@ __i915_gem_userptr_set_pages(struct drm_i915_gem_object *obj, {int ret; - ret =st_set_pages (&obj->pages, pvec, num_pages); + ret =st_set_pages (&obj->mm.pages, pvec, num_pages);if (ret)return ret; ret =i915_gem_gtt_prepare_object (obj);if (ret) { -sg_free_table (obj->pages); -kfree (obj->pages); - obj->pages =NULL ; +sg_free_table (obj->mm.pages); +kfree (obj->mm.pages); + obj->mm.pages =NULL ; }return ret;
@@ -526,8 +526,8 @@ __i915_gem_userptr_get_pages_worker(struct work_struct *_work)if (ret ==0 ) {list_add_tail (&obj->global_list, &to_i915 (dev)->mm.unbound_list); - obj->get_page.sg_pos = obj->pages->sgl; - obj->get_page.sg_idx =0 ; + obj->mm.get_page.sg_pos = obj->mm.pages->sgl; + obj->mm.get_page.sg_idx =0 ; pinned =0 ; } }
@@ -668,22 +668,22 @@ i915_gem_userptr_put_pages(struct drm_i915_gem_object *obj)BUG_ON (obj->userptr.work !=NULL );__i915_gem_userptr_set_active (obj, false); -if (obj->madv !=I915_MADV_WILLNEED ) - obj->dirty =0 ; +if (obj->mm.madv !=I915_MADV_WILLNEED ) + obj->mm.dirty = false;i915_gem_gtt_finish_object (obj); -for_each_sgt_page (page, sgt_iter, obj->pages) { -if (obj->dirty) +for_each_sgt_page (page, sgt_iter, obj->mm.pages) { +if (obj->mm.dirty)set_page_dirty (page);mark_page_accessed (page);put_page (page); } - obj->dirty =0 ; + obj->mm.dirty = false; -sg_free_table (obj->pages); -kfree (obj->pages); +sg_free_table (obj->mm.pages); +kfree (obj->mm.pages); } static void
drivers/gpu/drm/i915/i915_gpu_error.c +2 -2
@@ -881,8 +881,8 @@ static void capture_bo(struct drm_i915_error_buffer *err, err->write_domain = obj->base.write_domain; err->fence_reg = vma->fence ? vma->fence->id :-1 ; err->tiling =i915_gem_object_get_tiling (obj); - err->dirty = obj->dirty; - err->purgeable = obj->madv !=I915_MADV_WILLNEED ; + err->dirty = obj->mm.dirty; + err->purgeable = obj->mm.madv !=I915_MADV_WILLNEED ; err->userptr = obj->userptr.mm !=NULL ; err->cache_level = obj->cache_level; }
drivers/gpu/drm/i915/intel_lrc.c +3 -3
@@ -744,7 +744,7 @@ static int intel_lr_context_pin(struct i915_gem_context *ctx, ce->lrc_reg_state[CTX_RING_BUFFER_START +1 ] =i915_ggtt_offset (ce->ring->vma); - ce->state->obj->dirty = true; + ce->state->obj->mm.dirty = true;/* Invalidate GuC TLB. */ if (i915.enable_guc_submission) {
@@ -2042,7 +2042,7 @@ populate_lr_context(struct i915_gem_context *ctx,DRM_DEBUG_DRIVER ("Could not map object pages! (%d)\n" , ret);return ret; } - ctx_obj->dirty = true; + ctx_obj->mm.dirty = true;/* The second page of the context object contains some fields which must * be set up prior to the first execution. */
@@ -2179,7 +2179,7 @@ void intel_lr_context_resume(struct drm_i915_private *dev_priv) reg[CTX_RING_HEAD +1 ] =0 ; reg[CTX_RING_TAIL +1 ] =0 ; - ce->state->obj->dirty = true; + ce->state->obj->mm.dirty = true;i915_gem_object_unpin_map (ce->state->obj); ce->ring->head = ce->ring->tail =0 ;
Signature · 6 lines
2.9.3 _______________________________________________ Intel-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/intel-gfx
Re: [PATCH 16/42] drm/i915: Refactor object page API
Quoted 94 lines · Chris Wilson
The plan is to make obtaining the backing storage for the object avoid
struct_mutex(i.e. use its own locking). The first step is to update the API so that normal users only call pin/unpin whilst working on the backing storage.Signed-off-by Chris Wilson <[email protected]>--- drivers/gpu/drm/i915/i915_cmd_parser.c | 2 +- drivers/gpu/drm/i915/i915_debugfs.c | 17 +-- drivers/gpu/drm/i915/i915_drv.h | 89 ++++++++---- drivers/gpu/drm/i915/i915_gem.c | 207 +++++++++++++-------------- drivers/gpu/drm/i915/i915_gem_batch_pool.c | 3 +- drivers/gpu/drm/i915/i915_gem_dmabuf.c | 14 +- drivers/gpu/drm/i915/i915_gem_execbuffer.c | 2 +- drivers/gpu/drm/i915/i915_gem_fence.c | 4 +- drivers/gpu/drm/i915/i915_gem_gtt.c | 10 +- drivers/gpu/drm/i915/i915_gem_internal.c | 19 +-- drivers/gpu/drm/i915/i915_gem_render_state.c | 2 +- drivers/gpu/drm/i915/i915_gem_shrinker.c | 10 +- drivers/gpu/drm/i915/i915_gem_stolen.c | 24 ++-- drivers/gpu/drm/i915/i915_gem_tiling.c | 8 +- drivers/gpu/drm/i915/i915_gem_userptr.c | 30 ++-- drivers/gpu/drm/i915/i915_gpu_error.c | 4 +- drivers/gpu/drm/i915/intel_lrc.c | 6 +- 17 files changed, 234 insertions(+), 217 deletions(-)drivers/gpu/drm/i915/i915_cmd_parser.c +1
-1@@ -1290,7 +1290,7 @@ int intel_engine_cmd_parser(struct intel_engine_cs *engine, }if (ret ==0 && needs_clflush_after) -drm_clflush_virt_range (shadow_batch_obj->mapping, batch_len); +drm_clflush_virt_range (shadow_batch_obj->mm.mapping, batch_len);i915_gem_object_unpin_map (shadow_batch_obj);return ret;drivers/gpu/drm/i915/i915_debugfs.c +7
-7@@ -112,7 +112,7 @@ static char get_global_flag(struct drm_i915_gem_object *obj)static char get_pin_mapped_flag (struct drm_i915_gem_object *obj) { -return obj->mapping ?'M' :' ' ; +return obj->mm.mapping ?'M' :' ' ; }static u64 i915_gem_obj_total_ggtt_size (struct drm_i915_gem_object *obj)@@ -158,8 +158,8 @@ describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj)i915_gem_active_get_seqno (&obj->last_write, &obj->base.dev->struct_mutex),i915_cache_level_str (dev_priv, obj->cache_level), - obj->dirty ?" dirty" :"" , - obj->madv == I915_MADV_DONTNEED ? " purgeable" : ""); + obj->mm.dirty ?" dirty" :"" , + obj->mm.madv == I915_MADV_DONTNEED ? " purgeable" : "");if (obj->base.name)seq_printf (m," (name: %d)" , obj->base.name);list_for_each_entry (vma, &obj->vma_list, obj_link) {@@ -411,12 +411,12 @@ static int i915_gem_object_info(struct seq_file *m, void *data) size += obj->base.size; ++count; -if (obj->madv ==I915_MADV_DONTNEED ) { +if (obj->mm.madv ==I915_MADV_DONTNEED ) { purgeable_size += obj->base.size; ++purgeable_count; } -if (obj->mapping) { +if (obj->mm.mapping) { mapped_count++; mapped_size += obj->base.size; }@@ -433,12 +433,12 @@ static int i915_gem_object_info(struct seq_file *m, void *data) ++dpy_count; } -if (obj->madv ==I915_MADV_DONTNEED ) { +if (obj->mm.madv ==I915_MADV_DONTNEED ) { purgeable_size += obj->base.size; ++purgeable_count; } -if (obj->mapping) { +if (obj->mm.mapping) { mapped_count++; mapped_size += obj->base.size; }
drivers/gpu/drm/i915/i915_debugfs.c static void i915_dump_lrc_obj(struct seq_file *m, · line 2022
Quoted 6 lines
drivers/gpu/drm/i915/i915_debugfs.c +1
-1@@ -2018,7 +2018,7 @@ static void i915_dump_lrc_obj(struct seq_file *m,seq_printf (m,"\tBound in GGTT at 0x%08x\n" ,i915_ggtt_offset (vma)); -if (i915_gem_object_get_pages (vma->obj)) { +if (i915_gem_object_pin_pages (vma->obj)) {seq_puts (m,"\tFailed to get pages for context object\n\n" );
Error message needs updating to match the function call change.
Quoted 126 lines
drivers/gpu/drm/i915/i915_debugfs.c +1
-0return ; }@@ -2037,6 +2037,7 @@ static void i915_dump_lrc_obj(struct seq_file *m,kunmap_atomic (reg_state); } +i915_gem_object_unpin_pages (vma->obj);seq_putc (m,'\n' ); }drivers/gpu/drm/i915/i915_drv.h +55
-27@@ -2238,17 +2238,6 @@ struct drm_i915_gem_object {#define I915_BO_ACTIVE_REF (I915_BO_ACTIVE_SHIFT + I915_NUM_ENGINES)/** -* This is set if the object has been written to since last bound -* to the GTT -*/ - unsigned int dirty:1; - -/** -* Advice: are the backing pages purgeable? -*/ - unsigned int madv:2; - -/** * Whether the current gtt mapping needs to be mappable (and isn't just * mappable by accident). Track pin and fault separate for a more * accurate mappable working set. @@ -2276,16 +2265,31 @@ struct drm_i915_gem_object {unsigned int bind_count;unsigned int pin_display; -struct sg_table *pages; -int pages_pin_count; -struct i915_gem_object_page_iter { -struct scatterlist *sg_pos; -unsigned long sg_idx; +struct { +unsigned int pages_pin_count; + +struct sg_table *pages; +void *mapping; + +struct i915_gem_object_page_iter { +struct scatterlist *sg_pos; +unsigned long sg_idx; -struct radix_tree_root radix; -struct mutex lock; - } get_page; -void *mapping; +struct radix_tree_root radix; +struct mutex lock; + } get_page; + +/** +* Advice: are the backing pages purgeable? +*/ +unsigned int madv:2 ; + +/** +* This is set if the object has been written to since the +* pages were last acquired. +*/ +unsigned int dirty:1 ; + } mm;/** Breadcrumb of last rendering to the buffer. * There can only be one writer, but we allow for multiple readers. @@ -3160,13 +3164,10 @@ void i915_vma_close(struct i915_vma *vma);void i915_vma_destroy (struct i915_vma *vma);int i915_gem_object_unbind (struct drm_i915_gem_object *obj); -int i915_gem_object_put_pages (struct drm_i915_gem_object *obj);void i915_gem_release_all_mmaps (struct drm_i915_private *dev_priv);void i915_gem_release_mmap (struct drm_i915_gem_object *obj); -int __must_check i915_gem_object_get_pages (struct drm_i915_gem_object *obj); - -static inline int __sg_page_count (struct scatterlist *sg) +static inline int __sg_page_count (const struct scatterlist *sg) {return sg->length >>PAGE_SHIFT ; }@@ -3187,18 +3188,48 @@ dma_addr_t i915_gem_object_get_dma_address(struct drm_i915_gem_object *obj, unsigned long n); -static inline void i915_gem_object_pin_pages (struct drm_i915_gem_object *obj) +int __i915_gem_object_get_pages (struct drm_i915_gem_object *obj); + +static inline int __must_check +i915_gem_object_pin_pages (struct drm_i915_gem_object *obj) { -BUG_ON (obj->pages ==NULL ); - obj->pages_pin_count++; +lockdep_assert_held (&obj->base.dev->struct_mutex); +if (obj->mm.pages_pin_count++) +return 0 ; + +return __i915_gem_object_get_pages (obj); +} + +static inline void +__i915_gem_object_pin_pages (struct drm_i915_gem_object *obj) +{ +lockdep_assert_held (&obj->base.dev->struct_mutex); +GEM_BUG_ON (!obj->mm.pages); + obj->mm.pages_pin_count++; +} + +static inline bool +i915_gem_object_has_pinned_pages (struct drm_i915_gem_object *obj) +{ +return obj->mm.pages_pin_count; +} + +static inline void +__i915_gem_object_unpin_pages (struct drm_i915_gem_object *obj) +{ +lockdep_assert_held (&obj->base.dev->struct_mutex); +GEM_BUG_ON (!i915_gem_object_has_pinned_pages (obj));
drivers/gpu/drm/i915/i915_drv.h dma_addr_t · line 3229
Quoted 10 lines
drivers/gpu/drm/i915/i915_drv.h +3
-2+GEM_BUG_ON (!obj->mm.pages); + obj->mm.pages_pin_count--; }static inline void i915_gem_object_unpin_pages (struct drm_i915_gem_object *obj) { -BUG_ON (obj->pages_pin_count ==0 ); - obj->pages_pin_count--; +__i915_gem_object_unpin_pages (obj); }
Am I missing something or is the above identical to simply '#define i915_gem_object_unpin_pages __i915_gem_object_unpin_pages'? In which case, why bother with having two separate functions?
Quoted 1,102 lines
drivers/gpu/drm/i915/i915_drv.h +2
-0+int __i915_gem_object_put_pages (struct drm_i915_gem_object *obj); +enum i915_map_type {I915_MAP_WB =0 ,I915_MAP_WC ,drivers/gpu/drm/i915/i915_gem.c +96
-111@@ -216,7 +216,7 @@ i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj)sg_dma_address (sg) = obj->phys_handle->busaddr;sg_dma_len (sg) = obj->base.size; - obj->pages = st; + obj->mm.pages = st;return 0 ; }@@ -225,7 +225,7 @@ i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj) {int ret; -BUG_ON (obj->madv == __I915_MADV_PURGED); +GEM_BUG_ON (obj->mm.madv == __I915_MADV_PURGED); ret =i915_gem_object_set_to_cpu_domain (obj, true);if (WARN_ON (ret)) {@@ -235,10 +235,10 @@ i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj) obj->base.read_domains = obj->base.write_domain =I915_GEM_DOMAIN_CPU ; } -if (obj->madv ==I915_MADV_DONTNEED ) - obj->dirty =0 ; +if (obj->mm.madv ==I915_MADV_DONTNEED ) + obj->mm.dirty = false; -if (obj->dirty) { +if (obj->mm.dirty) {struct address_space *mapping = obj->base.filp->f_mapping;char *vaddr = obj->phys_handle->vaddr;int i;@@ -257,22 +257,23 @@ i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj)kunmap_atomic (dst);set_page_dirty (page); -if (obj->madv ==I915_MADV_WILLNEED ) +if (obj->mm.madv ==I915_MADV_WILLNEED )mark_page_accessed (page);put_page (page); vaddr +=PAGE_SIZE ; } - obj->dirty =0 ; + obj->mm.dirty = false; } -sg_free_table (obj->pages); -kfree (obj->pages); +sg_free_table (obj->mm.pages); +kfree (obj->mm.pages); }static void i915_gem_object_release_phys (struct drm_i915_gem_object *obj) {drm_pci_free (obj->base.dev, obj->phys_handle); +i915_gem_object_unpin_pages (obj); }static const struct drm_i915_gem_object_ops i915_gem_phys_ops = {@@ -506,7 +507,7 @@ i915_gem_object_attach_phys(struct drm_i915_gem_object *obj,return 0 ; } -if (obj->madv !=I915_MADV_WILLNEED ) +if (obj->mm.madv !=I915_MADV_WILLNEED )return -EFAULT ;if (obj->base.filp ==NULL )@@ -516,7 +517,7 @@ i915_gem_object_attach_phys(struct drm_i915_gem_object *obj,if (ret)return ret; - ret =i915_gem_object_put_pages (obj); + ret =__i915_gem_object_put_pages (obj);if (ret)return ret;@@ -528,7 +529,7 @@ i915_gem_object_attach_phys(struct drm_i915_gem_object *obj, obj->phys_handle = phys; obj->ops = &i915_gem_phys_ops; -return i915_gem_object_get_pages (obj); +return i915_gem_object_pin_pages (obj); } static int@@ -724,12 +725,10 @@ int i915_gem_obj_prepare_shmem_read(struct drm_i915_gem_object *obj,if (ret)return ret; - ret =i915_gem_object_get_pages (obj); + ret =i915_gem_object_pin_pages (obj);if (ret)return ret; -i915_gem_object_pin_pages (obj); -i915_gem_object_flush_gtt_write_domain (obj);/* If we're not in the cpu read domain, set ourself into the gtt @@ -777,12 +776,10 @@ int i915_gem_obj_prepare_shmem_write(struct drm_i915_gem_object *obj,if (ret)return ret; - ret =i915_gem_object_get_pages (obj); + ret =i915_gem_object_pin_pages (obj);if (ret)return ret; -i915_gem_object_pin_pages (obj); -i915_gem_object_flush_gtt_write_domain (obj);/* If we're not in the cpu write domain, set ourself into the @@ -812,7 +809,7 @@ int i915_gem_obj_prepare_shmem_write(struct drm_i915_gem_object *obj, obj->cache_dirty = true;intel_fb_obj_invalidate (obj,ORIGIN_CPU ); - obj->dirty =1 ; + obj->mm.dirty = true;/* return with the pages pinned */ return 0 ;@@ -949,13 +946,11 @@ i915_gem_gtt_pread(struct drm_device *dev,if (ret) goto out; - ret =i915_gem_object_get_pages (obj); + ret =i915_gem_object_pin_pages (obj);if (ret) {remove_mappable_node (&node); goto out; } - -i915_gem_object_pin_pages (obj); } ret =i915_gem_object_set_to_gtt_domain (obj, false);@@ -1062,7 +1057,7 @@ i915_gem_shmem_pread(struct drm_device *dev, offset = args->offset; remain = args->size; -for_each_sg_page (obj->pages->sgl, &sg_iter, obj->pages->nents, +for_each_sg_page (obj->mm.pages->sgl, &sg_iter, obj->mm.pages->nents, offset >>PAGE_SHIFT ) {struct page *page =sg_page_iter_page (&sg_iter);@@ -1254,13 +1249,11 @@ i915_gem_gtt_pwrite_fast(struct drm_i915_private *i915,if (ret) goto out; - ret =i915_gem_object_get_pages (obj); + ret =i915_gem_object_pin_pages (obj);if (ret) {remove_mappable_node (&node); goto out; } - -i915_gem_object_pin_pages (obj); } ret =i915_gem_object_set_to_gtt_domain (obj, true);@@ -1268,7 +1261,7 @@ i915_gem_gtt_pwrite_fast(struct drm_i915_private *i915, goto out_unpin;intel_fb_obj_invalidate (obj,ORIGIN_CPU ); - obj->dirty = true; + obj->mm.dirty = true; user_data =u64_to_user_ptr (args->data_ptr); offset = args->offset;@@ -1439,7 +1432,7 @@ i915_gem_shmem_pwrite(struct drm_device *dev, offset = args->offset; remain = args->size; -for_each_sg_page (obj->pages->sgl, &sg_iter, obj->pages->nents, +for_each_sg_page (obj->mm.pages->sgl, &sg_iter, obj->mm.pages->nents, offset >>PAGE_SHIFT ) {struct page *page =sg_page_iter_page (&sg_iter);int partial_cacheline_write;@@ -2228,7 +2221,7 @@ i915_gem_object_truncate(struct drm_i915_gem_object *obj)* backing pages, *now*. */ shmem_truncate_range (file_inode (obj->base.filp),0 , (loff_t)-1 ); - obj->madv = __I915_MADV_PURGED; + obj->mm.madv = __I915_MADV_PURGED; }/* Try to discard unwanted pages */ @@ -2237,7 +2230,7 @@ i915_gem_object_invalidate(struct drm_i915_gem_object *obj) {struct address_space *mapping; -switch (obj->madv) { +switch (obj->mm.madv) {case I915_MADV_DONTNEED :i915_gem_object_truncate (obj);case __I915_MADV_PURGED:@@ -2258,7 +2251,7 @@ i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)struct page *page;int ret; -BUG_ON (obj->madv == __I915_MADV_PURGED); +GEM_BUG_ON (obj->mm.madv == __I915_MADV_PURGED); ret =i915_gem_object_set_to_cpu_domain (obj, true);if (WARN_ON (ret)) {@@ -2274,22 +2267,22 @@ i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)if (i915_gem_object_needs_bit17_swizzle (obj))i915_gem_object_save_bit_17_swizzle (obj); -if (obj->madv ==I915_MADV_DONTNEED ) - obj->dirty =0 ; +if (obj->mm.madv ==I915_MADV_DONTNEED ) + obj->mm.dirty = false; -for_each_sgt_page (page, sgt_iter, obj->pages) { -if (obj->dirty) +for_each_sgt_page (page, sgt_iter, obj->mm.pages) { +if (obj->mm.dirty)set_page_dirty (page); -if (obj->madv ==I915_MADV_WILLNEED ) +if (obj->mm.madv ==I915_MADV_WILLNEED )mark_page_accessed (page);put_page (page); } - obj->dirty =0 ; + obj->mm.dirty = false; -sg_free_table (obj->pages); -kfree (obj->pages); +sg_free_table (obj->mm.pages); +kfree (obj->mm.pages); }static void __i915_gem_object_reset_page_iter (struct drm_i915_gem_object *obj)@@ -2297,21 +2290,20 @@ static void __i915_gem_object_reset_page_iter(struct drm_i915_gem_object *obj)struct radix_tree_iter iter;void **slot; -radix_tree_for_each_slot (slot, &obj->get_page.radix, &iter,0 ) -radix_tree_delete (&obj->get_page.radix, iter.index); +radix_tree_for_each_slot (slot, &obj->mm.get_page.radix, &iter,0 ) +radix_tree_delete (&obj->mm.get_page.radix, iter.index); } -int -i915_gem_object_put_pages (struct drm_i915_gem_object *obj) +int __i915_gem_object_put_pages (struct drm_i915_gem_object *obj) {const struct drm_i915_gem_object_ops *ops = obj->ops;lockdep_assert_held (&obj->base.dev->struct_mutex); -if (obj->pages ==NULL ) +if (!obj->mm.pages)return 0 ; -if (obj->pages_pin_count) +if (i915_gem_object_has_pinned_pages (obj))return -EBUSY ;GEM_BUG_ON (obj->bind_count);@@ -2321,22 +2313,22 @@ i915_gem_object_put_pages(struct drm_i915_gem_object *obj)* lists early. */ list_del (&obj->global_list); -if (obj->mapping) { +if (obj->mm.mapping) {void *ptr; - ptr =ptr_mask_bits (obj->mapping); + ptr =ptr_mask_bits (obj->mm.mapping);if (is_vmalloc_addr (ptr))vunmap (ptr);else kunmap (kmap_to_page (ptr)); - obj->mapping =NULL ; + obj->mm.mapping =NULL ; }__i915_gem_object_reset_page_iter (obj); ops->put_pages (obj); - obj->pages =NULL ; + obj->mm.pages =NULL ;i915_gem_object_invalidate (obj);@@ -2431,7 +2423,7 @@ i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj) if (!swiotlb_nr_tbl())#endif sg_mark_end (sg); - obj->pages = st; + obj->mm.pages = st; ret =i915_gem_gtt_prepare_object (obj);if (ret)@@ -2442,7 +2434,7 @@ i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)if (i915_gem_object_is_tiled (obj) && dev_priv->quirks &QUIRK_PIN_SWIZZLED_PAGES ) -i915_gem_object_pin_pages (obj); +__i915_gem_object_pin_pages (obj);return 0 ;@@ -2474,8 +2466,7 @@ err_pages:* either as a result of memory pressure (reaping pages under the shrinker) * or as the object is itself released. */ -int -i915_gem_object_get_pages (struct drm_i915_gem_object *obj) +int __i915_gem_object_get_pages (struct drm_i915_gem_object *obj) {struct drm_i915_private *dev_priv =to_i915 (obj->base.dev);const struct drm_i915_gem_object_ops *ops = obj->ops;@@ -2483,24 +2474,25 @@ i915_gem_object_get_pages(struct drm_i915_gem_object *obj)lockdep_assert_held (&obj->base.dev->struct_mutex); -if (obj->pages) +if (obj->mm.pages)return 0 ; -if (obj->madv !=I915_MADV_WILLNEED ) { +if (obj->mm.madv !=I915_MADV_WILLNEED ) {DRM_DEBUG ("Attempting to obtain a purgeable object\n" ); +__i915_gem_object_unpin_pages (obj);return -EFAULT ; } -BUG_ON (obj->pages_pin_count); - ret = ops->get_pages (obj); -if (ret) +if (ret) { +__i915_gem_object_unpin_pages (obj);return ret; + }list_add_tail (&obj->global_list, &dev_priv->mm.unbound_list); - obj->get_page.sg_pos = obj->pages->sgl; - obj->get_page.sg_idx =0 ; + obj->mm.get_page.sg_pos = obj->mm.pages->sgl; + obj->mm.get_page.sg_idx =0 ;return 0 ; }@@ -2510,7 +2502,7 @@ static void *i915_gem_object_map(const struct drm_i915_gem_object *obj, enum i915_map_type type) {unsigned long n_pages = obj->base.size >>PAGE_SHIFT ; -struct sg_table *sgt = obj->pages; +struct sg_table *sgt = obj->mm.pages;struct sgt_iter sgt_iter;struct page *page;struct page *stack_pages[32 ];@@ -2564,14 +2556,13 @@ void *i915_gem_object_pin_map(struct drm_i915_gem_object *obj,lockdep_assert_held (&obj->base.dev->struct_mutex);GEM_BUG_ON (!i915_gem_object_has_struct_page (obj)); - ret =i915_gem_object_get_pages (obj); + ret =i915_gem_object_pin_pages (obj);if (ret)return ERR_PTR (ret); -i915_gem_object_pin_pages (obj); - pinned = obj->pages_pin_count >1 ; + pinned = obj->mm.pages_pin_count >1 ; - ptr =ptr_unpack_bits (obj->mapping, has_type); + ptr =ptr_unpack_bits (obj->mm.mapping, has_type);if (ptr && has_type != type) {if (pinned) { ret = -EBUSY ;@@ -2583,7 +2574,7 @@ void *i915_gem_object_pin_map(struct drm_i915_gem_object *obj,else kunmap (kmap_to_page (ptr)); - ptr = obj->mapping =NULL ; + ptr = obj->mm.mapping =NULL ; }if (!ptr) {@@ -2593,7 +2584,7 @@ void *i915_gem_object_pin_map(struct drm_i915_gem_object *obj, goto err; } - obj->mapping =ptr_pack_bits (ptr, type); + obj->mm.mapping =ptr_pack_bits (ptr, type); }return ptr;@@ -3030,7 +3021,7 @@ int i915_vma_unbind(struct i915_vma *vma) goto destroy;GEM_BUG_ON (obj->bind_count ==0 ); -GEM_BUG_ON (!obj->pages); +GEM_BUG_ON (!obj->mm.pages);if (i915_vma_is_map_and_fenceable (vma)) {/* release the fence reg _after_ flushing */ @@ -3054,7 +3045,7 @@ int i915_vma_unbind(struct i915_vma *vma)drm_mm_remove_node (&vma->node);list_move_tail (&vma->vm_link, &vma->vm->unbound_list); -if (vma->pages != obj->pages) { +if (vma->pages != obj->mm.pages) {GEM_BUG_ON (!vma->pages);sg_free_table (vma->pages);kfree (vma->pages);@@ -3186,12 +3177,10 @@ i915_vma_insert(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)return -E2BIG ; } - ret =i915_gem_object_get_pages (obj); + ret =i915_gem_object_pin_pages (obj);if (ret)return ret; -i915_gem_object_pin_pages (obj); -if (flags &PIN_OFFSET_FIXED ) {u64 offset = flags &PIN_OFFSET_MASK ;if (offset & (alignment -1 ) || offset > end - size) {@@ -3270,7 +3259,7 @@ i915_gem_clflush_object(struct drm_i915_gem_object *obj,* to GPU, and we can ignore the cache flush because it'll happen * again at bind time. */ -if (obj->pages ==NULL ) +if (!obj->mm.pages)return false;/* @@ -3294,7 +3283,7 @@ i915_gem_clflush_object(struct drm_i915_gem_object *obj, }trace_i915_gem_object_clflush (obj); -drm_clflush_sg (obj->pages); +drm_clflush_sg (obj->mm.pages); obj->cache_dirty = false;return true;@@ -3408,7 +3397,7 @@ i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)* continue to assume that the obj remained out of the CPU cached * domain. */ - ret =i915_gem_object_get_pages (obj); + ret =i915_gem_object_pin_pages (obj);if (ret)return ret;@@ -3432,7 +3421,7 @@ i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)if (write) { obj->base.read_domains =I915_GEM_DOMAIN_GTT ; obj->base.write_domain =I915_GEM_DOMAIN_GTT ; - obj->dirty =1 ; + obj->mm.dirty = true; } trace_i915_gem_object_change_domain(obj,@@ -3441,6 +3430,7 @@ i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)/* And bump the LRU for this access */ i915_gem_object_bump_inactive_ggtt (obj); +i915_gem_object_unpin_pages (obj);return 0 ; }@@ -4210,23 +4200,23 @@ i915_gem_madvise_ioctl(struct drm_device *dev, void *data, goto unlock; } -if (obj->pages && +if (obj->mm.pages &&i915_gem_object_is_tiled (obj) && dev_priv->quirks &QUIRK_PIN_SWIZZLED_PAGES ) { -if (obj->madv ==I915_MADV_WILLNEED ) -i915_gem_object_unpin_pages (obj); +if (obj->mm.madv ==I915_MADV_WILLNEED ) +__i915_gem_object_unpin_pages (obj);if (args->madv ==I915_MADV_WILLNEED ) -i915_gem_object_pin_pages (obj); +__i915_gem_object_pin_pages (obj); } -if (obj->madv != __I915_MADV_PURGED) - obj->madv = args->madv; +if (obj->mm.madv != __I915_MADV_PURGED) + obj->mm.madv = args->madv;/* if the object is no longer attached, discard its backing storage */ -if (obj->madv ==I915_MADV_DONTNEED && obj->pages ==NULL ) +if (obj->mm.madv ==I915_MADV_DONTNEED && !obj->mm.pages)i915_gem_object_truncate (obj); - args->retained = obj->madv != __I915_MADV_PURGED; + args->retained = obj->mm.madv != __I915_MADV_PURGED;i915_gem_object_put (obj); unlock:@@ -4252,9 +4242,10 @@ void i915_gem_object_init(struct drm_i915_gem_object *obj, obj->ops = ops; obj->frontbuffer_ggtt_origin =ORIGIN_GTT ; - obj->madv =I915_MADV_WILLNEED ; -INIT_RADIX_TREE (&obj->get_page.radix,GFP_KERNEL ); -mutex_init (&obj->get_page.lock); + + obj->mm.madv =I915_MADV_WILLNEED ; +INIT_RADIX_TREE (&obj->mm.get_page.radix,GFP_KERNEL ); +mutex_init (&obj->mm.get_page.lock);i915_gem_info_add_obj (to_i915 (obj->base.dev), obj->base.size); }@@ -4331,7 +4322,7 @@ static bool discard_backing_storage(struct drm_i915_gem_object *obj)* back the contents from the GPU. */ -if (obj->madv !=I915_MADV_WILLNEED ) +if (obj->mm.madv !=I915_MADV_WILLNEED )return false;if (obj->base.filp ==NULL )@@ -4373,32 +4364,27 @@ void i915_gem_free_object(struct drm_gem_object *gem_obj) }GEM_BUG_ON (obj->bind_count); -/* Stolen objects don't hold a ref, but do hold pin count. Fix that up -* before progressing. */ -if (obj->stolen) -i915_gem_object_unpin_pages (obj); -WARN_ON (atomic_read (&obj->frontbuffer_bits)); -if (obj->pages && obj->madv ==I915_MADV_WILLNEED && +if (obj->mm.pages && obj->mm.madv ==I915_MADV_WILLNEED && dev_priv->quirks &QUIRK_PIN_SWIZZLED_PAGES &&i915_gem_object_is_tiled (obj)) -i915_gem_object_unpin_pages (obj); +__i915_gem_object_unpin_pages (obj); -if (WARN_ON (obj->pages_pin_count)) - obj->pages_pin_count =0 ; +if (obj->ops->release) + obj->ops->release (obj); + +if (WARN_ON (i915_gem_object_has_pinned_pages (obj))) + obj->mm.pages_pin_count =0 ;if (discard_backing_storage (obj)) - obj->madv =I915_MADV_DONTNEED ; -i915_gem_object_put_pages (obj); + obj->mm.madv =I915_MADV_DONTNEED ; +__i915_gem_object_put_pages (obj); -BUG_ON (obj->pages); +GEM_BUG_ON (obj->mm.pages);if (obj->base.import_attach)drm_prime_gem_destroy (&obj->base,NULL ); -if (obj->ops->release) - obj->ops->release (obj); -drm_gem_object_release (&obj->base);i915_gem_info_remove_obj (dev_priv, obj->base.size);@@ -4935,14 +4921,13 @@ i915_gem_object_create_from_data(struct drm_device *dev,if (ret) goto fail; - ret =i915_gem_object_get_pages (obj); + ret =i915_gem_object_pin_pages (obj);if (ret) goto fail; -i915_gem_object_pin_pages (obj); - sg = obj->pages; + sg = obj->mm.pages; bytes =sg_copy_from_buffer (sg->sgl, sg->nents, (void *)data, size); - obj->dirty =1 ;/* Backing store is now out of date */ + obj->mm.dirty = true;/* Backing store is now out of date */ i915_gem_object_unpin_pages (obj);if (WARN_ON (bytes != size)) {@@ -4962,7 +4947,7 @@ static struct scatterlist * __i915_gem_object_get_sg(struct drm_i915_gem_object *obj, unsigned long n, unsigned int *offset) { - struct scatterlist *sg = obj->pages->sgl; + struct scatterlist *sg = obj->mm.pages->sgl;int idx =0 ;while (idx +__sg_page_count (sg) <= n) {@@ -4980,11 +4965,11 @@ i915_gem_object_get_sg(struct drm_i915_gem_object *obj,unsigned long n, unsigned int *offset) { - struct i915_gem_object_page_iter *iter = &obj->get_page; + struct i915_gem_object_page_iter *iter = &obj->mm.get_page;struct scatterlist *sg;GEM_BUG_ON (n >= obj->base.size >>PAGE_SHIFT ); -GEM_BUG_ON (obj->pages_pin_count ==0 ); +GEM_BUG_ON (!i915_gem_object_has_pinned_pages (obj));if (n <READ_ONCE (iter->sg_idx)) goto lookup;@@ -5058,7 +5043,7 @@ i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj,struct page *page; page =i915_gem_object_get_page (obj, n); -if (!obj->dirty) +if (!obj->mm.dirty)set_page_dirty (page);return page;drivers/gpu/drm/i915/i915_gem_batch_pool.c +1
-2@@ -131,11 +131,10 @@ i915_gem_batch_pool_get(struct i915_gem_batch_pool *pool,return obj; } - ret =i915_gem_object_get_pages (obj); + ret =i915_gem_object_pin_pages (obj);if (ret)return ERR_PTR (ret);list_move_tail (&obj->batch_pool_link, list); -i915_gem_object_pin_pages (obj);return obj; }drivers/gpu/drm/i915/i915_gem_dmabuf.c +6
-8@@ -48,12 +48,10 @@ static struct sg_table *i915_gem_map_dma_buf(struct dma_buf_attachment *attachmeif (ret) goto err; - ret =i915_gem_object_get_pages (obj); + ret =i915_gem_object_pin_pages (obj);if (ret) goto err_unlock; -i915_gem_object_pin_pages (obj); -/* Copy sg so that we make an independent mapping */ st =kmalloc (sizeof (struct sg_table ),GFP_KERNEL );if (st ==NULL ) {@@ -61,13 +59,13 @@ static struct sg_table *i915_gem_map_dma_buf(struct dma_buf_attachment *attachme goto err_unpin; } - ret =sg_alloc_table (st, obj->pages->nents,GFP_KERNEL ); + ret =sg_alloc_table (st, obj->mm.pages->nents,GFP_KERNEL );if (ret) goto err_free; - src = obj->pages->sgl; + src = obj->mm.pages->sgl; dst = st->sgl; -for (i =0 ; i < obj->pages->nents; i++) { +for (i =0 ; i < obj->mm.pages->nents; i++) {sg_set_page (dst,sg_page (src), src->length,0 ); dst =sg_next (dst); src =sg_next (src);@@ -299,14 +297,14 @@ static int i915_gem_object_get_pages_dmabuf(struct drm_i915_gem_object *obj)if (IS_ERR (sg))return PTR_ERR (sg); - obj->pages = sg; + obj->mm.pages = sg;return 0 ; }static void i915_gem_object_put_pages_dmabuf (struct drm_i915_gem_object *obj) {dma_buf_unmap_attachment (obj->base.import_attach, - obj->pages,DMA_BIDIRECTIONAL ); + obj->mm.pages,DMA_BIDIRECTIONAL ); }static const struct drm_i915_gem_object_ops i915_gem_object_dmabuf_ops = {drivers/gpu/drm/i915/i915_gem_execbuffer.c +1
-1@@ -1290,7 +1290,7 @@ void i915_vma_move_to_active(struct i915_vma *vma,GEM_BUG_ON (!drm_mm_node_allocated (&vma->node)); - obj->dirty =1 ;/* be paranoid */ + obj->mm.dirty = true;/* be paranoid */ /* Add a reference if we're newly entering the active list. * The order in which we add operations to the retirement queue is drivers/gpu/drm/i915/i915_gem_fence.c +2
-2@@ -648,7 +648,7 @@ i915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object *obj)return ; i =0 ; -for_each_sgt_page (page, sgt_iter, obj->pages) { +for_each_sgt_page (page, sgt_iter, obj->mm.pages) {char new_bit_17 =page_to_phys (page) >>17 ;if ((new_bit_17 &0x1 ) != (test_bit (i, obj->bit_17) !=0 )) {@@ -687,7 +687,7 @@ i915_gem_object_save_bit_17_swizzle(struct drm_i915_gem_object *obj) i =0 ; -for_each_sgt_page (page, sgt_iter, obj->pages) { +for_each_sgt_page (page, sgt_iter, obj->mm.pages) {if (page_to_phys (page) & (1 <<17 ))__set_bit (i, obj->bit_17);else drivers/gpu/drm/i915/i915_gem_gtt.c +5
-5@@ -175,7 +175,7 @@ static int ppgtt_bind_vma(struct i915_vma *vma, {u32 pte_flags =0 ; - vma->pages = vma->obj->pages; + vma->pages = vma->obj->mm.pages;/* Currently applicable only to VLV */ if (vma->obj->gt_ro)@@ -2295,7 +2295,7 @@ void i915_gem_suspend_gtt_mappings(struct drm_device *dev)int i915_gem_gtt_prepare_object (struct drm_i915_gem_object *obj) {if (!dma_map_sg (&obj->base.dev->pdev->dev, - obj->pages->sgl, obj->pages->nents, + obj->mm.pages->sgl, obj->mm.pages->nents,PCI_DMA_BIDIRECTIONAL ))return -ENOSPC ;@@ -2685,7 +2685,7 @@ void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj) } } -dma_unmap_sg (kdev, obj->pages->sgl, obj->pages->nents, +dma_unmap_sg (kdev, obj->mm.pages->sgl, obj->mm.pages->nents,PCI_DMA_BIDIRECTIONAL ); }@@ -3526,7 +3526,7 @@ intel_rotate_fb_obj_pages(const struct intel_rotation_info *rot_info,/* Populate source page list from the object. */ i =0 ; -for_each_sgt_dma (dma_addr, sgt_iter, obj->pages) +for_each_sgt_dma (dma_addr, sgt_iter, obj->mm.pages) page_addr_list[i++] = dma_addr;GEM_BUG_ON (i != n_pages);@@ -3617,7 +3617,7 @@ i915_get_ggtt_vma_pages(struct i915_vma *vma)return 0 ;if (vma->ggtt_view.type ==I915_GGTT_VIEW_NORMAL ) - vma->pages = vma->obj->pages; + vma->pages = vma->obj->mm.pages;else if (vma->ggtt_view.type ==I915_GGTT_VIEW_ROTATED ) vma->pages =intel_rotate_fb_obj_pages (&vma->ggtt_view.params.rotated, vma->obj);drivers/gpu/drm/i915/i915_gem_internal.c +10
-9@@ -28,10 +28,11 @@static void internal_free_pages (struct sg_table *st) { -struct sg_page_iter sg_iter; +struct sgt_iter iter; +struct page *page; -for_each_sg_page (st->sgl, &sg_iter, st->nents,0 ) -put_page (sg_page_iter_page (&sg_iter)); +for_each_sgt_page (page, iter, st) +put_page (page);sg_free_table (st);kfree (st);@@ -94,10 +95,10 @@ static int i915_gem_object_get_pages_internal(struct drm_i915_gem_object *obj) if (!swiotlb_nr_tbl())#endif sg_mark_end (sg); - obj->pages = st; + obj->mm.pages = st;if (i915_gem_gtt_prepare_object (obj)) { - obj->pages =NULL ; + obj->mm.pages =NULL ; goto err; }@@ -106,7 +107,7 @@ static int i915_gem_object_get_pages_internal(struct drm_i915_gem_object *obj)* and the caller is expected to repopulate - the contents of this * object are only valid whilst active and pinned. */ - obj->madv =I915_MADV_DONTNEED ; + obj->mm.madv =I915_MADV_DONTNEED ;return 0 ; err:@@ -117,10 +118,10 @@ err:static void i915_gem_object_put_pages_internal (struct drm_i915_gem_object *obj) { -internal_free_pages (obj->pages); +internal_free_pages (obj->mm.pages); - obj->dirty =0 ; - obj->madv =I915_MADV_WILLNEED ; + obj->mm.dirty = false; + obj->mm.madv =I915_MADV_WILLNEED ; }static const struct drm_i915_gem_object_ops i915_gem_object_internal_ops = {drivers/gpu/drm/i915/i915_gem_render_state.c +1
-1@@ -230,7 +230,7 @@ int i915_gem_render_state_emit(struct drm_i915_gem_request *req)return 0 ;/* Recreate the page after shrinking */ -if (!so->vma->obj->pages) +if (!so->vma->obj->mm.pages) so->batch_offset =-1 ; ret =i915_vma_pin (so->vma,0 ,0 ,PIN_GLOBAL |PIN_HIGH );drivers/gpu/drm/i915/i915_gem_shrinker.c +5
-5@@ -78,7 +78,7 @@ static bool can_release_pages(struct drm_i915_gem_object *obj)* to the GPU, simply unbinding from the GPU is not going to succeed * in releasing our pin count on the pages themselves. */ -if (obj->pages_pin_count > obj->bind_count) +if (obj->mm.pages_pin_count > obj->bind_count)return false;if (any_vma_pinned (obj))@@ -88,7 +88,7 @@ static bool can_release_pages(struct drm_i915_gem_object *obj)* discard the contents (because the user has marked them as being * purgeable) or if we can move their contents out to swap. */ -return swap_available () || obj->madv ==I915_MADV_DONTNEED ; +return swap_available () || obj->mm.madv ==I915_MADV_DONTNEED ; }/** @@ -175,11 +175,11 @@ i915_gem_shrink(struct drm_i915_private *dev_priv,list_move_tail (&obj->global_list, &still_in_list);if (flags &I915_SHRINK_PURGEABLE && - obj->madv !=I915_MADV_DONTNEED ) + obj->mm.madv !=I915_MADV_DONTNEED )continue ;if (flags &I915_SHRINK_VMAPS && - !is_vmalloc_addr (obj->mapping)) + !is_vmalloc_addr (obj->mm.mapping))continue ; if ((flags & I915_SHRINK_ACTIVE) == 0 &&@@ -193,7 +193,7 @@ i915_gem_shrink(struct drm_i915_private *dev_priv,/* For the unbound phase, this should be a no-op! */ i915_gem_object_unbind (obj); -if (i915_gem_object_put_pages (obj) ==0 ) +if (__i915_gem_object_put_pages (obj) ==0 ) count += obj->base.size >>PAGE_SHIFT ;i915_gem_object_put (obj);drivers/gpu/drm/i915/i915_gem_stolen.c +13
-11@@ -554,16 +554,17 @@ static int i915_gem_object_get_pages_stolen(struct drm_i915_gem_object *obj)static void i915_gem_object_put_pages_stolen (struct drm_i915_gem_object *obj) {/* Should only be called during free */ -sg_free_table (obj->pages); -kfree (obj->pages); +sg_free_table (obj->mm.pages); +kfree (obj->mm.pages); } -static void i915_gem_object_release_stolen (struct drm_i915_gem_object *obj) {struct drm_i915_private *dev_priv =to_i915 (obj->base.dev); +__i915_gem_object_unpin_pages (obj); +if (obj->stolen) {i915_gem_stolen_remove_node (dev_priv, obj->stolen);kfree (obj->stolen);@@ -589,15 +590,16 @@ _i915_gem_object_create_stolen(struct drm_device *dev,drm_gem_private_object_init (dev, &obj->base, stolen->size);i915_gem_object_init (obj, &i915_gem_object_stolen_ops); - obj->pages =i915_pages_create_for_stolen (dev, - stolen->start, stolen->size); -if (obj->pages ==NULL ) + obj->mm.pages =i915_pages_create_for_stolen (dev, + stolen->start, + stolen->size); +if (!obj->mm.pages) goto cleanup; - obj->get_page.sg_pos = obj->pages->sgl; - obj->get_page.sg_idx =0 ; + obj->mm.get_page.sg_pos = obj->mm.pages->sgl; + obj->mm.get_page.sg_idx =0 ; -i915_gem_object_pin_pages (obj); +__i915_gem_object_pin_pages (obj); obj->stolen = stolen; obj->base.read_domains =I915_GEM_DOMAIN_CPU |I915_GEM_DOMAIN_GTT ;@@ -717,14 +719,14 @@ i915_gem_object_create_stolen_for_preallocated(struct drm_device *dev, goto err; } - vma->pages = obj->pages; + vma->pages = obj->mm.pages; vma->flags |=I915_VMA_GLOBAL_BIND ;__i915_vma_set_map_and_fenceable (vma);list_move_tail (&vma->vm_link, &ggtt->base.inactive_list); obj->bind_count++;list_add_tail (&obj->global_list, &dev_priv->mm.bound_list); -i915_gem_object_pin_pages (obj); +__i915_gem_object_pin_pages (obj);return obj;drivers/gpu/drm/i915/i915_gem_tiling.c +4
-4@@ -260,13 +260,13 @@ i915_gem_set_tiling(struct drm_device *dev, void *data,if (!err) {struct i915_vma *vma; -if (obj->pages && - obj->madv ==I915_MADV_WILLNEED && +if (obj->mm.pages && + obj->mm.madv ==I915_MADV_WILLNEED && dev_priv->quirks &QUIRK_PIN_SWIZZLED_PAGES ) {if (args->tiling_mode ==I915_TILING_NONE ) -i915_gem_object_unpin_pages (obj); +__i915_gem_object_unpin_pages (obj);if (!i915_gem_object_is_tiled (obj)) -i915_gem_object_pin_pages (obj); +__i915_gem_object_pin_pages (obj); }list_for_each_entry (vma, &obj->vma_list, obj_link) {drivers/gpu/drm/i915/i915_gem_userptr.c +15
-15@@ -73,10 +73,10 @@ static void cancel_userptr(struct work_struct *work)/* Cancel any active worker and force us to re-evaluate gup */ obj->userptr.work =NULL ; -if (obj->pages !=NULL ) { +if (obj->mm.pages) {/* We are inside a kthread context and can't be interrupted */ WARN_ON (i915_gem_object_unbind (obj)); -WARN_ON (i915_gem_object_put_pages (obj)); +WARN_ON (__i915_gem_object_put_pages (obj)); }i915_gem_object_put (obj);@@ -432,15 +432,15 @@ __i915_gem_userptr_set_pages(struct drm_i915_gem_object *obj, {int ret; - ret =st_set_pages (&obj->pages, pvec, num_pages); + ret =st_set_pages (&obj->mm.pages, pvec, num_pages);if (ret)return ret; ret =i915_gem_gtt_prepare_object (obj);if (ret) { -sg_free_table (obj->pages); -kfree (obj->pages); - obj->pages =NULL ; +sg_free_table (obj->mm.pages); +kfree (obj->mm.pages); + obj->mm.pages =NULL ; }return ret;@@ -526,8 +526,8 @@ __i915_gem_userptr_get_pages_worker(struct work_struct *_work)if (ret ==0 ) {list_add_tail (&obj->global_list, &to_i915 (dev)->mm.unbound_list); - obj->get_page.sg_pos = obj->pages->sgl; - obj->get_page.sg_idx =0 ; + obj->mm.get_page.sg_pos = obj->mm.pages->sgl; + obj->mm.get_page.sg_idx =0 ; pinned =0 ; } }@@ -668,22 +668,22 @@ i915_gem_userptr_put_pages(struct drm_i915_gem_object *obj)BUG_ON (obj->userptr.work !=NULL );__i915_gem_userptr_set_active (obj, false); -if (obj->madv !=I915_MADV_WILLNEED ) - obj->dirty =0 ; +if (obj->mm.madv !=I915_MADV_WILLNEED ) + obj->mm.dirty = false;i915_gem_gtt_finish_object (obj); -for_each_sgt_page (page, sgt_iter, obj->pages) { -if (obj->dirty) +for_each_sgt_page (page, sgt_iter, obj->mm.pages) { +if (obj->mm.dirty)set_page_dirty (page);mark_page_accessed (page);put_page (page); } - obj->dirty =0 ; + obj->mm.dirty = false; -sg_free_table (obj->pages); -kfree (obj->pages); +sg_free_table (obj->mm.pages); +kfree (obj->mm.pages); } static voiddrivers/gpu/drm/i915/i915_gpu_error.c +2
-2@@ -881,8 +881,8 @@ static void capture_bo(struct drm_i915_error_buffer *err, err->write_domain = obj->base.write_domain; err->fence_reg = vma->fence ? vma->fence->id :-1 ; err->tiling =i915_gem_object_get_tiling (obj); - err->dirty = obj->dirty; - err->purgeable = obj->madv !=I915_MADV_WILLNEED ; + err->dirty = obj->mm.dirty; + err->purgeable = obj->mm.madv !=I915_MADV_WILLNEED ; err->userptr = obj->userptr.mm !=NULL ; err->cache_level = obj->cache_level; }drivers/gpu/drm/i915/intel_lrc.c +2
-2@@ -744,7 +744,7 @@ static int intel_lr_context_pin(struct i915_gem_context *ctx, ce->lrc_reg_state[CTX_RING_BUFFER_START +1 ] =i915_ggtt_offset (ce->ring->vma); - ce->state->obj->dirty = true; + ce->state->obj->mm.dirty = true;/* Invalidate GuC TLB. */ if (i915.enable_guc_submission) {@@ -2042,7 +2042,7 @@ populate_lr_context(struct i915_gem_context *ctx,DRM_DEBUG_DRIVER ("Could not map object pages! (%d)\n" , ret);return ret; } - ctx_obj->dirty = true; + ctx_obj->mm.dirty = true;/* The second page of the context object contains some fields which must * be set up prior to the first execution. */
drivers/gpu/drm/i915/intel_lrc.c void intel_lr_context_resume(struct drm_i915_private *dev_priv) · line 2185
Quoted 8 lines
drivers/gpu/drm/i915/intel_lrc.c +1
-1@@ -2179,7 +2179,7 @@ void intel_lr_context_resume(struct drm_i915_private *dev_priv) reg[CTX_RING_HEAD +1 ] =0 ; reg[CTX_RING_TAIL +1 ] =0 ; - ce->state->obj->dirty = true; + ce->state->obj->mm.dirty = true;i915_gem_object_unpin_map (ce->state->obj); ce->ring->head = ce->ring->tail =0 ;
_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
Re: [PATCH 16/42] drm/i915: Refactor object page API
In reply to Chris WilsonQuoted 1,338 lines · Chris Wilson
The plan is to make obtaining the backing storage for the object avoid
struct_mutex(i.e. use its own locking). The first step is to update the API so that normal users only call pin/unpin whilst working on the backing storage.Signed-off-by Chris Wilson <[email protected]>--- drivers/gpu/drm/i915/i915_cmd_parser.c | 2 +- drivers/gpu/drm/i915/i915_debugfs.c | 17 +-- drivers/gpu/drm/i915/i915_drv.h | 89 ++++++++---- drivers/gpu/drm/i915/i915_gem.c | 207 +++++++++++++-------------- drivers/gpu/drm/i915/i915_gem_batch_pool.c | 3 +- drivers/gpu/drm/i915/i915_gem_dmabuf.c | 14 +- drivers/gpu/drm/i915/i915_gem_execbuffer.c | 2 +- drivers/gpu/drm/i915/i915_gem_fence.c | 4 +- drivers/gpu/drm/i915/i915_gem_gtt.c | 10 +- drivers/gpu/drm/i915/i915_gem_internal.c | 19 +-- drivers/gpu/drm/i915/i915_gem_render_state.c | 2 +- drivers/gpu/drm/i915/i915_gem_shrinker.c | 10 +- drivers/gpu/drm/i915/i915_gem_stolen.c | 24 ++-- drivers/gpu/drm/i915/i915_gem_tiling.c | 8 +- drivers/gpu/drm/i915/i915_gem_userptr.c | 30 ++-- drivers/gpu/drm/i915/i915_gpu_error.c | 4 +- drivers/gpu/drm/i915/intel_lrc.c | 6 +- 17 files changed, 234 insertions(+), 217 deletions(-)drivers/gpu/drm/i915/i915_cmd_parser.c +1
-1@@ -1290,7 +1290,7 @@ int intel_engine_cmd_parser(struct intel_engine_cs *engine, }if (ret ==0 && needs_clflush_after) -drm_clflush_virt_range (shadow_batch_obj->mapping, batch_len); +drm_clflush_virt_range (shadow_batch_obj->mm.mapping, batch_len);i915_gem_object_unpin_map (shadow_batch_obj);return ret;drivers/gpu/drm/i915/i915_debugfs.c +9
-8@@ -112,7 +112,7 @@ static char get_global_flag(struct drm_i915_gem_object *obj)static char get_pin_mapped_flag (struct drm_i915_gem_object *obj) { -return obj->mapping ?'M' :' ' ; +return obj->mm.mapping ?'M' :' ' ; }static u64 i915_gem_obj_total_ggtt_size (struct drm_i915_gem_object *obj)@@ -158,8 +158,8 @@ describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj)i915_gem_active_get_seqno (&obj->last_write, &obj->base.dev->struct_mutex),i915_cache_level_str (dev_priv, obj->cache_level), - obj->dirty ?" dirty" :"" , - obj->madv == I915_MADV_DONTNEED ? " purgeable" : ""); + obj->mm.dirty ?" dirty" :"" , + obj->mm.madv == I915_MADV_DONTNEED ? " purgeable" : "");if (obj->base.name)seq_printf (m," (name: %d)" , obj->base.name);list_for_each_entry (vma, &obj->vma_list, obj_link) {@@ -411,12 +411,12 @@ static int i915_gem_object_info(struct seq_file *m, void *data) size += obj->base.size; ++count; -if (obj->madv ==I915_MADV_DONTNEED ) { +if (obj->mm.madv ==I915_MADV_DONTNEED ) { purgeable_size += obj->base.size; ++purgeable_count; } -if (obj->mapping) { +if (obj->mm.mapping) { mapped_count++; mapped_size += obj->base.size; }@@ -433,12 +433,12 @@ static int i915_gem_object_info(struct seq_file *m, void *data) ++dpy_count; } -if (obj->madv ==I915_MADV_DONTNEED ) { +if (obj->mm.madv ==I915_MADV_DONTNEED ) { purgeable_size += obj->base.size; ++purgeable_count; } -if (obj->mapping) { +if (obj->mm.mapping) { mapped_count++; mapped_size += obj->base.size; }@@ -2018,7 +2018,7 @@ static void i915_dump_lrc_obj(struct seq_file *m,seq_printf (m,"\tBound in GGTT at 0x%08x\n" ,i915_ggtt_offset (vma)); -if (i915_gem_object_get_pages (vma->obj)) { +if (i915_gem_object_pin_pages (vma->obj)) {seq_puts (m,"\tFailed to get pages for context object\n\n" );return ; }@@ -2037,6 +2037,7 @@ static void i915_dump_lrc_obj(struct seq_file *m,kunmap_atomic (reg_state); } +i915_gem_object_unpin_pages (vma->obj);seq_putc (m,'\n' ); }drivers/gpu/drm/i915/i915_drv.h +60
-29@@ -2238,17 +2238,6 @@ struct drm_i915_gem_object {#define I915_BO_ACTIVE_REF (I915_BO_ACTIVE_SHIFT + I915_NUM_ENGINES)/** -* This is set if the object has been written to since last bound -* to the GTT -*/ - unsigned int dirty:1; - -/** -* Advice: are the backing pages purgeable? -*/ - unsigned int madv:2; - -/** * Whether the current gtt mapping needs to be mappable (and isn't just * mappable by accident). Track pin and fault separate for a more * accurate mappable working set. @@ -2276,16 +2265,31 @@ struct drm_i915_gem_object {unsigned int bind_count;unsigned int pin_display; -struct sg_table *pages; -int pages_pin_count; -struct i915_gem_object_page_iter { -struct scatterlist *sg_pos; -unsigned long sg_idx; +struct { +unsigned int pages_pin_count; + +struct sg_table *pages; +void *mapping; + +struct i915_gem_object_page_iter { +struct scatterlist *sg_pos; +unsigned long sg_idx; -struct radix_tree_root radix; -struct mutex lock; - } get_page; -void *mapping; +struct radix_tree_root radix; +struct mutex lock; + } get_page; + +/** +* Advice: are the backing pages purgeable? +*/ +unsigned int madv:2 ; + +/** +* This is set if the object has been written to since the +* pages were last acquired. +*/ +unsigned int dirty:1 ; + } mm;/** Breadcrumb of last rendering to the buffer. * There can only be one writer, but we allow for multiple readers. @@ -3160,13 +3164,10 @@ void i915_vma_close(struct i915_vma *vma);void i915_vma_destroy (struct i915_vma *vma);int i915_gem_object_unbind (struct drm_i915_gem_object *obj); -int i915_gem_object_put_pages (struct drm_i915_gem_object *obj);void i915_gem_release_all_mmaps (struct drm_i915_private *dev_priv);void i915_gem_release_mmap (struct drm_i915_gem_object *obj); -int __must_check i915_gem_object_get_pages (struct drm_i915_gem_object *obj); - -static inline int __sg_page_count (struct scatterlist *sg) +static inline int __sg_page_count (const struct scatterlist *sg) {return sg->length >>PAGE_SHIFT ; }@@ -3187,18 +3188,48 @@ dma_addr_t i915_gem_object_get_dma_address(struct drm_i915_gem_object *obj, unsigned long n); -static inline void i915_gem_object_pin_pages (struct drm_i915_gem_object *obj) +int __i915_gem_object_get_pages (struct drm_i915_gem_object *obj); + +static inline int __must_check +i915_gem_object_pin_pages (struct drm_i915_gem_object *obj) { -BUG_ON (obj->pages ==NULL ); - obj->pages_pin_count++; +lockdep_assert_held (&obj->base.dev->struct_mutex); +if (obj->mm.pages_pin_count++) +return 0 ; + +return __i915_gem_object_get_pages (obj); +} + +static inline void +__i915_gem_object_pin_pages (struct drm_i915_gem_object *obj) +{ +lockdep_assert_held (&obj->base.dev->struct_mutex); +GEM_BUG_ON (!obj->mm.pages); + obj->mm.pages_pin_count++; +} + +static inline bool +i915_gem_object_has_pinned_pages (struct drm_i915_gem_object *obj) +{ +return obj->mm.pages_pin_count; +} + +static inline void +__i915_gem_object_unpin_pages (struct drm_i915_gem_object *obj) +{ +lockdep_assert_held (&obj->base.dev->struct_mutex); +GEM_BUG_ON (!i915_gem_object_has_pinned_pages (obj)); +GEM_BUG_ON (!obj->mm.pages); + obj->mm.pages_pin_count--; }static inline void i915_gem_object_unpin_pages (struct drm_i915_gem_object *obj) { -BUG_ON (obj->pages_pin_count ==0 ); - obj->pages_pin_count--; +__i915_gem_object_unpin_pages (obj); } +int __i915_gem_object_put_pages (struct drm_i915_gem_object *obj); +enum i915_map_type {I915_MAP_WB =0 ,I915_MAP_WC ,drivers/gpu/drm/i915/i915_gem.c +96
-111@@ -216,7 +216,7 @@ i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj)sg_dma_address (sg) = obj->phys_handle->busaddr;sg_dma_len (sg) = obj->base.size; - obj->pages = st; + obj->mm.pages = st;return 0 ; }@@ -225,7 +225,7 @@ i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj) {int ret; -BUG_ON (obj->madv == __I915_MADV_PURGED); +GEM_BUG_ON (obj->mm.madv == __I915_MADV_PURGED); ret =i915_gem_object_set_to_cpu_domain (obj, true);if (WARN_ON (ret)) {@@ -235,10 +235,10 @@ i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj) obj->base.read_domains = obj->base.write_domain =I915_GEM_DOMAIN_CPU ; } -if (obj->madv ==I915_MADV_DONTNEED ) - obj->dirty =0 ; +if (obj->mm.madv ==I915_MADV_DONTNEED ) + obj->mm.dirty = false; -if (obj->dirty) { +if (obj->mm.dirty) {struct address_space *mapping = obj->base.filp->f_mapping;char *vaddr = obj->phys_handle->vaddr;int i;@@ -257,22 +257,23 @@ i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj)kunmap_atomic (dst);set_page_dirty (page); -if (obj->madv ==I915_MADV_WILLNEED ) +if (obj->mm.madv ==I915_MADV_WILLNEED )mark_page_accessed (page);put_page (page); vaddr +=PAGE_SIZE ; } - obj->dirty =0 ; + obj->mm.dirty = false; } -sg_free_table (obj->pages); -kfree (obj->pages); +sg_free_table (obj->mm.pages); +kfree (obj->mm.pages); }static void i915_gem_object_release_phys (struct drm_i915_gem_object *obj) {drm_pci_free (obj->base.dev, obj->phys_handle); +i915_gem_object_unpin_pages (obj); }static const struct drm_i915_gem_object_ops i915_gem_phys_ops = {@@ -506,7 +507,7 @@ i915_gem_object_attach_phys(struct drm_i915_gem_object *obj,return 0 ; } -if (obj->madv !=I915_MADV_WILLNEED ) +if (obj->mm.madv !=I915_MADV_WILLNEED )return -EFAULT ;if (obj->base.filp ==NULL )@@ -516,7 +517,7 @@ i915_gem_object_attach_phys(struct drm_i915_gem_object *obj,if (ret)return ret; - ret =i915_gem_object_put_pages (obj); + ret =__i915_gem_object_put_pages (obj);if (ret)return ret;@@ -528,7 +529,7 @@ i915_gem_object_attach_phys(struct drm_i915_gem_object *obj, obj->phys_handle = phys; obj->ops = &i915_gem_phys_ops; -return i915_gem_object_get_pages (obj); +return i915_gem_object_pin_pages (obj); } static int@@ -724,12 +725,10 @@ int i915_gem_obj_prepare_shmem_read(struct drm_i915_gem_object *obj,if (ret)return ret; - ret =i915_gem_object_get_pages (obj); + ret =i915_gem_object_pin_pages (obj);if (ret)return ret; -i915_gem_object_pin_pages (obj); -i915_gem_object_flush_gtt_write_domain (obj);/* If we're not in the cpu read domain, set ourself into the gtt @@ -777,12 +776,10 @@ int i915_gem_obj_prepare_shmem_write(struct drm_i915_gem_object *obj,if (ret)return ret; - ret =i915_gem_object_get_pages (obj); + ret =i915_gem_object_pin_pages (obj);if (ret)return ret; -i915_gem_object_pin_pages (obj); -i915_gem_object_flush_gtt_write_domain (obj);/* If we're not in the cpu write domain, set ourself into the @@ -812,7 +809,7 @@ int i915_gem_obj_prepare_shmem_write(struct drm_i915_gem_object *obj, obj->cache_dirty = true;intel_fb_obj_invalidate (obj,ORIGIN_CPU ); - obj->dirty =1 ; + obj->mm.dirty = true;/* return with the pages pinned */ return 0 ;@@ -949,13 +946,11 @@ i915_gem_gtt_pread(struct drm_device *dev,if (ret) goto out; - ret =i915_gem_object_get_pages (obj); + ret =i915_gem_object_pin_pages (obj);if (ret) {remove_mappable_node (&node); goto out; } - -i915_gem_object_pin_pages (obj); } ret =i915_gem_object_set_to_gtt_domain (obj, false);@@ -1062,7 +1057,7 @@ i915_gem_shmem_pread(struct drm_device *dev, offset = args->offset; remain = args->size; -for_each_sg_page (obj->pages->sgl, &sg_iter, obj->pages->nents, +for_each_sg_page (obj->mm.pages->sgl, &sg_iter, obj->mm.pages->nents, offset >>PAGE_SHIFT ) {struct page *page =sg_page_iter_page (&sg_iter);@@ -1254,13 +1249,11 @@ i915_gem_gtt_pwrite_fast(struct drm_i915_private *i915,if (ret) goto out; - ret =i915_gem_object_get_pages (obj); + ret =i915_gem_object_pin_pages (obj);if (ret) {remove_mappable_node (&node); goto out; } - -i915_gem_object_pin_pages (obj); } ret =i915_gem_object_set_to_gtt_domain (obj, true);@@ -1268,7 +1261,7 @@ i915_gem_gtt_pwrite_fast(struct drm_i915_private *i915, goto out_unpin;intel_fb_obj_invalidate (obj,ORIGIN_CPU ); - obj->dirty = true; + obj->mm.dirty = true; user_data =u64_to_user_ptr (args->data_ptr); offset = args->offset;@@ -1439,7 +1432,7 @@ i915_gem_shmem_pwrite(struct drm_device *dev, offset = args->offset; remain = args->size; -for_each_sg_page (obj->pages->sgl, &sg_iter, obj->pages->nents, +for_each_sg_page (obj->mm.pages->sgl, &sg_iter, obj->mm.pages->nents, offset >>PAGE_SHIFT ) {struct page *page =sg_page_iter_page (&sg_iter);int partial_cacheline_write;@@ -2228,7 +2221,7 @@ i915_gem_object_truncate(struct drm_i915_gem_object *obj)* backing pages, *now*. */ shmem_truncate_range (file_inode (obj->base.filp),0 , (loff_t)-1 ); - obj->madv = __I915_MADV_PURGED; + obj->mm.madv = __I915_MADV_PURGED; }/* Try to discard unwanted pages */ @@ -2237,7 +2230,7 @@ i915_gem_object_invalidate(struct drm_i915_gem_object *obj) {struct address_space *mapping; -switch (obj->madv) { +switch (obj->mm.madv) {case I915_MADV_DONTNEED :i915_gem_object_truncate (obj);case __I915_MADV_PURGED:@@ -2258,7 +2251,7 @@ i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)struct page *page;int ret; -BUG_ON (obj->madv == __I915_MADV_PURGED); +GEM_BUG_ON (obj->mm.madv == __I915_MADV_PURGED); ret =i915_gem_object_set_to_cpu_domain (obj, true);if (WARN_ON (ret)) {@@ -2274,22 +2267,22 @@ i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)if (i915_gem_object_needs_bit17_swizzle (obj))i915_gem_object_save_bit_17_swizzle (obj); -if (obj->madv ==I915_MADV_DONTNEED ) - obj->dirty =0 ; +if (obj->mm.madv ==I915_MADV_DONTNEED ) + obj->mm.dirty = false; -for_each_sgt_page (page, sgt_iter, obj->pages) { -if (obj->dirty) +for_each_sgt_page (page, sgt_iter, obj->mm.pages) { +if (obj->mm.dirty)set_page_dirty (page); -if (obj->madv ==I915_MADV_WILLNEED ) +if (obj->mm.madv ==I915_MADV_WILLNEED )mark_page_accessed (page);put_page (page); } - obj->dirty =0 ; + obj->mm.dirty = false; -sg_free_table (obj->pages); -kfree (obj->pages); +sg_free_table (obj->mm.pages); +kfree (obj->mm.pages); }static void __i915_gem_object_reset_page_iter (struct drm_i915_gem_object *obj)@@ -2297,21 +2290,20 @@ static void __i915_gem_object_reset_page_iter(struct drm_i915_gem_object *obj)struct radix_tree_iter iter;void **slot; -radix_tree_for_each_slot (slot, &obj->get_page.radix, &iter,0 ) -radix_tree_delete (&obj->get_page.radix, iter.index); +radix_tree_for_each_slot (slot, &obj->mm.get_page.radix, &iter,0 ) +radix_tree_delete (&obj->mm.get_page.radix, iter.index); } -int -i915_gem_object_put_pages (struct drm_i915_gem_object *obj) +int __i915_gem_object_put_pages (struct drm_i915_gem_object *obj) {const struct drm_i915_gem_object_ops *ops = obj->ops;lockdep_assert_held (&obj->base.dev->struct_mutex); -if (obj->pages ==NULL ) +if (!obj->mm.pages)return 0 ; -if (obj->pages_pin_count) +if (i915_gem_object_has_pinned_pages (obj))return -EBUSY ;GEM_BUG_ON (obj->bind_count);@@ -2321,22 +2313,22 @@ i915_gem_object_put_pages(struct drm_i915_gem_object *obj)* lists early. */ list_del (&obj->global_list); -if (obj->mapping) { +if (obj->mm.mapping) {void *ptr; - ptr =ptr_mask_bits (obj->mapping); + ptr =ptr_mask_bits (obj->mm.mapping);if (is_vmalloc_addr (ptr))vunmap (ptr);else kunmap (kmap_to_page (ptr)); - obj->mapping =NULL ; + obj->mm.mapping =NULL ; }__i915_gem_object_reset_page_iter (obj); ops->put_pages (obj); - obj->pages =NULL ; + obj->mm.pages =NULL ;i915_gem_object_invalidate (obj);@@ -2431,7 +2423,7 @@ i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj) if (!swiotlb_nr_tbl())#endif sg_mark_end (sg); - obj->pages = st; + obj->mm.pages = st; ret =i915_gem_gtt_prepare_object (obj);if (ret)@@ -2442,7 +2434,7 @@ i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)if (i915_gem_object_is_tiled (obj) && dev_priv->quirks &QUIRK_PIN_SWIZZLED_PAGES ) -i915_gem_object_pin_pages (obj); +__i915_gem_object_pin_pages (obj);return 0 ;@@ -2474,8 +2466,7 @@ err_pages:* either as a result of memory pressure (reaping pages under the shrinker) * or as the object is itself released. */ -int -i915_gem_object_get_pages (struct drm_i915_gem_object *obj) +int __i915_gem_object_get_pages (struct drm_i915_gem_object *obj) {struct drm_i915_private *dev_priv =to_i915 (obj->base.dev);const struct drm_i915_gem_object_ops *ops = obj->ops;@@ -2483,24 +2474,25 @@ i915_gem_object_get_pages(struct drm_i915_gem_object *obj)lockdep_assert_held (&obj->base.dev->struct_mutex); -if (obj->pages) +if (obj->mm.pages)return 0 ; -if (obj->madv !=I915_MADV_WILLNEED ) { +if (obj->mm.madv !=I915_MADV_WILLNEED ) {DRM_DEBUG ("Attempting to obtain a purgeable object\n" ); +__i915_gem_object_unpin_pages (obj);return -EFAULT ; } -BUG_ON (obj->pages_pin_count); - ret = ops->get_pages (obj); -if (ret) +if (ret) { +__i915_gem_object_unpin_pages (obj);return ret; + }list_add_tail (&obj->global_list, &dev_priv->mm.unbound_list); - obj->get_page.sg_pos = obj->pages->sgl; - obj->get_page.sg_idx =0 ; + obj->mm.get_page.sg_pos = obj->mm.pages->sgl; + obj->mm.get_page.sg_idx =0 ;return 0 ; }@@ -2510,7 +2502,7 @@ static void *i915_gem_object_map(const struct drm_i915_gem_object *obj, enum i915_map_type type) {unsigned long n_pages = obj->base.size >>PAGE_SHIFT ; -struct sg_table *sgt = obj->pages; +struct sg_table *sgt = obj->mm.pages;struct sgt_iter sgt_iter;struct page *page;struct page *stack_pages[32 ];@@ -2564,14 +2556,13 @@ void *i915_gem_object_pin_map(struct drm_i915_gem_object *obj,lockdep_assert_held (&obj->base.dev->struct_mutex);GEM_BUG_ON (!i915_gem_object_has_struct_page (obj)); - ret =i915_gem_object_get_pages (obj); + ret =i915_gem_object_pin_pages (obj);if (ret)return ERR_PTR (ret); -i915_gem_object_pin_pages (obj); - pinned = obj->pages_pin_count >1 ; + pinned = obj->mm.pages_pin_count >1 ; - ptr =ptr_unpack_bits (obj->mapping, has_type); + ptr =ptr_unpack_bits (obj->mm.mapping, has_type);if (ptr && has_type != type) {if (pinned) { ret = -EBUSY ;@@ -2583,7 +2574,7 @@ void *i915_gem_object_pin_map(struct drm_i915_gem_object *obj,else kunmap (kmap_to_page (ptr)); - ptr = obj->mapping =NULL ; + ptr = obj->mm.mapping =NULL ; }if (!ptr) {@@ -2593,7 +2584,7 @@ void *i915_gem_object_pin_map(struct drm_i915_gem_object *obj, goto err; } - obj->mapping =ptr_pack_bits (ptr, type); + obj->mm.mapping =ptr_pack_bits (ptr, type); }return ptr;@@ -3030,7 +3021,7 @@ int i915_vma_unbind(struct i915_vma *vma) goto destroy;GEM_BUG_ON (obj->bind_count ==0 ); -GEM_BUG_ON (!obj->pages); +GEM_BUG_ON (!obj->mm.pages);if (i915_vma_is_map_and_fenceable (vma)) {/* release the fence reg _after_ flushing */ @@ -3054,7 +3045,7 @@ int i915_vma_unbind(struct i915_vma *vma)drm_mm_remove_node (&vma->node);list_move_tail (&vma->vm_link, &vma->vm->unbound_list); -if (vma->pages != obj->pages) { +if (vma->pages != obj->mm.pages) {GEM_BUG_ON (!vma->pages);sg_free_table (vma->pages);kfree (vma->pages);@@ -3186,12 +3177,10 @@ i915_vma_insert(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)return -E2BIG ; } - ret =i915_gem_object_get_pages (obj); + ret =i915_gem_object_pin_pages (obj);if (ret)return ret; -i915_gem_object_pin_pages (obj); -if (flags &PIN_OFFSET_FIXED ) {u64 offset = flags &PIN_OFFSET_MASK ;if (offset & (alignment -1 ) || offset > end - size) {@@ -3270,7 +3259,7 @@ i915_gem_clflush_object(struct drm_i915_gem_object *obj,* to GPU, and we can ignore the cache flush because it'll happen * again at bind time. */ -if (obj->pages ==NULL ) +if (!obj->mm.pages)return false;/* @@ -3294,7 +3283,7 @@ i915_gem_clflush_object(struct drm_i915_gem_object *obj, }trace_i915_gem_object_clflush (obj); -drm_clflush_sg (obj->pages); +drm_clflush_sg (obj->mm.pages); obj->cache_dirty = false;return true;@@ -3408,7 +3397,7 @@ i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)* continue to assume that the obj remained out of the CPU cached * domain. */ - ret =i915_gem_object_get_pages (obj); + ret =i915_gem_object_pin_pages (obj);if (ret)return ret;@@ -3432,7 +3421,7 @@ i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)if (write) { obj->base.read_domains =I915_GEM_DOMAIN_GTT ; obj->base.write_domain =I915_GEM_DOMAIN_GTT ; - obj->dirty =1 ; + obj->mm.dirty = true; } trace_i915_gem_object_change_domain(obj,@@ -3441,6 +3430,7 @@ i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)/* And bump the LRU for this access */ i915_gem_object_bump_inactive_ggtt (obj); +i915_gem_object_unpin_pages (obj);return 0 ; }@@ -4210,23 +4200,23 @@ i915_gem_madvise_ioctl(struct drm_device *dev, void *data, goto unlock; } -if (obj->pages && +if (obj->mm.pages &&i915_gem_object_is_tiled (obj) && dev_priv->quirks &QUIRK_PIN_SWIZZLED_PAGES ) { -if (obj->madv ==I915_MADV_WILLNEED ) -i915_gem_object_unpin_pages (obj); +if (obj->mm.madv ==I915_MADV_WILLNEED ) +__i915_gem_object_unpin_pages (obj);if (args->madv ==I915_MADV_WILLNEED ) -i915_gem_object_pin_pages (obj); +__i915_gem_object_pin_pages (obj); } -if (obj->madv != __I915_MADV_PURGED) - obj->madv = args->madv; +if (obj->mm.madv != __I915_MADV_PURGED) + obj->mm.madv = args->madv;/* if the object is no longer attached, discard its backing storage */ -if (obj->madv ==I915_MADV_DONTNEED && obj->pages ==NULL ) +if (obj->mm.madv ==I915_MADV_DONTNEED && !obj->mm.pages)i915_gem_object_truncate (obj); - args->retained = obj->madv != __I915_MADV_PURGED; + args->retained = obj->mm.madv != __I915_MADV_PURGED;i915_gem_object_put (obj); unlock:@@ -4252,9 +4242,10 @@ void i915_gem_object_init(struct drm_i915_gem_object *obj, obj->ops = ops; obj->frontbuffer_ggtt_origin =ORIGIN_GTT ; - obj->madv =I915_MADV_WILLNEED ; -INIT_RADIX_TREE (&obj->get_page.radix,GFP_KERNEL ); -mutex_init (&obj->get_page.lock); + + obj->mm.madv =I915_MADV_WILLNEED ; +INIT_RADIX_TREE (&obj->mm.get_page.radix,GFP_KERNEL ); +mutex_init (&obj->mm.get_page.lock);i915_gem_info_add_obj (to_i915 (obj->base.dev), obj->base.size); }@@ -4331,7 +4322,7 @@ static bool discard_backing_storage(struct drm_i915_gem_object *obj)* back the contents from the GPU. */ -if (obj->madv !=I915_MADV_WILLNEED ) +if (obj->mm.madv !=I915_MADV_WILLNEED )return false;if (obj->base.filp ==NULL )@@ -4373,32 +4364,27 @@ void i915_gem_free_object(struct drm_gem_object *gem_obj) }GEM_BUG_ON (obj->bind_count); -/* Stolen objects don't hold a ref, but do hold pin count. Fix that up -* before progressing. */ -if (obj->stolen) -i915_gem_object_unpin_pages (obj); -WARN_ON (atomic_read (&obj->frontbuffer_bits)); -if (obj->pages && obj->madv ==I915_MADV_WILLNEED && +if (obj->mm.pages && obj->mm.madv ==I915_MADV_WILLNEED && dev_priv->quirks &QUIRK_PIN_SWIZZLED_PAGES &&i915_gem_object_is_tiled (obj)) -i915_gem_object_unpin_pages (obj); +__i915_gem_object_unpin_pages (obj); -if (WARN_ON (obj->pages_pin_count)) - obj->pages_pin_count =0 ; +if (obj->ops->release) + obj->ops->release (obj); + +if (WARN_ON (i915_gem_object_has_pinned_pages (obj))) + obj->mm.pages_pin_count =0 ;if (discard_backing_storage (obj)) - obj->madv =I915_MADV_DONTNEED ; -i915_gem_object_put_pages (obj); + obj->mm.madv =I915_MADV_DONTNEED ; +__i915_gem_object_put_pages (obj); -BUG_ON (obj->pages); +GEM_BUG_ON (obj->mm.pages);if (obj->base.import_attach)drm_prime_gem_destroy (&obj->base,NULL ); -if (obj->ops->release) - obj->ops->release (obj); -drm_gem_object_release (&obj->base);i915_gem_info_remove_obj (dev_priv, obj->base.size);@@ -4935,14 +4921,13 @@ i915_gem_object_create_from_data(struct drm_device *dev,if (ret) goto fail; - ret =i915_gem_object_get_pages (obj); + ret =i915_gem_object_pin_pages (obj);if (ret) goto fail; -i915_gem_object_pin_pages (obj); - sg = obj->pages; + sg = obj->mm.pages; bytes =sg_copy_from_buffer (sg->sgl, sg->nents, (void *)data, size); - obj->dirty =1 ;/* Backing store is now out of date */ + obj->mm.dirty = true;/* Backing store is now out of date */ i915_gem_object_unpin_pages (obj);if (WARN_ON (bytes != size)) {@@ -4962,7 +4947,7 @@ static struct scatterlist * __i915_gem_object_get_sg(struct drm_i915_gem_object *obj, unsigned long n, unsigned int *offset) { - struct scatterlist *sg = obj->pages->sgl; + struct scatterlist *sg = obj->mm.pages->sgl;int idx =0 ;while (idx +__sg_page_count (sg) <= n) {@@ -4980,11 +4965,11 @@ i915_gem_object_get_sg(struct drm_i915_gem_object *obj,unsigned long n, unsigned int *offset) { - struct i915_gem_object_page_iter *iter = &obj->get_page; + struct i915_gem_object_page_iter *iter = &obj->mm.get_page;struct scatterlist *sg;GEM_BUG_ON (n >= obj->base.size >>PAGE_SHIFT ); -GEM_BUG_ON (obj->pages_pin_count ==0 ); +GEM_BUG_ON (!i915_gem_object_has_pinned_pages (obj));if (n <READ_ONCE (iter->sg_idx)) goto lookup;@@ -5058,7 +5043,7 @@ i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj,struct page *page; page =i915_gem_object_get_page (obj, n); -if (!obj->dirty) +if (!obj->mm.dirty)set_page_dirty (page);return page;drivers/gpu/drm/i915/i915_gem_batch_pool.c +1
-2@@ -131,11 +131,10 @@ i915_gem_batch_pool_get(struct i915_gem_batch_pool *pool,return obj; } - ret =i915_gem_object_get_pages (obj); + ret =i915_gem_object_pin_pages (obj);if (ret)return ERR_PTR (ret);list_move_tail (&obj->batch_pool_link, list); -i915_gem_object_pin_pages (obj);return obj; }drivers/gpu/drm/i915/i915_gem_dmabuf.c +6
-8@@ -48,12 +48,10 @@ static struct sg_table *i915_gem_map_dma_buf(struct dma_buf_attachment *attachmeif (ret) goto err; - ret =i915_gem_object_get_pages (obj); + ret =i915_gem_object_pin_pages (obj);if (ret) goto err_unlock; -i915_gem_object_pin_pages (obj); -/* Copy sg so that we make an independent mapping */ st =kmalloc (sizeof (struct sg_table ),GFP_KERNEL );if (st ==NULL ) {@@ -61,13 +59,13 @@ static struct sg_table *i915_gem_map_dma_buf(struct dma_buf_attachment *attachme goto err_unpin; } - ret =sg_alloc_table (st, obj->pages->nents,GFP_KERNEL ); + ret =sg_alloc_table (st, obj->mm.pages->nents,GFP_KERNEL );if (ret) goto err_free; - src = obj->pages->sgl; + src = obj->mm.pages->sgl; dst = st->sgl; -for (i =0 ; i < obj->pages->nents; i++) { +for (i =0 ; i < obj->mm.pages->nents; i++) {sg_set_page (dst,sg_page (src), src->length,0 ); dst =sg_next (dst); src =sg_next (src);@@ -299,14 +297,14 @@ static int i915_gem_object_get_pages_dmabuf(struct drm_i915_gem_object *obj)if (IS_ERR (sg))return PTR_ERR (sg); - obj->pages = sg; + obj->mm.pages = sg;return 0 ; }static void i915_gem_object_put_pages_dmabuf (struct drm_i915_gem_object *obj) {dma_buf_unmap_attachment (obj->base.import_attach, - obj->pages,DMA_BIDIRECTIONAL ); + obj->mm.pages,DMA_BIDIRECTIONAL ); }static const struct drm_i915_gem_object_ops i915_gem_object_dmabuf_ops = {drivers/gpu/drm/i915/i915_gem_execbuffer.c +1
-1@@ -1290,7 +1290,7 @@ void i915_vma_move_to_active(struct i915_vma *vma,GEM_BUG_ON (!drm_mm_node_allocated (&vma->node)); - obj->dirty =1 ;/* be paranoid */ + obj->mm.dirty = true;/* be paranoid */ /* Add a reference if we're newly entering the active list. * The order in which we add operations to the retirement queue is drivers/gpu/drm/i915/i915_gem_fence.c +2
-2@@ -648,7 +648,7 @@ i915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object *obj)return ; i =0 ; -for_each_sgt_page (page, sgt_iter, obj->pages) { +for_each_sgt_page (page, sgt_iter, obj->mm.pages) {char new_bit_17 =page_to_phys (page) >>17 ;if ((new_bit_17 &0x1 ) != (test_bit (i, obj->bit_17) !=0 )) {@@ -687,7 +687,7 @@ i915_gem_object_save_bit_17_swizzle(struct drm_i915_gem_object *obj) i =0 ; -for_each_sgt_page (page, sgt_iter, obj->pages) { +for_each_sgt_page (page, sgt_iter, obj->mm.pages) {if (page_to_phys (page) & (1 <<17 ))__set_bit (i, obj->bit_17);else drivers/gpu/drm/i915/i915_gem_gtt.c +5
-5@@ -175,7 +175,7 @@ static int ppgtt_bind_vma(struct i915_vma *vma, {u32 pte_flags =0 ; - vma->pages = vma->obj->pages; + vma->pages = vma->obj->mm.pages;/* Currently applicable only to VLV */ if (vma->obj->gt_ro)@@ -2295,7 +2295,7 @@ void i915_gem_suspend_gtt_mappings(struct drm_device *dev)int i915_gem_gtt_prepare_object (struct drm_i915_gem_object *obj) {if (!dma_map_sg (&obj->base.dev->pdev->dev, - obj->pages->sgl, obj->pages->nents, + obj->mm.pages->sgl, obj->mm.pages->nents,PCI_DMA_BIDIRECTIONAL ))return -ENOSPC ;@@ -2685,7 +2685,7 @@ void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj) } } -dma_unmap_sg (kdev, obj->pages->sgl, obj->pages->nents, +dma_unmap_sg (kdev, obj->mm.pages->sgl, obj->mm.pages->nents,PCI_DMA_BIDIRECTIONAL ); }@@ -3526,7 +3526,7 @@ intel_rotate_fb_obj_pages(const struct intel_rotation_info *rot_info,/* Populate source page list from the object. */ i =0 ; -for_each_sgt_dma (dma_addr, sgt_iter, obj->pages) +for_each_sgt_dma (dma_addr, sgt_iter, obj->mm.pages) page_addr_list[i++] = dma_addr;GEM_BUG_ON (i != n_pages);@@ -3617,7 +3617,7 @@ i915_get_ggtt_vma_pages(struct i915_vma *vma)return 0 ;if (vma->ggtt_view.type ==I915_GGTT_VIEW_NORMAL ) - vma->pages = vma->obj->pages; + vma->pages = vma->obj->mm.pages;else if (vma->ggtt_view.type ==I915_GGTT_VIEW_ROTATED ) vma->pages =intel_rotate_fb_obj_pages (&vma->ggtt_view.params.rotated, vma->obj);drivers/gpu/drm/i915/i915_gem_internal.c +10
-9@@ -28,10 +28,11 @@static void internal_free_pages (struct sg_table *st) { -struct sg_page_iter sg_iter; +struct sgt_iter iter; +struct page *page; -for_each_sg_page (st->sgl, &sg_iter, st->nents,0 ) -put_page (sg_page_iter_page (&sg_iter)); +for_each_sgt_page (page, iter, st) +put_page (page);sg_free_table (st);kfree (st);@@ -94,10 +95,10 @@ static int i915_gem_object_get_pages_internal(struct drm_i915_gem_object *obj) if (!swiotlb_nr_tbl())#endif sg_mark_end (sg); - obj->pages = st; + obj->mm.pages = st;if (i915_gem_gtt_prepare_object (obj)) { - obj->pages =NULL ; + obj->mm.pages =NULL ; goto err; }@@ -106,7 +107,7 @@ static int i915_gem_object_get_pages_internal(struct drm_i915_gem_object *obj)* and the caller is expected to repopulate - the contents of this * object are only valid whilst active and pinned. */ - obj->madv =I915_MADV_DONTNEED ; + obj->mm.madv =I915_MADV_DONTNEED ;return 0 ; err:@@ -117,10 +118,10 @@ err:static void i915_gem_object_put_pages_internal (struct drm_i915_gem_object *obj) { -internal_free_pages (obj->pages); +internal_free_pages (obj->mm.pages); - obj->dirty =0 ; - obj->madv =I915_MADV_WILLNEED ; + obj->mm.dirty = false; + obj->mm.madv =I915_MADV_WILLNEED ; }static const struct drm_i915_gem_object_ops i915_gem_object_internal_ops = {drivers/gpu/drm/i915/i915_gem_render_state.c +1
-1@@ -230,7 +230,7 @@ int i915_gem_render_state_emit(struct drm_i915_gem_request *req)return 0 ;/* Recreate the page after shrinking */ -if (!so->vma->obj->pages) +if (!so->vma->obj->mm.pages) so->batch_offset =-1 ; ret =i915_vma_pin (so->vma,0 ,0 ,PIN_GLOBAL |PIN_HIGH );drivers/gpu/drm/i915/i915_gem_shrinker.c +5
-5@@ -78,7 +78,7 @@ static bool can_release_pages(struct drm_i915_gem_object *obj)* to the GPU, simply unbinding from the GPU is not going to succeed * in releasing our pin count on the pages themselves. */ -if (obj->pages_pin_count > obj->bind_count) +if (obj->mm.pages_pin_count > obj->bind_count)return false;if (any_vma_pinned (obj))@@ -88,7 +88,7 @@ static bool can_release_pages(struct drm_i915_gem_object *obj)* discard the contents (because the user has marked them as being * purgeable) or if we can move their contents out to swap. */ -return swap_available () || obj->madv ==I915_MADV_DONTNEED ; +return swap_available () || obj->mm.madv ==I915_MADV_DONTNEED ; }/** @@ -175,11 +175,11 @@ i915_gem_shrink(struct drm_i915_private *dev_priv,list_move_tail (&obj->global_list, &still_in_list);if (flags &I915_SHRINK_PURGEABLE && - obj->madv !=I915_MADV_DONTNEED ) + obj->mm.madv !=I915_MADV_DONTNEED )continue ;if (flags &I915_SHRINK_VMAPS && - !is_vmalloc_addr (obj->mapping)) + !is_vmalloc_addr (obj->mm.mapping))continue ; if ((flags & I915_SHRINK_ACTIVE) == 0 &&@@ -193,7 +193,7 @@ i915_gem_shrink(struct drm_i915_private *dev_priv,/* For the unbound phase, this should be a no-op! */ i915_gem_object_unbind (obj); -if (i915_gem_object_put_pages (obj) ==0 ) +if (__i915_gem_object_put_pages (obj) ==0 ) count += obj->base.size >>PAGE_SHIFT ;i915_gem_object_put (obj);drivers/gpu/drm/i915/i915_gem_stolen.c +13
-11@@ -554,16 +554,17 @@ static int i915_gem_object_get_pages_stolen(struct drm_i915_gem_object *obj)static void i915_gem_object_put_pages_stolen (struct drm_i915_gem_object *obj) {/* Should only be called during free */ -sg_free_table (obj->pages); -kfree (obj->pages); +sg_free_table (obj->mm.pages); +kfree (obj->mm.pages); } -static void i915_gem_object_release_stolen (struct drm_i915_gem_object *obj) {struct drm_i915_private *dev_priv =to_i915 (obj->base.dev); +__i915_gem_object_unpin_pages (obj); +if (obj->stolen) {i915_gem_stolen_remove_node (dev_priv, obj->stolen);kfree (obj->stolen);@@ -589,15 +590,16 @@ _i915_gem_object_create_stolen(struct drm_device *dev,drm_gem_private_object_init (dev, &obj->base, stolen->size);i915_gem_object_init (obj, &i915_gem_object_stolen_ops); - obj->pages =i915_pages_create_for_stolen (dev, - stolen->start, stolen->size); -if (obj->pages ==NULL ) + obj->mm.pages =i915_pages_create_for_stolen (dev, + stolen->start, + stolen->size); +if (!obj->mm.pages) goto cleanup; - obj->get_page.sg_pos = obj->pages->sgl; - obj->get_page.sg_idx =0 ; + obj->mm.get_page.sg_pos = obj->mm.pages->sgl; + obj->mm.get_page.sg_idx =0 ; -i915_gem_object_pin_pages (obj); +__i915_gem_object_pin_pages (obj); obj->stolen = stolen; obj->base.read_domains =I915_GEM_DOMAIN_CPU |I915_GEM_DOMAIN_GTT ;@@ -717,14 +719,14 @@ i915_gem_object_create_stolen_for_preallocated(struct drm_device *dev, goto err; } - vma->pages = obj->pages; + vma->pages = obj->mm.pages; vma->flags |=I915_VMA_GLOBAL_BIND ;__i915_vma_set_map_and_fenceable (vma);list_move_tail (&vma->vm_link, &ggtt->base.inactive_list); obj->bind_count++;list_add_tail (&obj->global_list, &dev_priv->mm.bound_list); -i915_gem_object_pin_pages (obj); +__i915_gem_object_pin_pages (obj);return obj;drivers/gpu/drm/i915/i915_gem_tiling.c +4
-4@@ -260,13 +260,13 @@ i915_gem_set_tiling(struct drm_device *dev, void *data,if (!err) {struct i915_vma *vma; -if (obj->pages && - obj->madv ==I915_MADV_WILLNEED && +if (obj->mm.pages && + obj->mm.madv ==I915_MADV_WILLNEED && dev_priv->quirks &QUIRK_PIN_SWIZZLED_PAGES ) {if (args->tiling_mode ==I915_TILING_NONE ) -i915_gem_object_unpin_pages (obj); +__i915_gem_object_unpin_pages (obj);if (!i915_gem_object_is_tiled (obj)) -i915_gem_object_pin_pages (obj); +__i915_gem_object_pin_pages (obj); }list_for_each_entry (vma, &obj->vma_list, obj_link) {drivers/gpu/drm/i915/i915_gem_userptr.c +15
-15@@ -73,10 +73,10 @@ static void cancel_userptr(struct work_struct *work)/* Cancel any active worker and force us to re-evaluate gup */ obj->userptr.work =NULL ; -if (obj->pages !=NULL ) { +if (obj->mm.pages) {/* We are inside a kthread context and can't be interrupted */ WARN_ON (i915_gem_object_unbind (obj)); -WARN_ON (i915_gem_object_put_pages (obj)); +WARN_ON (__i915_gem_object_put_pages (obj)); }i915_gem_object_put (obj);@@ -432,15 +432,15 @@ __i915_gem_userptr_set_pages(struct drm_i915_gem_object *obj, {int ret; - ret =st_set_pages (&obj->pages, pvec, num_pages); + ret =st_set_pages (&obj->mm.pages, pvec, num_pages);if (ret)return ret; ret =i915_gem_gtt_prepare_object (obj);if (ret) { -sg_free_table (obj->pages); -kfree (obj->pages); - obj->pages =NULL ; +sg_free_table (obj->mm.pages); +kfree (obj->mm.pages); + obj->mm.pages =NULL ; }return ret;@@ -526,8 +526,8 @@ __i915_gem_userptr_get_pages_worker(struct work_struct *_work)if (ret ==0 ) {list_add_tail (&obj->global_list, &to_i915 (dev)->mm.unbound_list); - obj->get_page.sg_pos = obj->pages->sgl; - obj->get_page.sg_idx =0 ; + obj->mm.get_page.sg_pos = obj->mm.pages->sgl; + obj->mm.get_page.sg_idx =0 ; pinned =0 ; } }@@ -668,22 +668,22 @@ i915_gem_userptr_put_pages(struct drm_i915_gem_object *obj)BUG_ON (obj->userptr.work !=NULL );__i915_gem_userptr_set_active (obj, false); -if (obj->madv !=I915_MADV_WILLNEED ) - obj->dirty =0 ; +if (obj->mm.madv !=I915_MADV_WILLNEED ) + obj->mm.dirty = false;i915_gem_gtt_finish_object (obj); -for_each_sgt_page (page, sgt_iter, obj->pages) { -if (obj->dirty) +for_each_sgt_page (page, sgt_iter, obj->mm.pages) { +if (obj->mm.dirty)set_page_dirty (page);mark_page_accessed (page);put_page (page); } - obj->dirty =0 ; + obj->mm.dirty = false; -sg_free_table (obj->pages); -kfree (obj->pages); +sg_free_table (obj->mm.pages); +kfree (obj->mm.pages); } static voiddrivers/gpu/drm/i915/i915_gpu_error.c +2
-2@@ -881,8 +881,8 @@ static void capture_bo(struct drm_i915_error_buffer *err, err->write_domain = obj->base.write_domain; err->fence_reg = vma->fence ? vma->fence->id :-1 ; err->tiling =i915_gem_object_get_tiling (obj); - err->dirty = obj->dirty; - err->purgeable = obj->madv !=I915_MADV_WILLNEED ; + err->dirty = obj->mm.dirty; + err->purgeable = obj->mm.madv !=I915_MADV_WILLNEED ; err->userptr = obj->userptr.mm !=NULL ; err->cache_level = obj->cache_level; }drivers/gpu/drm/i915/intel_lrc.c +2
-2@@ -744,7 +744,7 @@ static int intel_lr_context_pin(struct i915_gem_context *ctx, ce->lrc_reg_state[CTX_RING_BUFFER_START +1 ] =i915_ggtt_offset (ce->ring->vma); - ce->state->obj->dirty = true; + ce->state->obj->mm.dirty = true;/* Invalidate GuC TLB. */ if (i915.enable_guc_submission) {@@ -2042,7 +2042,7 @@ populate_lr_context(struct i915_gem_context *ctx,DRM_DEBUG_DRIVER ("Could not map object pages! (%d)\n" , ret);return ret; } - ctx_obj->dirty = true; + ctx_obj->mm.dirty = true;/* The second page of the context object contains some fields which must * be set up prior to the first execution. */
drivers/gpu/drm/i915/intel_lrc.c void intel_lr_context_resume(struct drm_i915_private *dev_priv) · line 2185
Quoted 8 lines
drivers/gpu/drm/i915/intel_lrc.c +1
-1@@ -2179,7 +2179,7 @@ void intel_lr_context_resume(struct drm_i915_private *dev_priv) reg[CTX_RING_HEAD +1 ] =0 ; reg[CTX_RING_TAIL +1 ] =0 ; - ce->state->obj->dirty = true; + ce->state->obj->mm.dirty = true;i915_gem_object_unpin_map (ce->state->obj); ce->ring->head = ce->ring->tail =0 ;
Did not spot any issues. Hopefully CI is good enough to helps us if we missed something.
It even looks better (simpler) than the current API which was always slightly confusing.
Regards,
Tvrtko
_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
Re: [PATCH 16/42] drm/i915: Refactor object page API
In reply to Chris Wilsondrivers/gpu/drm/i915/i915_drv.h dma_addr_t · line 3196
Quoted 6 lines · Chris Wilson
drivers/gpu/drm/i915/i915_drv.h +2
-0+static inline int __must_check +i915_gem_object_pin_pages (struct drm_i915_gem_object *obj){
drivers/gpu/drm/i915/i915_drv.h +1
-2-BUG_ON (obj->pages ==NULL ); - obj->pages_pin_count++; +lockdep_assert_held (&obj->base.dev->struct_mutex);
\n here.
drivers/gpu/drm/i915/i915_drv.h dma_addr_t · line 3200
Quoted 4 lines
drivers/gpu/drm/i915/i915_drv.h +4
-0+if (obj->mm.pages_pin_count++) +return 0 ; + +return __i915_gem_object_get_pages (obj);
I also second John here, what's up with the dummy wrapper convetion. Add TODO comments if you intend to introduce new stuff relying on that.
drivers/gpu/drm/i915/i915_drv.h dma_addr_t · line 3206
Quoted 6 lines
drivers/gpu/drm/i915/i915_drv.h +6
-0+} + +static inline void +__i915_gem_object_pin_pages (struct drm_i915_gem_object *obj) +{ +lockdep_assert_held (&obj->base.dev->struct_mutex);
Ditto.
Quoted 6 lines
drivers/gpu/drm/i915/i915_drv.h +6
-0+GEM_BUG_ON (!obj->mm.pages); + obj->mm.pages_pin_count++; +} + +static inline bool +i915_gem_object_has_pinned_pages (struct drm_i915_gem_object *obj)
drivers/gpu/drm/i915/i915_drv.h dma_addr_t · line 3222
Quoted 10 lines
drivers/gpu/drm/i915/i915_drv.h +10
-0+{ +return obj->mm.pages_pin_count; +} + +static inline void +__i915_gem_object_unpin_pages (struct drm_i915_gem_object *obj) +{ +lockdep_assert_held (&obj->base.dev->struct_mutex); +GEM_BUG_ON (!i915_gem_object_has_pinned_pages (obj)); +GEM_BUG_ON (!obj->mm.pages);
\n at least here.
Quoted 3 lines
drivers/gpu/drm/i915/i915_drv.h +1
-0+ obj->mm.pages_pin_count--;}
<SNIP>
Review
Quoted 6 lines
drivers/gpu/drm/i915/i915_gem.c +0
-0@@ -812,7 +809,7 @@ int i915_gem_obj_prepare_shmem_write(struct drm_i915_gem_object *obj, obj->cache_dirty = true;
intel_fb_obj_invalidate(obj, ORIGIN_CPU);(quoted diff) +1
-1- obj->dirty = 1; + obj->mm.dirty = true;
The type is not bool dirty : 1, do we get yelled at by compiler?
drivers/gpu/drm/i915/i915_gem.c i915_gem_gtt_pwrite_fast(struct drm_i915_private *i915, · line 1264
Quoted 6 lines
drivers/gpu/drm/i915/i915_gem.c +0
-0@@ -1268,7 +1261,7 @@ i915_gem_gtt_pwrite_fast(struct drm_i915_private *i915, goto out_unpin;
intel_fb_obj_invalidate(obj, ORIGIN_CPU);drivers/gpu/drm/i915/i915_gem.c +1
-1- obj->dirty = true; + obj->mm.dirty = true;
I assume not as this has not been changed. Consistency is good anyway.
Review
Quoted 12 lines
(quoted diff) +0
-0@@ -2483,24 +2474,25 @@ i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
lockdep_assert_held(&obj->base.dev->struct_mutex);drivers/gpu/drm/i915/i915_gem.c +1
-1-if (obj->pages) +if (obj->mm.pages)return 0;
drivers/gpu/drm/i915/i915_gem.c +1
-1-if (obj->madv !=I915_MADV_WILLNEED ) { +if (obj->mm.madv !=I915_MADV_WILLNEED ) {
DRM_DEBUG("Attempting to obtain a purgeable object\n");(quoted diff) +1
-0+ __i915_gem_object_unpin_pages(obj);
Confusing to have teardown of another function in here.
drivers/gpu/drm/i915/i915_gem.c i915_gem_object_get_pages(struct drm_i915_gem_object *obj) · line 2488
Quoted 9 lines
return
-EFAULT;
}drivers/gpu/drm/i915/i915_gem.c +0
-2-BUG_ON (obj->pages_pin_count); -ret =
ops->get_pages(obj);drivers/gpu/drm/i915/i915_gem.c +2
-1-if (ret) +if (ret) { +__i915_gem_object_unpin_pages (obj);
And if you like *really* have to, at least try not to duplicate code. Bonus points form moving this to be proper teardown path where it has a counter-part.
Strange side effects on failing function call do not lead to very understandable code structure, but major spaghetti. Can be fixed in follow-up patch too.
Regards, Joonas
Signature · 7 lines
Joonas Lahtinen Open Source Technology Center Intel Corporation _______________________________________________ Intel-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/intel-gfx
Re: [PATCH 16/42] drm/i915: Refactor object page API
Quoted 16 lines · Joonas Lahtinen
Review
Quoted 12 lines · Chris Wilson
(quoted diff) +0
-0@@ -2483,24 +2474,25 @@ i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
lockdep_assert_held(&obj->base.dev->struct_mutex);drivers/gpu/drm/i915/i915_gem.c +1
-1-if (obj->pages) +if (obj->mm.pages)return 0;
drivers/gpu/drm/i915/i915_gem.c +1
-1-if (obj->madv !=I915_MADV_WILLNEED ) { +if (obj->mm.madv !=I915_MADV_WILLNEED ) {
DRM_DEBUG("Attempting to obtain a purgeable object\n");(quoted diff) +1
-0+ __i915_gem_object_unpin_pages(obj);Confusing to have teardown of another function in here.
Quoted 13 lines
drivers/gpu/drm/i915/i915_gem.c i915_gem_object_get_pages(struct drm_i915_gem_object *obj) · line 2488
Quoted 9 lines
return
-EFAULT;
}drivers/gpu/drm/i915/i915_gem.c +0
-2-BUG_ON (obj->pages_pin_count); -ret =
ops->get_pages(obj);drivers/gpu/drm/i915/i915_gem.c +2
-1-if (ret) +if (ret) { +__i915_gem_object_unpin_pages (obj);And if you like *really* have to, at least try not to duplicate code. Bonus points form moving this to be proper teardown path where it has a counter-part.
It is here, so that the unlikely error handling is not inside the inlined function, but where we expect the code to grow to handle the more complex locking requirements.
-Chris
Signature · 5 lines
Chris Wilson, Intel Open Source Technology Centre _______________________________________________ Intel-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH 17/42 drm/i915: Pass around sg_table to get_pages/put_pages backend
In reply to Chris WilsonThe plan is to move obj->pages out from under the struct_mutex into its own per-object lock. We need to prune any assumption of the struct_mutex from the get_pages/put_pages backends, and to make it easier we pass around the sg_table to operate on rather than indirectly via the obj.
--- drivers/gpu/drm/i915/i915_drv.h | 36 +++++-- drivers/gpu/drm/i915/i915_gem.c | 172 +++++++++++++++---------------- drivers/gpu/drm/i915/i915_gem_dmabuf.c | 20 ++-- drivers/gpu/drm/i915/i915_gem_fence.c | 18 ++-- drivers/gpu/drm/i915/i915_gem_gtt.c | 19 ++-- drivers/gpu/drm/i915/i915_gem_gtt.h | 6 +- drivers/gpu/drm/i915/i915_gem_internal.c | 22 ++-- drivers/gpu/drm/i915/i915_gem_shrinker.c | 11 +- drivers/gpu/drm/i915/i915_gem_stolen.c | 43 ++++---- drivers/gpu/drm/i915/i915_gem_userptr.c | 88 ++++++++-------- 10 files changed, 227 insertions(+), 208 deletions(-)
drivers/gpu/drm/i915/i915_drv.h +27 -9
@@ -2175,8 +2175,8 @@ struct drm_i915_gem_object_ops {* being released or under memory pressure (where we attempt to * reap pages for the shrinker). */ -int (*get_pages)(struct drm_i915_gem_object *); -void (*put_pages)(struct drm_i915_gem_object *); +struct sg_table *(*get_pages)(struct drm_i915_gem_object *); +void (*put_pages)(struct drm_i915_gem_object *,struct sg_table *);int (*dmabuf_export)(struct drm_i915_gem_object *);void (*release)(struct drm_i915_gem_object *);
@@ -2313,8 +2313,6 @@ struct drm_i915_gem_object {struct i915_gem_userptr {uintptr_t ptr;unsigned read_only :1 ; -unsigned workers :4 ; -#define I915_GEM_USERPTR_MAX_WORKERS 15struct i915_mm_struct *mm;struct i915_mmu_object *mmu_object;
@@ -2376,6 +2374,19 @@ __deprecatedextern void drm_gem_object_unreference_unlocked (struct drm_gem_object *);static inline bool +i915_gem_object_is_dead (const struct drm_i915_gem_object *obj) +{ +return atomic_read (&obj->base.refcount.refcount) ==0 ; +} + +#if IS_ENABLED (CONFIG_LOCKDEP ) +#define lockdep_assert_held_unless (lock, cond) \ + GEM_BUG_ON(!lockdep_is_held(lock) && !(cond)) +#else +#define lockdep_assert_held_unless (lock, cond) +#endif + +static inline bool i915_gem_object_has_struct_page (const struct drm_i915_gem_object *obj) {return obj->ops->flags &I915_GEM_OBJECT_HAS_STRUCT_PAGE ;
@@ -3188,6 +3199,8 @@ dma_addr_t i915_gem_object_get_dma_address(struct drm_i915_gem_object *obj, unsigned long n); +void __i915_gem_object_set_pages (struct drm_i915_gem_object *obj, +struct sg_table *pages);int __i915_gem_object_get_pages (struct drm_i915_gem_object *obj); static inline int __must_check
@@ -3203,7 +3216,8 @@ i915_gem_object_pin_pages(struct drm_i915_gem_object *obj)static inline void __i915_gem_object_pin_pages (struct drm_i915_gem_object *obj) { -lockdep_assert_held (&obj->base.dev->struct_mutex); +lockdep_assert_held_unless (&obj->base.dev->struct_mutex, +i915_gem_object_is_dead (obj));GEM_BUG_ON (!obj->mm.pages); obj->mm.pages_pin_count++; }
@@ -3217,7 +3231,8 @@ i915_gem_object_has_pinned_pages(struct drm_i915_gem_object *obj)static inline void __i915_gem_object_unpin_pages (struct drm_i915_gem_object *obj) { -lockdep_assert_held (&obj->base.dev->struct_mutex); +lockdep_assert_held_unless (&obj->base.dev->struct_mutex, +i915_gem_object_is_dead (obj));GEM_BUG_ON (!i915_gem_object_has_pinned_pages (obj));GEM_BUG_ON (!obj->mm.pages); obj->mm.pages_pin_count--;
@@ -3228,7 +3243,8 @@ static inline void i915_gem_object_unpin_pages(struct drm_i915_gem_object *obj)__i915_gem_object_unpin_pages (obj); } -int __i915_gem_object_put_pages (struct drm_i915_gem_object *obj); +void __i915_gem_object_put_pages (struct drm_i915_gem_object *obj); +void __i915_gem_object_invalidate (struct drm_i915_gem_object *obj);enum i915_map_type {I915_MAP_WB =0 ,
@@ -3451,8 +3467,10 @@ i915_vma_unpin_fence(struct i915_vma *vma)void i915_gem_restore_fences (struct drm_device *dev);void i915_gem_detect_bit_6_swizzle (struct drm_device *dev); -void i915_gem_object_do_bit_17_swizzle (struct drm_i915_gem_object *obj); -void i915_gem_object_save_bit_17_swizzle (struct drm_i915_gem_object *obj); +void i915_gem_object_do_bit_17_swizzle (struct drm_i915_gem_object *obj, +struct sg_table *pages); +void i915_gem_object_save_bit_17_swizzle (struct drm_i915_gem_object *obj, +struct sg_table *pages);/* i915_gem_context.c */ int __must_check i915_gem_context_init (struct drm_device *dev);
drivers/gpu/drm/i915/i915_gem.c +83 -89
338 lines not on this page · raw message
drivers/gpu/drm/i915/i915_gem_dmabuf.c +8 -12
@@ -289,22 +289,18 @@ struct dma_buf *i915_gem_prime_export(struct drm_device *dev,return dma_buf; } -static int i915_gem_object_get_pages_dmabuf (struct drm_i915_gem_object *obj) +static struct sg_table * +i915_gem_object_get_pages_dmabuf (struct drm_i915_gem_object *obj) { -struct sg_table *sg; - - sg =dma_buf_map_attachment (obj->base.import_attach,DMA_BIDIRECTIONAL ); -if (IS_ERR (sg)) -return PTR_ERR (sg); - - obj->mm.pages = sg; -return 0 ; +return dma_buf_map_attachment (obj->base.import_attach, +DMA_BIDIRECTIONAL ); } -static void i915_gem_object_put_pages_dmabuf (struct drm_i915_gem_object *obj) +static void i915_gem_object_put_pages_dmabuf (struct drm_i915_gem_object *obj, +struct sg_table *pages) { -dma_buf_unmap_attachment (obj->base.import_attach, - obj->mm.pages,DMA_BIDIRECTIONAL ); +dma_buf_unmap_attachment (obj->base.import_attach, pages, +DMA_BIDIRECTIONAL ); }static const struct drm_i915_gem_object_ops i915_gem_object_dmabuf_ops = {
drivers/gpu/drm/i915/i915_gem_fence.c +10 -8
@@ -628,6 +628,7 @@ i915_gem_swizzle_page(struct page *page)/** * i915_gem_object_do_bit_17_swizzle - fixup bit 17 swizzling * @obj: i915 GEM buffer object +* @pages: the scattergather list of physical pages * * This function fixes up the swizzling in case any page frame number for this * object has changed in bit 17 since that state has been saved with
@@ -638,7 +639,8 @@ i915_gem_swizzle_page(struct page *page)* by swapping them out and back in again). */ void -i915_gem_object_do_bit_17_swizzle (struct drm_i915_gem_object *obj) +i915_gem_object_do_bit_17_swizzle (struct drm_i915_gem_object *obj, +struct sg_table *pages) {struct sgt_iter sgt_iter;struct page *page;
@@ -648,10 +650,9 @@ i915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object *obj)return ; i =0 ; -for_each_sgt_page (page, sgt_iter, obj->mm.pages) { +for_each_sgt_page (page, sgt_iter, pages) {char new_bit_17 =page_to_phys (page) >>17 ; -if ((new_bit_17 &0x1 ) != - (test_bit (i, obj->bit_17) !=0 )) { +if ((new_bit_17 &0x1 ) != (test_bit (i, obj->bit_17) !=0 )) {i915_gem_swizzle_page (page);set_page_dirty (page); }
@@ -662,21 +663,22 @@ i915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object *obj)/** * i915_gem_object_save_bit_17_swizzle - save bit 17 swizzling * @obj: i915 GEM buffer object +* @pages: the scattergather list of physical pages * * This function saves the bit 17 of each page frame number so that swizzling * can be fixed up later on with i915_gem_object_do_bit_17_swizzle(). This must * be called before the backing storage can be unpinned. */ void -i915_gem_object_save_bit_17_swizzle (struct drm_i915_gem_object *obj) +i915_gem_object_save_bit_17_swizzle (struct drm_i915_gem_object *obj, +struct sg_table *pages) {struct sgt_iter sgt_iter;struct page *page; -int page_count = obj->base.size >>PAGE_SHIFT ;int i;if (obj->bit_17 ==NULL ) { - obj->bit_17 =kcalloc (BITS_TO_LONGS (page_count), + obj->bit_17 =kcalloc (BITS_TO_LONGS (obj->base.size >>PAGE_SHIFT ),sizeof (long ),GFP_KERNEL );if (obj->bit_17 ==NULL ) { DRM_ERROR("Failed to allocate memory for bit 17 "
@@ -687,7 +689,7 @@ i915_gem_object_save_bit_17_swizzle(struct drm_i915_gem_object *obj) i =0 ; -for_each_sgt_page (page, sgt_iter, obj->mm.pages) { +for_each_sgt_page (page, sgt_iter, pages) {if (page_to_phys (page) & (1 <<17 ))__set_bit (i, obj->bit_17);else
drivers/gpu/drm/i915/i915_gem_gtt.c +10 -9
@@ -2292,14 +2292,15 @@ void i915_gem_suspend_gtt_mappings(struct drm_device *dev)i915_ggtt_flush (dev_priv); } -int i915_gem_gtt_prepare_object (struct drm_i915_gem_object *obj) +int i915_gem_gtt_prepare_pages (struct drm_i915_gem_object *obj, +struct sg_table *pages) { -if (!dma_map_sg (&obj->base.dev->pdev->dev, - obj->mm.pages->sgl, obj->mm.pages->nents, -PCI_DMA_BIDIRECTIONAL )) -return -ENOSPC ; +if (dma_map_sg (&obj->base.dev->pdev->dev, + pages->sgl, pages->nents, +PCI_DMA_BIDIRECTIONAL )) +return 0 ; -return 0 ; +return -ENOSPC ; }static void gen8_set_pte (void __iomem *addr,gen8_pte_t pte)
@@ -2671,7 +2672,8 @@ static void ggtt_unbind_vma(struct i915_vma *vma) true); } -void i915_gem_gtt_finish_object (struct drm_i915_gem_object *obj) +void i915_gem_gtt_finish_pages (struct drm_i915_gem_object *obj, +struct sg_table *pages) {struct drm_i915_private *dev_priv =to_i915 (obj->base.dev);struct device *kdev = &dev_priv->drm.pdev->dev;
@@ -2685,8 +2687,7 @@ void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj) } } -dma_unmap_sg (kdev, obj->mm.pages->sgl, obj->mm.pages->nents, -PCI_DMA_BIDIRECTIONAL ); +dma_unmap_sg (kdev, pages->sgl, pages->nents,PCI_DMA_BIDIRECTIONAL ); } static void i915_gtt_color_adjust(struct drm_mm_node *node,
drivers/gpu/drm/i915/i915_gem_gtt.h +4 -2
@@ -629,8 +629,10 @@ void i915_check_and_clear_faults(struct drm_i915_private *dev_priv);void i915_gem_suspend_gtt_mappings (struct drm_device *dev);void i915_gem_restore_gtt_mappings (struct drm_device *dev); -int __must_check i915_gem_gtt_prepare_object (struct drm_i915_gem_object *obj); -void i915_gem_gtt_finish_object (struct drm_i915_gem_object *obj); +int __must_check i915_gem_gtt_prepare_pages (struct drm_i915_gem_object *obj, +struct sg_table *pages); +void i915_gem_gtt_finish_pages (struct drm_i915_gem_object *obj, +struct sg_table *pages);/* Flags used by pin/bind&friends. */ #define PIN_NONBLOCK BIT(0)
drivers/gpu/drm/i915/i915_gem_internal.c +11 -11
@@ -38,7 +38,8 @@ static void internal_free_pages(struct sg_table *st)kfree (st); } -static int i915_gem_object_get_pages_internal (struct drm_i915_gem_object *obj) +static struct sg_table * +i915_gem_object_get_pages_internal (struct drm_i915_gem_object *obj) {const unsigned int npages = obj->base.size /PAGE_SIZE ;struct sg_table *st;
@@ -49,11 +50,11 @@ static int i915_gem_object_get_pages_internal(struct drm_i915_gem_object *obj) st =kmalloc (sizeof (*st),GFP_KERNEL );if (!st) -return -ENOMEM ; +return ERR_PTR (-ENOMEM );if (sg_alloc_table (st, npages,GFP_KERNEL )) {kfree (st); -return -ENOMEM ; +return ERR_PTR (-ENOMEM ); } sg = st->sgl;
@@ -95,12 +96,9 @@ static int i915_gem_object_get_pages_internal(struct drm_i915_gem_object *obj) if (!swiotlb_nr_tbl())#endif sg_mark_end (sg); - obj->mm.pages = st; -if (i915_gem_gtt_prepare_object (obj)) { - obj->mm.pages =NULL ; +if (i915_gem_gtt_prepare_pages (obj, st)) goto err; - }/* Mark the pages as dontneed whilst they are still pinned. As soon * as they are unpinned they are allowed to be reaped by the shrinker,
@@ -108,17 +106,19 @@ static int i915_gem_object_get_pages_internal(struct drm_i915_gem_object *obj)* object are only valid whilst active and pinned. */ obj->mm.madv =I915_MADV_DONTNEED ; -return 0 ; +return st; err:sg_mark_end (sg);internal_free_pages (st); -return -ENOMEM ; +return ERR_PTR (-ENOMEM ); } -static void i915_gem_object_put_pages_internal (struct drm_i915_gem_object *obj) +static void i915_gem_object_put_pages_internal (struct drm_i915_gem_object *obj, +struct sg_table *pages) { -internal_free_pages (obj->mm.pages); +i915_gem_gtt_finish_pages (obj, pages); +internal_free_pages (pages); obj->mm.dirty = false; obj->mm.madv =I915_MADV_WILLNEED ;
drivers/gpu/drm/i915/i915_gem_shrinker.c +8 -3
@@ -91,6 +91,13 @@ static bool can_release_pages(struct drm_i915_gem_object *obj)return swap_available () || obj->mm.madv ==I915_MADV_DONTNEED ; } +static bool unsafe_drop_pages (struct drm_i915_gem_object *obj) +{ +if (i915_gem_object_unbind (obj) ==0 ) +__i915_gem_object_put_pages (obj); +return !obj->mm.pages; +} +/** * i915_gem_shrink - Shrink buffer object caches * @dev_priv: i915 device
@@ -191,9 +198,7 @@ i915_gem_shrink(struct drm_i915_private *dev_priv,i915_gem_object_get (obj); -/* For the unbound phase, this should be a no-op! */ -i915_gem_object_unbind (obj); -if (__i915_gem_object_put_pages (obj) ==0 ) +if (unsafe_drop_pages (obj)) count += obj->base.size >>PAGE_SHIFT ;i915_gem_object_put (obj);
drivers/gpu/drm/i915/i915_gem_stolen.c +21 -22
89 lines not on this page · raw message
drivers/gpu/drm/i915/i915_gem_userptr.c +45 -43
196 lines not on this page · raw message
Signature · 6 lines
2.9.3 _______________________________________________ Intel-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/intel-gfx
Re: [PATCH 17/42] drm/i915: Pass around sg_table to get_pages/put_pages backend
drivers/gpu/drm/i915/i915_drv.h __deprecated · line 2380
Quoted 8 lines · Chris Wilson
drivers/gpu/drm/i915/i915_drv.h +0
-0@@ -2376,6 +2374,19 @@ __deprecated extern void drm_gem_object_unreference_unlocked(struct drm_gem_object *);
static inline booldrivers/gpu/drm/i915/i915_drv.h +4
-0+i915_gem_object_is_dead(const struct drm_i915_gem_object *obj) +{ +return atomic_read (&obj->base.refcount.refcount) ==0 ; +}
Seems like this ought to be in DRM? Todo item.
drivers/gpu/drm/i915/i915_drv.h __deprecated · line 2387
Quoted 7 lines
drivers/gpu/drm/i915/i915_drv.h +7
-0+ +#if IS_ENABLED (CONFIG_LOCKDEP ) +#define lockdep_assert_held_unless (lock, cond) \ + GEM_BUG_ON(!lockdep_is_held(lock) && !(cond)) +#else +#define lockdep_assert_held_unless (lock, cond) +#endif
Ugh.
drivers/gpu/drm/i915/i915_gem_userptr.c i915_gem_userptr_get_pages(struct drm_i915_gem_object *obj) · line 650
Quoted 6 lines
(quoted diff) +0
-0@@ -646,21 +647,22 @@ i915_gem_userptr_get_pages(struct drm_i915_gem_object *obj)
active = false;
if (pinned < 0)drivers/gpu/drm/i915/i915_gem_userptr.c +1
-1- ret = pinned, pinned =0 ; + pages =ERR_PTR (pinned), pinned =0 ;
Just convert to curly braces.
Quoted 14 lines
else if (pinned <
num_pages)drivers/gpu/drm/i915/i915_gem_userptr.c +1
-1- ret =__i915_gem_userptr_get_pages_schedule (obj, &active); + pages =__i915_gem_userptr_get_pages_schedule (obj, &active);else
drivers/gpu/drm/i915/i915_gem_userptr.c +2
-2- ret =__i915_gem_userptr_set_pages (obj, pvec, num_pages); -if (ret) { + pages =__i915_gem_userptr_set_pages (obj, pvec, num_pages); +if (IS_ERR (pages)) {
__i915_gem_userptr_set_active(obj, active);release_pages(pvec, pinned, 0);
}
drm_free_large(pvec);(quoted diff) +1
-1- return ret; + return pages;
Quoted 2 lines
}
Horrible patch to review, mix of different changes all in one.
Regars, Joonas
Signature · 11 lines
Joonas Lahtinen Open Source Technology Center Intel Corporation -- Joonas Lahtinen Open Source Technology Center Intel Corporation _______________________________________________ Intel-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/intel-gfx
Re: [PATCH 17/42] drm/i915: Pass around sg_table to get_pages/put_pages backend
Quoted 11 lines · Joonas Lahtinen
drivers/gpu/drm/i915/i915_drv.h __deprecated · line 2380
Quoted 8 lines · Chris Wilson
drivers/gpu/drm/i915/i915_drv.h +0
-0@@ -2376,6 +2374,19 @@ __deprecated extern void drm_gem_object_unreference_unlocked(struct drm_gem_object *);
static inline booldrivers/gpu/drm/i915/i915_drv.h +4
-0+i915_gem_object_is_dead(const struct drm_i915_gem_object *obj) +{ +return atomic_read (&obj->base.refcount.refcount) ==0 ; +}Seems like this ought to be in DRM? Todo item.
Quoted 9 lines
drivers/gpu/drm/i915/i915_drv.h __deprecated · line 2387
Quoted 7 lines
drivers/gpu/drm/i915/i915_drv.h +7
-0+ +#if IS_ENABLED (CONFIG_LOCKDEP ) +#define lockdep_assert_held_unless (lock, cond) \ + GEM_BUG_ON(!lockdep_is_held(lock) && !(cond)) +#else +#define lockdep_assert_held_unless (lock, cond) +#endif Ugh.
These are temporaries to keep lockdep asserts working until removed later in the series (but I hope they illustrate the change in locking regime being prepared for).
-Chris
Signature · 5 lines
Chris Wilson, Intel Open Source Technology Centre _______________________________________________ Intel-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/intel-gfx
Re: [PATCH 17/42] drm/i915: Pass around sg_table to get_pages/put_pages backend
In reply to Chris Wilson[sending the draft reply until I get another look at it]
Quoted 24 lines · Chris Wilson
The plan is to move
obj->pagesout from under thestruct_mutexinto its own per-object lock. We need to prune any assumption of thestruct_mutexfrom the get_pages/put_pages backends, and to make it easier we pass around thesg_tableto operate on rather than indirectly via the obj.Signed-off-by Chris Wilson<[email protected]>--- drivers/gpu/drm/i915/i915_drv.h | 36 +++++-- drivers/gpu/drm/i915/i915_gem.c | 172 +++++++++++++++---------------- drivers/gpu/drm/i915/i915_gem_dmabuf.c | 20 ++-- drivers/gpu/drm/i915/i915_gem_fence.c | 18 ++-- drivers/gpu/drm/i915/i915_gem_gtt.c | 19 ++-- drivers/gpu/drm/i915/i915_gem_gtt.h | 6 +- drivers/gpu/drm/i915/i915_gem_internal.c | 22 ++-- drivers/gpu/drm/i915/i915_gem_shrinker.c | 11 +- drivers/gpu/drm/i915/i915_gem_stolen.c | 43 ++++---- drivers/gpu/drm/i915/i915_gem_userptr.c | 88 ++++++++-------- 10 files changed, 227 insertions(+), 208 deletions(-)drivers/gpu/drm/i915/i915_drv.h +0
-0
drivers/gpu/drm/i915/i915_drv.h struct drm_i915_gem_object_ops { · line 2179
Quoted 7 lines
drivers/gpu/drm/i915/i915_drv.h +2
-2@@ -2175,8 +2175,8 @@ struct drm_i915_gem_object_ops {* being released or under memory pressure (where we attempt to * reap pages for the shrinker). */ -int (*get_pages)(struct drm_i915_gem_object *); -void (*put_pages)(struct drm_i915_gem_object *); +struct sg_table *(*get_pages)(struct drm_i915_gem_object *); +void (*put_pages)(struct drm_i915_gem_object *,struct sg_table *);
Idea is that put_pages vfunc does not need struct mutex? Or it acquires it on demand inside it, which means struct mutex will nest inside the mm.lock?
Would it be cleaner to have two vfuncs?
drivers/gpu/drm/i915/i915_drv.h struct drm_i915_gem_object { · line 2317
Quoted 9 lines
drivers/gpu/drm/i915/i915_drv.h +0
-2int (*dmabuf_export)(struct drm_i915_gem_object *);void (*release)(struct drm_i915_gem_object *);@@ -2313,8 +2313,6 @@ struct drm_i915_gem_object {struct i915_gem_userptr {uintptr_t ptr;unsigned read_only :1 ; -unsigned workers :4 ; -#define I915_GEM_USERPTR_MAX_WORKERS 15
Why are you removing this? Perhaps I am misremembering that the reason for having it was to avoid queuing up to many userptr get_pages workers? It cannot happen any longer?
[this is as far as I got to the other day, then I started looking forward in patches to try to pick up the design but it became too much]
Regards,
Tvrtko
Quoted 996 lines
drivers/gpu/drm/i915/i915_drv.h +25
-5struct i915_mm_struct *mm;struct i915_mmu_object *mmu_object;@@ -2376,6 +2374,19 @@ __deprecatedextern void drm_gem_object_unreference_unlocked (struct drm_gem_object *);static inline bool +i915_gem_object_is_dead (const struct drm_i915_gem_object *obj) +{ +return atomic_read (&obj->base.refcount.refcount) ==0 ; +} + +#if IS_ENABLED (CONFIG_LOCKDEP ) +#define lockdep_assert_held_unless (lock, cond) \ + GEM_BUG_ON(!lockdep_is_held(lock) && !(cond)) +#else +#define lockdep_assert_held_unless (lock, cond) +#endif + +static inline bool i915_gem_object_has_struct_page (const struct drm_i915_gem_object *obj) {return obj->ops->flags &I915_GEM_OBJECT_HAS_STRUCT_PAGE ;@@ -3188,6 +3199,8 @@ dma_addr_t i915_gem_object_get_dma_address(struct drm_i915_gem_object *obj, unsigned long n); +void __i915_gem_object_set_pages (struct drm_i915_gem_object *obj, +struct sg_table *pages);int __i915_gem_object_get_pages (struct drm_i915_gem_object *obj); static inline int __must_check@@ -3203,7 +3216,8 @@ i915_gem_object_pin_pages(struct drm_i915_gem_object *obj)static inline void __i915_gem_object_pin_pages (struct drm_i915_gem_object *obj) { -lockdep_assert_held (&obj->base.dev->struct_mutex); +lockdep_assert_held_unless (&obj->base.dev->struct_mutex, +i915_gem_object_is_dead (obj));GEM_BUG_ON (!obj->mm.pages); obj->mm.pages_pin_count++; }@@ -3217,7 +3231,8 @@ i915_gem_object_has_pinned_pages(struct drm_i915_gem_object *obj)static inline void __i915_gem_object_unpin_pages (struct drm_i915_gem_object *obj) { -lockdep_assert_held (&obj->base.dev->struct_mutex); +lockdep_assert_held_unless (&obj->base.dev->struct_mutex, +i915_gem_object_is_dead (obj));GEM_BUG_ON (!i915_gem_object_has_pinned_pages (obj));GEM_BUG_ON (!obj->mm.pages); obj->mm.pages_pin_count--;@@ -3228,7 +3243,8 @@ static inline void i915_gem_object_unpin_pages(struct drm_i915_gem_object *obj)__i915_gem_object_unpin_pages (obj); } -int __i915_gem_object_put_pages (struct drm_i915_gem_object *obj); +void __i915_gem_object_put_pages (struct drm_i915_gem_object *obj); +void __i915_gem_object_invalidate (struct drm_i915_gem_object *obj);enum i915_map_type {I915_MAP_WB =0 ,@@ -3451,8 +3467,10 @@ i915_vma_unpin_fence(struct i915_vma *vma)void i915_gem_restore_fences (struct drm_device *dev);void i915_gem_detect_bit_6_swizzle (struct drm_device *dev); -void i915_gem_object_do_bit_17_swizzle (struct drm_i915_gem_object *obj); -void i915_gem_object_save_bit_17_swizzle (struct drm_i915_gem_object *obj); +void i915_gem_object_do_bit_17_swizzle (struct drm_i915_gem_object *obj, +struct sg_table *pages); +void i915_gem_object_save_bit_17_swizzle (struct drm_i915_gem_object *obj, +struct sg_table *pages);/* i915_gem_context.c */ int __must_check i915_gem_context_init (struct drm_device *dev);drivers/gpu/drm/i915/i915_gem.c +83
-89@@ -169,7 +169,7 @@ i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,return 0 ; } -static int +static struct sg_table *i915_gem_object_get_pages_phys (struct drm_i915_gem_object *obj) {struct address_space *mapping = obj->base.filp->f_mapping;@@ -179,7 +179,7 @@ i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj)int i;if (WARN_ON (i915_gem_object_needs_bit17_swizzle (obj))) -return -EINVAL ; +return ERR_PTR (-EINVAL );for (i =0 ; i < obj->base.size /PAGE_SIZE ; i++) {struct page *page;@@ -187,7 +187,7 @@ i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj) page =shmem_read_mapping_page (mapping, i);if (IS_ERR (page)) -return PTR_ERR (page); +return ERR_CAST (page); src =kmap_atomic (page);memcpy (vaddr, src,PAGE_SIZE );@@ -202,11 +202,11 @@ i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj) st =kmalloc (sizeof (*st),GFP_KERNEL );if (st ==NULL ) -return -ENOMEM ; +return ERR_PTR (-ENOMEM );if (sg_alloc_table (st,1 ,GFP_KERNEL )) {kfree (st); -return -ENOMEM ; +return ERR_PTR (-ENOMEM ); } sg = st->sgl;@@ -216,28 +216,30 @@ i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj)sg_dma_address (sg) = obj->phys_handle->busaddr;sg_dma_len (sg) = obj->base.size; - obj->mm.pages = st; -return 0 ; +return st; }static void -i915_gem_object_put_pages_phys (struct drm_i915_gem_object *obj) +__i915_gem_object_release_shmem (struct drm_i915_gem_object *obj) { -int ret; -GEM_BUG_ON (obj->mm.madv == __I915_MADV_PURGED); - ret =i915_gem_object_set_to_cpu_domain (obj, true); -if (WARN_ON (ret)) { -/* In the event of a disaster, abandon all caches and -* hope for the best. -*/ - obj->base.read_domains = obj->base.write_domain =I915_GEM_DOMAIN_CPU ; - } -if (obj->mm.madv ==I915_MADV_DONTNEED ) obj->mm.dirty = false; +if ((obj->base.read_domains &I915_GEM_DOMAIN_CPU ) ==0 ) +i915_gem_clflush_object (obj, false); + + obj->base.read_domains =I915_GEM_DOMAIN_CPU ; + obj->base.write_domain =I915_GEM_DOMAIN_CPU ; +} + +static void +i915_gem_object_put_pages_phys (struct drm_i915_gem_object *obj, +struct sg_table *pages) +{ +__i915_gem_object_release_shmem (obj); +if (obj->mm.dirty) {struct address_space *mapping = obj->base.filp->f_mapping;char *vaddr = obj->phys_handle->vaddr;@@ -265,8 +267,8 @@ i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj) obj->mm.dirty = false; } -sg_free_table (obj->mm.pages); -kfree (obj->mm.pages); +sg_free_table (pages); +kfree (pages); } static void@@ -517,9 +519,9 @@ i915_gem_object_attach_phys(struct drm_i915_gem_object *obj,if (ret)return ret; - ret =__i915_gem_object_put_pages (obj); -if (ret) -return ret; +__i915_gem_object_put_pages (obj); +if (obj->mm.pages) +return -EBUSY ;/* create a new object */ phys =drm_pci_alloc (obj->base.dev, obj->base.size, align);@@ -535,7 +537,7 @@ i915_gem_object_attach_phys(struct drm_i915_gem_object *obj,static int i915_gem_phys_pwrite (struct drm_i915_gem_object *obj,struct drm_i915_gem_pwrite *args, -struct drm_file *file_priv) +struct drm_file *file) {struct drm_device *dev = obj->base.dev;void *vaddr = obj->phys_handle->vaddr + args->offset;@@ -551,7 +553,7 @@ i915_gem_phys_pwrite(struct drm_i915_gem_object *obj,I915_WAIT_LOCKED |I915_WAIT_ALL ,MAX_SCHEDULE_TIMEOUT , - to_rps_client(file_priv)); + to_rps_client(file));if (ret)return ret;@@ -2225,8 +2227,7 @@ i915_gem_object_truncate(struct drm_i915_gem_object *obj) }/* Try to discard unwanted pages */ -static void -i915_gem_object_invalidate (struct drm_i915_gem_object *obj) +void __i915_gem_object_invalidate (struct drm_i915_gem_object *obj) {struct address_space *mapping;@@ -2245,32 +2246,20 @@ i915_gem_object_invalidate(struct drm_i915_gem_object *obj) }static void -i915_gem_object_put_pages_gtt (struct drm_i915_gem_object *obj) +i915_gem_object_put_pages_gtt (struct drm_i915_gem_object *obj, +struct sg_table *pages) {struct sgt_iter sgt_iter;struct page *page; -int ret; -GEM_BUG_ON (obj->mm.madv == __I915_MADV_PURGED); - - ret =i915_gem_object_set_to_cpu_domain (obj, true); -if (WARN_ON (ret)) { -/* In the event of a disaster, abandon all caches and -* hope for the best. -*/ -i915_gem_clflush_object (obj, true); - obj->base.read_domains = obj->base.write_domain =I915_GEM_DOMAIN_CPU ; - } +__i915_gem_object_release_shmem (obj); -i915_gem_gtt_finish_object (obj); +i915_gem_gtt_finish_pages (obj, pages);if (i915_gem_object_needs_bit17_swizzle (obj)) -i915_gem_object_save_bit_17_swizzle (obj); - -if (obj->mm.madv ==I915_MADV_DONTNEED ) - obj->mm.dirty = false; +i915_gem_object_save_bit_17_swizzle (obj, pages); -for_each_sgt_page (page, sgt_iter, obj->mm.pages) { +for_each_sgt_page (page, sgt_iter, pages) {if (obj->mm.dirty)set_page_dirty (page);@@ -2281,8 +2270,8 @@ i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj) } obj->mm.dirty = false; -sg_free_table (obj->mm.pages); -kfree (obj->mm.pages); +sg_free_table (pages); +kfree (pages); }static void __i915_gem_object_reset_page_iter (struct drm_i915_gem_object *obj)@@ -2294,24 +2283,22 @@ static void __i915_gem_object_reset_page_iter(struct drm_i915_gem_object *obj)radix_tree_delete (&obj->mm.get_page.radix, iter.index); } -int __i915_gem_object_put_pages (struct drm_i915_gem_object *obj) +void __i915_gem_object_put_pages (struct drm_i915_gem_object *obj) { -const struct drm_i915_gem_object_ops *ops = obj->ops; +struct sg_table *pages;lockdep_assert_held (&obj->base.dev->struct_mutex); -if (!obj->mm.pages) -return 0 ; -if (i915_gem_object_has_pinned_pages (obj)) -return -EBUSY ; +return ;GEM_BUG_ON (obj->bind_count);/* ->put_pages might need to allocate memory for the bit17 swizzle * array, hence protect them from being reaped by removing them from gtt * lists early. */ -list_del (&obj->global_list); + pages =fetch_and_zero (&obj->mm.pages); +GEM_BUG_ON (!pages);if (obj->mm.mapping) {void *ptr;@@ -2327,15 +2314,10 @@ int __i915_gem_object_put_pages(struct drm_i915_gem_object *obj)__i915_gem_object_reset_page_iter (obj); - ops->put_pages (obj); - obj->mm.pages =NULL ; - -i915_gem_object_invalidate (obj); - -return 0 ; + obj->ops->put_pages (obj, pages); } -static int +static struct sg_table *i915_gem_object_get_pages_gtt (struct drm_i915_gem_object *obj) {struct drm_i915_private *dev_priv =to_i915 (obj->base.dev);@@ -2353,17 +2335,17 @@ i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)* wasn't in the GTT, there shouldn't be any way it could have been in * a GPU cache */ -BUG_ON (obj->base.read_domains &I915_GEM_GPU_DOMAINS ); -BUG_ON (obj->base.write_domain &I915_GEM_GPU_DOMAINS ); +GEM_BUG_ON (obj->base.read_domains &I915_GEM_GPU_DOMAINS ); +GEM_BUG_ON (obj->base.write_domain &I915_GEM_GPU_DOMAINS ); st =kmalloc (sizeof (*st),GFP_KERNEL );if (st ==NULL ) -return -ENOMEM ; +return ERR_PTR (-ENOMEM ); page_count = obj->base.size /PAGE_SIZE ;if (sg_alloc_table (st, page_count,GFP_KERNEL )) {kfree (st); -return -ENOMEM ; +return ERR_PTR (-ENOMEM ); }/* Get the list of pages out of our struct file. They'll be pinned @@ -2423,20 +2405,19 @@ i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj) if (!swiotlb_nr_tbl())#endif sg_mark_end (sg); - obj->mm.pages = st; - ret =i915_gem_gtt_prepare_object (obj); + ret =i915_gem_gtt_prepare_pages (obj, st);if (ret) goto err_pages;if (i915_gem_object_needs_bit17_swizzle (obj)) -i915_gem_object_do_bit_17_swizzle (obj); +i915_gem_object_do_bit_17_swizzle (obj, st);if (i915_gem_object_is_tiled (obj) && dev_priv->quirks &QUIRK_PIN_SWIZZLED_PAGES )__i915_gem_object_pin_pages (obj); -return 0 ; +return st; err_pages:sg_mark_end (sg);@@ -2456,7 +2437,35 @@ err_pages:if (ret == -ENOSPC ) ret = -ENOMEM ; -return ret; +return ERR_PTR (ret); +} + +void __i915_gem_object_set_pages (struct drm_i915_gem_object *obj, +struct sg_table *pages) +{ +lockdep_assert_held (&obj->base.dev->struct_mutex); + + obj->mm.get_page.sg_pos = pages->sgl; + obj->mm.get_page.sg_idx =0 ; + + obj->mm.pages = pages; +} + +static int ____i915_gem_object_get_pages (struct drm_i915_gem_object *obj) +{ +struct sg_table *pages; + +if (unlikely (obj->mm.madv !=I915_MADV_WILLNEED )) { +DRM_DEBUG ("Attempting to obtain a purgeable object\n" ); +return -EFAULT ; + } + + pages = obj->ops->get_pages (obj); +if (unlikely (IS_ERR (pages))) +return PTR_ERR (pages); + +__i915_gem_object_set_pages (obj, pages); +return 0 ; }/* Ensure that the associated pages are gathered from the backing storage @@ -2468,33 +2477,18 @@ err_pages:*/ int __i915_gem_object_get_pages (struct drm_i915_gem_object *obj) { -struct drm_i915_private *dev_priv =to_i915 (obj->base.dev); -const struct drm_i915_gem_object_ops *ops = obj->ops; -int ret; +int err;lockdep_assert_held (&obj->base.dev->struct_mutex);if (obj->mm.pages)return 0 ; -if (obj->mm.madv !=I915_MADV_WILLNEED ) { -DRM_DEBUG ("Attempting to obtain a purgeable object\n" ); -__i915_gem_object_unpin_pages (obj); -return -EFAULT ; - } - - ret = ops->get_pages (obj); -if (ret) { + err =____i915_gem_object_get_pages (obj); +if (err)__i915_gem_object_unpin_pages (obj); -return ret; - } -list_add_tail (&obj->global_list, &dev_priv->mm.unbound_list); - - obj->mm.get_page.sg_pos = obj->mm.pages->sgl; - obj->mm.get_page.sg_idx =0 ; - -return 0 ; +return err; }/* The 'mapping' part of i915_gem_object_pin_map() below */ drivers/gpu/drm/i915/i915_gem_dmabuf.c +8
-12@@ -289,22 +289,18 @@ struct dma_buf *i915_gem_prime_export(struct drm_device *dev,return dma_buf; } -static int i915_gem_object_get_pages_dmabuf (struct drm_i915_gem_object *obj) +static struct sg_table * +i915_gem_object_get_pages_dmabuf (struct drm_i915_gem_object *obj) { -struct sg_table *sg; - - sg =dma_buf_map_attachment (obj->base.import_attach,DMA_BIDIRECTIONAL ); -if (IS_ERR (sg)) -return PTR_ERR (sg); - - obj->mm.pages = sg; -return 0 ; +return dma_buf_map_attachment (obj->base.import_attach, +DMA_BIDIRECTIONAL ); } -static void i915_gem_object_put_pages_dmabuf (struct drm_i915_gem_object *obj) +static void i915_gem_object_put_pages_dmabuf (struct drm_i915_gem_object *obj, +struct sg_table *pages) { -dma_buf_unmap_attachment (obj->base.import_attach, - obj->mm.pages,DMA_BIDIRECTIONAL ); +dma_buf_unmap_attachment (obj->base.import_attach, pages, +DMA_BIDIRECTIONAL ); }static const struct drm_i915_gem_object_ops i915_gem_object_dmabuf_ops = {drivers/gpu/drm/i915/i915_gem_fence.c +10
-8@@ -628,6 +628,7 @@ i915_gem_swizzle_page(struct page *page)/** * i915_gem_object_do_bit_17_swizzle - fixup bit 17 swizzling * @obj: i915 GEM buffer object +* @pages: the scattergather list of physical pages * * This function fixes up the swizzling in case any page frame number for this * object has changed in bit 17 since that state has been saved with @@ -638,7 +639,8 @@ i915_gem_swizzle_page(struct page *page)* by swapping them out and back in again). */ void -i915_gem_object_do_bit_17_swizzle (struct drm_i915_gem_object *obj) +i915_gem_object_do_bit_17_swizzle (struct drm_i915_gem_object *obj, +struct sg_table *pages) {struct sgt_iter sgt_iter;struct page *page;@@ -648,10 +650,9 @@ i915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object *obj)return ; i =0 ; -for_each_sgt_page (page, sgt_iter, obj->mm.pages) { +for_each_sgt_page (page, sgt_iter, pages) {char new_bit_17 =page_to_phys (page) >>17 ; -if ((new_bit_17 &0x1 ) != - (test_bit (i, obj->bit_17) !=0 )) { +if ((new_bit_17 &0x1 ) != (test_bit (i, obj->bit_17) !=0 )) {i915_gem_swizzle_page (page);set_page_dirty (page); }@@ -662,21 +663,22 @@ i915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object *obj)/** * i915_gem_object_save_bit_17_swizzle - save bit 17 swizzling * @obj: i915 GEM buffer object +* @pages: the scattergather list of physical pages * * This function saves the bit 17 of each page frame number so that swizzling * can be fixed up later on with i915_gem_object_do_bit_17_swizzle(). This must * be called before the backing storage can be unpinned. */ void -i915_gem_object_save_bit_17_swizzle (struct drm_i915_gem_object *obj) +i915_gem_object_save_bit_17_swizzle (struct drm_i915_gem_object *obj, +struct sg_table *pages) {struct sgt_iter sgt_iter;struct page *page; -int page_count = obj->base.size >>PAGE_SHIFT ;int i;if (obj->bit_17 ==NULL ) { - obj->bit_17 =kcalloc (BITS_TO_LONGS (page_count), + obj->bit_17 =kcalloc (BITS_TO_LONGS (obj->base.size >>PAGE_SHIFT ),sizeof (long ),GFP_KERNEL );if (obj->bit_17 ==NULL ) { DRM_ERROR("Failed to allocate memory for bit 17 "@@ -687,7 +689,7 @@ i915_gem_object_save_bit_17_swizzle(struct drm_i915_gem_object *obj) i =0 ; -for_each_sgt_page (page, sgt_iter, obj->mm.pages) { +for_each_sgt_page (page, sgt_iter, pages) {if (page_to_phys (page) & (1 <<17 ))__set_bit (i, obj->bit_17);else drivers/gpu/drm/i915/i915_gem_gtt.c +10
-9@@ -2292,14 +2292,15 @@ void i915_gem_suspend_gtt_mappings(struct drm_device *dev)i915_ggtt_flush (dev_priv); } -int i915_gem_gtt_prepare_object (struct drm_i915_gem_object *obj) +int i915_gem_gtt_prepare_pages (struct drm_i915_gem_object *obj, +struct sg_table *pages) { -if (!dma_map_sg (&obj->base.dev->pdev->dev, - obj->mm.pages->sgl, obj->mm.pages->nents, -PCI_DMA_BIDIRECTIONAL )) -return -ENOSPC ; +if (dma_map_sg (&obj->base.dev->pdev->dev, + pages->sgl, pages->nents, +PCI_DMA_BIDIRECTIONAL )) +return 0 ; -return 0 ; +return -ENOSPC ; }static void gen8_set_pte (void __iomem *addr,gen8_pte_t pte)@@ -2671,7 +2672,8 @@ static void ggtt_unbind_vma(struct i915_vma *vma) true); } -void i915_gem_gtt_finish_object (struct drm_i915_gem_object *obj) +void i915_gem_gtt_finish_pages (struct drm_i915_gem_object *obj, +struct sg_table *pages) {struct drm_i915_private *dev_priv =to_i915 (obj->base.dev);struct device *kdev = &dev_priv->drm.pdev->dev;@@ -2685,8 +2687,7 @@ void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj) } } -dma_unmap_sg (kdev, obj->mm.pages->sgl, obj->mm.pages->nents, -PCI_DMA_BIDIRECTIONAL ); +dma_unmap_sg (kdev, pages->sgl, pages->nents,PCI_DMA_BIDIRECTIONAL ); } static void i915_gtt_color_adjust(struct drm_mm_node *node,drivers/gpu/drm/i915/i915_gem_gtt.h +4
-2@@ -629,8 +629,10 @@ void i915_check_and_clear_faults(struct drm_i915_private *dev_priv);void i915_gem_suspend_gtt_mappings (struct drm_device *dev);void i915_gem_restore_gtt_mappings (struct drm_device *dev); -int __must_check i915_gem_gtt_prepare_object (struct drm_i915_gem_object *obj); -void i915_gem_gtt_finish_object (struct drm_i915_gem_object *obj); +int __must_check i915_gem_gtt_prepare_pages (struct drm_i915_gem_object *obj, +struct sg_table *pages); +void i915_gem_gtt_finish_pages (struct drm_i915_gem_object *obj, +struct sg_table *pages);/* Flags used by pin/bind&friends. */ #define PIN_NONBLOCK BIT(0)drivers/gpu/drm/i915/i915_gem_internal.c +11
-11@@ -38,7 +38,8 @@ static void internal_free_pages(struct sg_table *st)kfree (st); } -static int i915_gem_object_get_pages_internal (struct drm_i915_gem_object *obj) +static struct sg_table * +i915_gem_object_get_pages_internal (struct drm_i915_gem_object *obj) {const unsigned int npages = obj->base.size /PAGE_SIZE ;struct sg_table *st;@@ -49,11 +50,11 @@ static int i915_gem_object_get_pages_internal(struct drm_i915_gem_object *obj) st =kmalloc (sizeof (*st),GFP_KERNEL );if (!st) -return -ENOMEM ; +return ERR_PTR (-ENOMEM );if (sg_alloc_table (st, npages,GFP_KERNEL )) {kfree (st); -return -ENOMEM ; +return ERR_PTR (-ENOMEM ); } sg = st->sgl;@@ -95,12 +96,9 @@ static int i915_gem_object_get_pages_internal(struct drm_i915_gem_object *obj) if (!swiotlb_nr_tbl())#endif sg_mark_end (sg); - obj->mm.pages = st; -if (i915_gem_gtt_prepare_object (obj)) { - obj->mm.pages =NULL ; +if (i915_gem_gtt_prepare_pages (obj, st)) goto err; - }/* Mark the pages as dontneed whilst they are still pinned. As soon * as they are unpinned they are allowed to be reaped by the shrinker, @@ -108,17 +106,19 @@ static int i915_gem_object_get_pages_internal(struct drm_i915_gem_object *obj)* object are only valid whilst active and pinned. */ obj->mm.madv =I915_MADV_DONTNEED ; -return 0 ; +return st; err:sg_mark_end (sg);internal_free_pages (st); -return -ENOMEM ; +return ERR_PTR (-ENOMEM ); } -static void i915_gem_object_put_pages_internal (struct drm_i915_gem_object *obj) +static void i915_gem_object_put_pages_internal (struct drm_i915_gem_object *obj, +struct sg_table *pages) { -internal_free_pages (obj->mm.pages); +i915_gem_gtt_finish_pages (obj, pages); +internal_free_pages (pages); obj->mm.dirty = false; obj->mm.madv =I915_MADV_WILLNEED ;drivers/gpu/drm/i915/i915_gem_shrinker.c +8
-3@@ -91,6 +91,13 @@ static bool can_release_pages(struct drm_i915_gem_object *obj)return swap_available () || obj->mm.madv ==I915_MADV_DONTNEED ; } +static bool unsafe_drop_pages (struct drm_i915_gem_object *obj) +{ +if (i915_gem_object_unbind (obj) ==0 ) +__i915_gem_object_put_pages (obj); +return !obj->mm.pages; +} +/** * i915_gem_shrink - Shrink buffer object caches * @dev_priv: i915 device @@ -191,9 +198,7 @@ i915_gem_shrink(struct drm_i915_private *dev_priv,i915_gem_object_get (obj); -/* For the unbound phase, this should be a no-op! */ -i915_gem_object_unbind (obj); -if (__i915_gem_object_put_pages (obj) ==0 ) +if (unsafe_drop_pages (obj)) count += obj->base.size >>PAGE_SHIFT ;i915_gem_object_put (obj);drivers/gpu/drm/i915/i915_gem_stolen.c +21
-22@@ -545,17 +545,20 @@ i915_pages_create_for_stolen(struct drm_device *dev,return st; } -static int i915_gem_object_get_pages_stolen (struct drm_i915_gem_object *obj) +static struct sg_table * +i915_gem_object_get_pages_stolen (struct drm_i915_gem_object *obj) { -BUG (); -return -EINVAL ; +return i915_pages_create_for_stolen (obj->base.dev, + obj->stolen->start, + obj->stolen->size); } -static void i915_gem_object_put_pages_stolen (struct drm_i915_gem_object *obj) +static void i915_gem_object_put_pages_stolen (struct drm_i915_gem_object *obj, +struct sg_table *pages) {/* Should only be called during free */ -sg_free_table (obj->mm.pages); -kfree (obj->mm.pages); +sg_free_table (pages); +kfree (pages); } static void@@ -590,21 +593,13 @@ _i915_gem_object_create_stolen(struct drm_device *dev,drm_gem_private_object_init (dev, &obj->base, stolen->size);i915_gem_object_init (obj, &i915_gem_object_stolen_ops); - obj->mm.pages =i915_pages_create_for_stolen (dev, - stolen->start, - stolen->size); -if (!obj->mm.pages) - goto cleanup; - - obj->mm.get_page.sg_pos = obj->mm.pages->sgl; - obj->mm.get_page.sg_idx =0 ; - -__i915_gem_object_pin_pages (obj); obj->stolen = stolen; - obj->base.read_domains =I915_GEM_DOMAIN_CPU |I915_GEM_DOMAIN_GTT ; obj->cache_level =HAS_LLC (dev) ?I915_CACHE_LLC :I915_CACHE_NONE ; +if (i915_gem_object_pin_pages (obj)) + goto cleanup; +return obj; cleanup:@@ -699,10 +694,14 @@ i915_gem_object_create_stolen_for_preallocated(struct drm_device *dev,if (gtt_offset ==I915_GTT_OFFSET_NONE )return obj; + ret =i915_gem_object_pin_pages (obj); +if (ret) + goto err; + vma =i915_gem_obj_lookup_or_create_vma (obj, &ggtt->base,NULL );if (IS_ERR (vma)) { ret =PTR_ERR (vma); - goto err; + goto err_pages; }/* To simplify the initialisation sequence between KMS and GTT, @@ -716,20 +715,20 @@ i915_gem_object_create_stolen_for_preallocated(struct drm_device *dev, ret =drm_mm_reserve_node (&ggtt->base.mm, &vma->node);if (ret) {DRM_DEBUG_KMS ("failed to allocate stolen GTT space\n" ); - goto err; + goto err_pages; } vma->pages = obj->mm.pages; vma->flags |=I915_VMA_GLOBAL_BIND ;__i915_vma_set_map_and_fenceable (vma);list_move_tail (&vma->vm_link, &ggtt->base.inactive_list); +list_move_tail (&obj->global_list, &dev_priv->mm.bound_list); obj->bind_count++; -list_add_tail (&obj->global_list, &dev_priv->mm.bound_list); -__i915_gem_object_pin_pages (obj); -return obj; +err_pages: +i915_gem_object_unpin_pages (obj); err:i915_gem_object_put (obj);return NULL ;drivers/gpu/drm/i915/i915_gem_userptr.c +43
-41@@ -73,11 +73,14 @@ static void cancel_userptr(struct work_struct *work)/* Cancel any active worker and force us to re-evaluate gup */ obj->userptr.work =NULL ; -if (obj->mm.pages) { -/* We are inside a kthread context and can't be interrupted */ -WARN_ON (i915_gem_object_unbind (obj)); -WARN_ON (__i915_gem_object_put_pages (obj)); - } +/* We are inside a kthread context and can't be interrupted */ +if (i915_gem_object_unbind (obj) ==0 ) +__i915_gem_object_put_pages (obj); +WARN_ONCE (obj->mm.pages, +"Failed to release pages: bind_count=%d, pages_pin_count=%d, pin_display=%d\n" , + obj->bind_count, + obj->mm.pages_pin_count, + obj->pin_display);i915_gem_object_put (obj);mutex_unlock (&dev->struct_mutex);@@ -426,24 +429,25 @@ err:return ret; } -static int +static struct sg_table *__i915_gem_userptr_set_pages (struct drm_i915_gem_object *obj,struct page **pvec,int num_pages) { +struct sg_table *pages;int ret; - ret =st_set_pages (&obj->mm.pages, pvec, num_pages); + ret =st_set_pages (&pages, pvec, num_pages);if (ret) -return ret; +return ERR_PTR (ret); - ret =i915_gem_gtt_prepare_object (obj); + ret =i915_gem_gtt_prepare_pages (obj, pages);if (ret) { -sg_free_table (obj->mm.pages); -kfree (obj->mm.pages); - obj->mm.pages =NULL ; +sg_free_table (pages); +kfree (pages); +return ERR_PTR (ret); } -return ret; +return pages; } static int@@ -521,20 +525,20 @@ __i915_gem_userptr_get_pages_worker(struct work_struct *_work)mutex_lock (&dev->struct_mutex);if (obj->userptr.work == &work->work) { +struct sg_table *pages =ERR_PTR (ret); +if (pinned == npages) { - ret =__i915_gem_userptr_set_pages (obj, pvec, npages); -if (ret ==0 ) { -list_add_tail (&obj->global_list, - &to_i915 (dev)->mm.unbound_list); - obj->mm.get_page.sg_pos = obj->mm.pages->sgl; - obj->mm.get_page.sg_idx =0 ; + pages =__i915_gem_userptr_set_pages (obj, pvec, npages); +if (!IS_ERR (pages)) { +__i915_gem_object_set_pages (obj, pages); pinned =0 ; + pages =NULL ; } } - obj->userptr.work =ERR_PTR (ret); + + obj->userptr.work =ERR_CAST (pages); } - obj->userptr.workers--;i915_gem_object_put (obj);mutex_unlock (&dev->struct_mutex);@@ -545,7 +549,7 @@ __i915_gem_userptr_get_pages_worker(struct work_struct *_work)kfree (work); } -static int +static struct sg_table *__i915_gem_userptr_get_pages_schedule (struct drm_i915_gem_object *obj,bool *active) {@@ -570,15 +574,11 @@ __i915_gem_userptr_get_pages_schedule(struct drm_i915_gem_object *obj,* that error back to this function through * obj->userptr.work = ERR_PTR. */ -if (obj->userptr.workers >=I915_GEM_USERPTR_MAX_WORKERS ) -return -EAGAIN ; - work =kmalloc (sizeof (*work),GFP_KERNEL );if (work ==NULL ) -return -ENOMEM ; +return ERR_PTR (-ENOMEM ); obj->userptr.work = &work->work; - obj->userptr.workers++; work->obj =i915_gem_object_get (obj);@@ -589,14 +589,15 @@ __i915_gem_userptr_get_pages_schedule(struct drm_i915_gem_object *obj,schedule_work (&work->work); *active = true; -return -EAGAIN ; +return ERR_PTR (-EAGAIN ); } -static int +static struct sg_table *i915_gem_userptr_get_pages (struct drm_i915_gem_object *obj) {const int num_pages = obj->base.size >>PAGE_SHIFT ;struct page **pvec; +struct sg_table *pages;int pinned, ret;bool active;@@ -620,15 +621,15 @@ i915_gem_userptr_get_pages(struct drm_i915_gem_object *obj)if (obj->userptr.work) {/* active flag should still be held for the pending work */ if (IS_ERR (obj->userptr.work)) -return PTR_ERR (obj->userptr.work); +return ERR_CAST (obj->userptr.work);else -return -EAGAIN ; +return ERR_PTR (-EAGAIN ); }/* Let the mmu-notifier know that we have begun and need cancellation */ ret =__i915_gem_userptr_set_active (obj, true);if (ret) -return ret; +return ERR_PTR (ret); pvec =NULL ; pinned =0 ;@@ -637,7 +638,7 @@ i915_gem_userptr_get_pages(struct drm_i915_gem_object *obj) GFP_TEMPORARY);if (pvec ==NULL ) {__i915_gem_userptr_set_active (obj, false); -return -ENOMEM ; +return ERR_PTR (-ENOMEM ); } pinned = __get_user_pages_fast(obj->userptr.ptr, num_pages,@@ -646,21 +647,22 @@ i915_gem_userptr_get_pages(struct drm_i915_gem_object *obj) active = false;if (pinned <0 ) - ret = pinned, pinned =0 ; + pages =ERR_PTR (pinned), pinned =0 ;else if (pinned < num_pages) - ret =__i915_gem_userptr_get_pages_schedule (obj, &active); + pages =__i915_gem_userptr_get_pages_schedule (obj, &active);else - ret =__i915_gem_userptr_set_pages (obj, pvec, num_pages); -if (ret) { + pages =__i915_gem_userptr_set_pages (obj, pvec, num_pages); +if (IS_ERR (pages)) {__i915_gem_userptr_set_active (obj, active);release_pages (pvec, pinned,0 ); }drm_free_large (pvec); -return ret; +return pages; }static void -i915_gem_userptr_put_pages (struct drm_i915_gem_object *obj) +i915_gem_userptr_put_pages (struct drm_i915_gem_object *obj, +struct sg_table *pages) {struct sgt_iter sgt_iter;struct page *page;@@ -671,9 +673,9 @@ i915_gem_userptr_put_pages(struct drm_i915_gem_object *obj)if (obj->mm.madv !=I915_MADV_WILLNEED ) obj->mm.dirty = false; -i915_gem_gtt_finish_object (obj); +i915_gem_gtt_finish_pages (obj, pages); -for_each_sgt_page (page, sgt_iter, obj->mm.pages) { +for_each_sgt_page (page, sgt_iter, pages) {if (obj->mm.dirty)set_page_dirty (page);
drivers/gpu/drm/i915/i915_gem_userptr.c i915_gem_userptr_put_pages(struct drm_i915_gem_object *obj) · line 691
Quoted 10 lines
drivers/gpu/drm/i915/i915_gem_userptr.c +2
-2@@ -682,8 +684,8 @@ i915_gem_userptr_put_pages(struct drm_i915_gem_object *obj) } obj->mm.dirty = false; -sg_free_table (obj->mm.pages); -kfree (obj->mm.pages); +sg_free_table (pages); +kfree (pages); } static void
_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
Re: [PATCH 17/42] drm/i915: Pass around sg_table to get_pages/put_pages backend
Quoted 8 lines · Tvrtko Ursulin
diff --git
a/drivers/gpu/drm/i915/i915_drv.hb/drivers/gpu/drm/i915/i915_drv.hindex3c22d49005fe..271e63c8f037100644
---a/drivers/gpu/drm/i915/i915_drv.h
+++b/drivers/gpu/drm/i915/i915_drv.h
@@ -2175,8 +2175,8 @@ structdrm_i915_gem_object_ops{
* being released or under memory pressure (where we attempt to
* reap pages for the shrinker).
*/
Quoted 8 lines
- int (*get_pages)(struct drm_i915_gem_object *); - void (*put_pages)(struct drm_i915_gem_object *); + struct sg_table *(*get_pages)(struct drm_i915_gem_object *); + void (*put_pages)(struct drm_i915_gem_object *, struct sg_table *);Idea is that
put_pagesvfunc does not need struct mutex? Or it acquires it on demand inside it, which means struct mutex will nest inside the mm.lock?
Right, it loses the mutex within get/put pages and I am no longer concerned about the number of workers. Though that is mainly because of another patch to improve execbuf + userptr workloads.
-Chris
Signature · 5 lines
Chris Wilson, Intel Open Source Technology Centre _______________________________________________ Intel-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/intel-gfx
Re: [PATCH 17/42] drm/i915: Pass around sg_table to get_pages/put_pages backend
Quoted 14 lines · Chris Wilson
Quoted 13 lines · Tvrtko Ursulin
diff --git
a/drivers/gpu/drm/i915/i915_drv.hb/drivers/gpu/drm/i915/i915_drv.hindex3c22d49005fe..271e63c8f037100644
---a/drivers/gpu/drm/i915/i915_drv.h
+++b/drivers/gpu/drm/i915/i915_drv.h
@@ -2175,8 +2175,8 @@ structdrm_i915_gem_object_ops{
* being released or under memory pressure (where we attempt to
* reap pages for the shrinker).
*/- int (*get_pages)(struct drm_i915_gem_object *); - void (*put_pages)(struct drm_i915_gem_object *); + struct sg_table *(*get_pages)(struct drm_i915_gem_object *); + void (*put_pages)(struct drm_i915_gem_object *, struct sg_table *);
Quoted 6 lines
Quoted 3 lines
Idea is that
put_pagesvfunc does not need struct mutex? Or it acquires it on demand inside it, which means struct mutex will nest inside the mm.lock?Right, it loses the mutex within get/put pages and I am no longer concerned about the number of workers. Though that is mainly because of another patch to improve execbuf + userptr workloads.
Is that other patch before or after this one? Or in a different series altogether? In any case, why should this change be in this patch, especially when it is not commented anywhere?
Regards,
Tvrtko
_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
Re: [PATCH 17/42] drm/i915: Pass around sg_table to get_pages/put_pages backend
Quoted 19 lines · Tvrtko Ursulin
Quoted 17 lines · Chris Wilson
Quoted 16 lines · Tvrtko Ursulin
diff --git
a/drivers/gpu/drm/i915/i915_drv.hb/drivers/gpu/drm/i915/i915_drv.hindex3c22d49005fe..271e63c8f037100644
---a/drivers/gpu/drm/i915/i915_drv.h
+++b/drivers/gpu/drm/i915/i915_drv.h
@@ -2175,8 +2175,8 @@ structdrm_i915_gem_object_ops{
* being released or under memory pressure (where we attempt to
* reap pages for the shrinker).
*/- int (*get_pages)(struct drm_i915_gem_object *); - void (*put_pages)(struct drm_i915_gem_object *); + struct sg_table *(*get_pages)(struct drm_i915_gem_object *); + void (*put_pages)(struct drm_i915_gem_object *, struct sg_table *);Idea is that
put_pagesvfunc does not need struct mutex? Or it acquires it on demand inside it, which means struct mutex will nest inside the mm.lock?
Quoted 7 lines
Quoted 3 lines
Right, it loses the mutex within get/put pages and I am no longer concerned about the number of workers. Though that is mainly because of another patch to improve execbuf + userptr workloads.
Is that other patch before or after this one? Or in a different series altogether? In any case, why should this change be in this patch, especially when it is not commented anywhere?
Actually, it follows from
commit 364c8172edb5ff0b4650bbd4c45eaab46b84b008
Author: Chris Wilson <[email protected]> Date: Thu Aug 18 17:16:58 2016 +0100
drm/i915/userptr: Make gup errors stickier
which was the prelude to the patch I was thinking of. The current code already doesn't use more than one worker simultaneously. -Chris
Signature · 5 lines
Chris Wilson, Intel Open Source Technology Centre _______________________________________________ Intel-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH 18/42 drm/i915: Move object backing storage manipulation to its own locking
In reply to Chris WilsonBreak the allocation of the backing storage away from struct_mutex into a per-object lock. This allows parallel page allocation, provided we can do so outside of struct_mutex (i.e. set-domain-ioctl, pwrite, GTT fault), i.e. before execbuf! The increased cost of the atomic counters are hidden behind i915_vma_pin() for the typical case of execbuf, i.e. as the object is typically bound between execbufs, the page_pin_count is static. The cost will be felt around set-domain and pwrite, but offset by the improvement from reduced struct_mutex contention.
--- drivers/gpu/drm/i915/i915_drv.h | 26 ++++------ drivers/gpu/drm/i915/i915_gem.c | 89 +++++++++++++++++++++----------- drivers/gpu/drm/i915/i915_gem_shrinker.c | 51 ++++++++++++------ drivers/gpu/drm/i915/i915_gem_tiling.c | 2 + drivers/gpu/drm/i915/i915_gem_userptr.c | 10 ++-- 5 files changed, 110 insertions(+), 68 deletions(-)
drivers/gpu/drm/i915/i915_drv.h +10 -16
@@ -2266,7 +2266,8 @@ struct drm_i915_gem_object {unsigned int pin_display;struct { -unsigned int pages_pin_count; +struct mutex lock; +atomic_t pages_pin_count;struct sg_table *pages;void *mapping;
@@ -3206,8 +3207,7 @@ int __i915_gem_object_get_pages(struct drm_i915_gem_object *obj);static inline int __must_check i915_gem_object_pin_pages (struct drm_i915_gem_object *obj) { -lockdep_assert_held (&obj->base.dev->struct_mutex); -if (obj->mm.pages_pin_count++) +if (atomic_inc_not_zero (&obj->mm.pages_pin_count))return 0 ;return __i915_gem_object_get_pages (obj);
@@ -3216,29 +3216,26 @@ i915_gem_object_pin_pages(struct drm_i915_gem_object *obj)static inline void __i915_gem_object_pin_pages (struct drm_i915_gem_object *obj) { -lockdep_assert_held_unless (&obj->base.dev->struct_mutex, -i915_gem_object_is_dead (obj));GEM_BUG_ON (!obj->mm.pages); - obj->mm.pages_pin_count++; +atomic_inc (&obj->mm.pages_pin_count); }static inline bool i915_gem_object_has_pinned_pages (struct drm_i915_gem_object *obj) { -return obj->mm.pages_pin_count; +return atomic_read (&obj->mm.pages_pin_count); }static inline void __i915_gem_object_unpin_pages (struct drm_i915_gem_object *obj) { -lockdep_assert_held_unless (&obj->base.dev->struct_mutex, -i915_gem_object_is_dead (obj));GEM_BUG_ON (!i915_gem_object_has_pinned_pages (obj));GEM_BUG_ON (!obj->mm.pages); - obj->mm.pages_pin_count--; +atomic_dec (&obj->mm.pages_pin_count); } -static inline void i915_gem_object_unpin_pages (struct drm_i915_gem_object *obj) +static inline void +i915_gem_object_unpin_pages (struct drm_i915_gem_object *obj) {__i915_gem_object_unpin_pages (obj); }
@@ -3261,8 +3258,8 @@ enum i915_map_type {* the kernel address space. Based on the @type of mapping, the PTE will be * set to either WriteBack or WriteCombine (via pgprot_t). * -* The caller must hold the struct_mutex, and is responsible for calling -* i915_gem_object_unpin_map() when the mapping is no longer required. +* The caller is responsible for calling i915_gem_object_unpin_map() when the +* mapping is no longer required. * * Returns the pointer through which to access the mapped object, or an * ERR_PTR() on error.
@@ -3278,12 +3275,9 @@ void *__must_check i915_gem_object_pin_map(struct drm_i915_gem_object *obj,* with your access, call i915_gem_object_unpin_map() to release the pin * upon the mapping. Once the pin count reaches zero, that mapping may be * removed. -* -* The caller must hold the struct_mutex. */ static inline void i915_gem_object_unpin_map (struct drm_i915_gem_object *obj) { -lockdep_assert_held (&obj->base.dev->struct_mutex);i915_gem_object_unpin_pages (obj); }
drivers/gpu/drm/i915/i915_gem.c +60 -29
190 lines not on this page · raw message
drivers/gpu/drm/i915/i915_gem_shrinker.c +34 -17
99 lines not on this page · raw message
drivers/gpu/drm/i915/i915_gem_tiling.c +2 -0
@@ -260,6 +260,7 @@ i915_gem_set_tiling(struct drm_device *dev, void *data,if (!err) {struct i915_vma *vma; +mutex_lock (&obj->mm.lock);if (obj->mm.pages && obj->mm.madv ==I915_MADV_WILLNEED && dev_priv->quirks &QUIRK_PIN_SWIZZLED_PAGES ) {
@@ -268,6 +269,7 @@ i915_gem_set_tiling(struct drm_device *dev, void *data,if (!i915_gem_object_is_tiled (obj))__i915_gem_object_pin_pages (obj); } +mutex_unlock (&obj->mm.lock);list_for_each_entry (vma, &obj->vma_list, obj_link) {if (!vma->fence)
drivers/gpu/drm/i915/i915_gem_userptr.c +4 -6
@@ -79,7 +79,7 @@ static void cancel_userptr(struct work_struct *work)WARN_ONCE (obj->mm.pages,"Failed to release pages: bind_count=%d, pages_pin_count=%d, pin_display=%d\n" , obj->bind_count, - obj->mm.pages_pin_count, +atomic_read (&obj->mm.pages_pin_count), obj->pin_display);i915_gem_object_put (obj);
@@ -491,7 +491,6 @@ __i915_gem_userptr_get_pages_worker(struct work_struct *_work) {struct get_pages_work *work =container_of (_work,typeof (*work), work);struct drm_i915_gem_object *obj = work->obj; -struct drm_device *dev = obj->base.dev;const int npages = obj->base.size >>PAGE_SHIFT ;struct page **pvec;int pinned, ret;
@@ -523,7 +522,7 @@ __i915_gem_userptr_get_pages_worker(struct work_struct *_work) } } -mutex_lock (&dev->struct_mutex); +mutex_lock (&obj->mm.lock);if (obj->userptr.work == &work->work) {struct sg_table *pages =ERR_PTR (ret);
@@ -538,13 +537,12 @@ __i915_gem_userptr_get_pages_worker(struct work_struct *_work) obj->userptr.work =ERR_CAST (pages); } - -i915_gem_object_put (obj); -mutex_unlock (&dev->struct_mutex); +mutex_unlock (&obj->mm.lock);release_pages (pvec, pinned,0 );drm_free_large (pvec); +i915_gem_object_put_unlocked (obj);put_task_struct (work->task);kfree (work); }
Signature · 6 lines
2.9.3 _______________________________________________ Intel-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/intel-gfx
Re: [PATCH 18/42] drm/i915: Move object backing storage manipulation to its own locking
Quoted 7 lines · Chris Wilson
(quoted diff) +0
-0@@ -4211,10 +4240,10 @@ i915_gem_madvise_ioctl(struct drm_device *dev, void *data, i915_gem_object_truncate(obj);
args->retained=obj->mm.madv!=__I915_MADV_PURGED;(quoted diff) +1
-0+ mutex_unlock(&obj->mm.lock);
+err:
Shared, so 'out:'
Other than that, looks good.
Am I to assume that now when we introduced a new lock, every single function that assumes the lock taken is now tagged with assert_lockdep_held?
Regards, Joonas
Signature · 7 lines
Joonas Lahtinen Open Source Technology Center Intel Corporation _______________________________________________ Intel-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/intel-gfx
Re: [PATCH 18/42] drm/i915: Move object backing storage manipulation to its own locking
Quoted 11 lines · Joonas Lahtinen
Quoted 7 lines · Chris Wilson
(quoted diff) +0
-0@@ -4211,10 +4240,10 @@ i915_gem_madvise_ioctl(struct drm_device *dev, void *data, i915_gem_object_truncate(obj);
args->retained=obj->mm.madv!=__I915_MADV_PURGED;(quoted diff) +1
-0+ mutex_unlock(&obj->mm.lock);
+err:Shared, so 'out:'
Quoted 7 lines
Other than that, looks good.
Reviewed-by Joonas Lahtinen <[email protected]>Am I to assume that now when we introduced a new lock, every single function that assumes the lock taken is now tagged with
assert_lockdep_held?
There's are two external calls that assume the caller holds the obj->mm.lock, and yes they are so marked.
-Chris
Signature · 5 lines
Chris Wilson, Intel Open Source Technology Centre _______________________________________________ Intel-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH 19/42 drm/i915/dmabuf: Acquire the backing storage outside of struct_mutex
In reply to Chris WilsonUse the per-object mm.lock to allocate the backing storage (and hold a reference to it across the dmabuf access) without resorting to struct_mutex.
--- drivers/gpu/drm/i915/i915_gem_dmabuf.c | 51 ++++++++++++++-------------------- 1 file changed, 21 insertions(+), 30 deletions(-)
drivers/gpu/drm/i915/i915_gem_dmabuf.c +21 -30
@@ -44,19 +44,15 @@ static struct sg_table *i915_gem_map_dma_buf(struct dma_buf_attachment *attachmestruct scatterlist *src, *dst;int ret, i; - ret =i915_mutex_lock_interruptible (obj->base.dev); -if (ret) - goto err; - ret =i915_gem_object_pin_pages (obj);if (ret) - goto err_unlock; + goto err;/* Copy sg so that we make an independent mapping */ st =kmalloc (sizeof (struct sg_table ),GFP_KERNEL );if (st ==NULL ) { ret = -ENOMEM ; - goto err_unpin; + goto err_put_pages; } ret =sg_alloc_table (st, obj->mm.pages->nents,GFP_KERNEL );
@@ -72,21 +68,18 @@ static struct sg_table *i915_gem_map_dma_buf(struct dma_buf_attachment *attachme }if (!dma_map_sg (attachment->dev, st->sgl, st->nents, dir)) { - ret =-ENOMEM ; + ret = -ENOMEM ; goto err_free_sg; } -mutex_unlock (&obj->base.dev->struct_mutex);return st; err_free_sg:sg_free_table (st); err_free:kfree (st); -err_unpin: +err_put_pages:i915_gem_object_unpin_pages (obj); -err_unlock: -mutex_unlock (&obj->base.dev->struct_mutex); err:return ERR_PTR (ret); }
@@ -101,36 +94,21 @@ static void i915_gem_unmap_dma_buf(struct dma_buf_attachment *attachment,sg_free_table (sg);kfree (sg); -mutex_lock (&obj->base.dev->struct_mutex);i915_gem_object_unpin_pages (obj); -mutex_unlock (&obj->base.dev->struct_mutex); }static void *i915_gem_dmabuf_vmap (struct dma_buf *dma_buf) {struct drm_i915_gem_object *obj =dma_buf_to_obj (dma_buf); -struct drm_device *dev = obj->base.dev; -void *addr; -int ret; - ret =i915_mutex_lock_interruptible (dev); -if (ret) -return ERR_PTR (ret); - - addr =i915_gem_object_pin_map (obj,I915_MAP_WB ); -mutex_unlock (&dev->struct_mutex); - -return addr; +return i915_gem_object_pin_map (obj,I915_MAP_WB ); }static void i915_gem_dmabuf_vunmap (struct dma_buf *dma_buf,void *vaddr) {struct drm_i915_gem_object *obj =dma_buf_to_obj (dma_buf); -struct drm_device *dev = obj->base.dev; -mutex_lock (&dev->struct_mutex);i915_gem_object_unpin_map (obj); -mutex_unlock (&dev->struct_mutex); }static void *i915_gem_dmabuf_kmap_atomic (struct dma_buf *dma_buf,unsigned long page_num)
@@ -177,15 +155,22 @@ static int i915_gem_begin_cpu_access(struct dma_buf *dma_buf, enum dma_data_dire {struct drm_i915_gem_object *obj =dma_buf_to_obj (dma_buf);struct drm_device *dev = obj->base.dev; -int ret;bool write = (direction ==DMA_BIDIRECTIONAL || direction ==DMA_TO_DEVICE ); +int ret; - ret =i915_mutex_lock_interruptible (dev); + ret =i915_gem_object_pin_pages (obj);if (ret)return ret; + ret =i915_mutex_lock_interruptible (dev); +if (ret) + goto err; + ret =i915_gem_object_set_to_cpu_domain (obj, write);mutex_unlock (&dev->struct_mutex); + +err: +i915_gem_object_unpin_pages (obj);return ret; }
@@ -195,13 +180,19 @@ static int i915_gem_end_cpu_access(struct dma_buf *dma_buf, enum dma_data_directstruct drm_device *dev = obj->base.dev;int ret; - ret =i915_mutex_lock_interruptible (dev); + ret =i915_gem_object_pin_pages (obj);if (ret)return ret; + ret =i915_mutex_lock_interruptible (dev); +if (ret) + goto err; + ret =i915_gem_object_set_to_gtt_domain (obj, false);mutex_unlock (&dev->struct_mutex); +err: +i915_gem_object_unpin_pages (obj);return ret; }
Signature · 6 lines
2.9.3 _______________________________________________ Intel-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/intel-gfx
Re: [PATCH 19/42] drm/i915/dmabuf: Acquire the backing storage outside of struct_mutex
Quoted 18 lines · Chris Wilson
(quoted diff) +0
-0@@ -72,21 +68,18 @@ static struct sg_table *i915_gem_map_dma_buf(struct dma_buf_attachment *attachme }
if (!dma_map_sg(attachment->dev,st->sgl,st->nents, dir)) {drivers/gpu/drm/i915/i915_gem_dmabuf.c +1
-1- ret =-ENOMEM ; + ret = -ENOMEM ;goto
err_free_sg;
}(quoted diff) +0
-1- mutex_unlock(&obj->base.dev->struct_mutex);return st;
err_free_sg:
sg_free_table(st);
err_free:
kfree(st);drivers/gpu/drm/i915/i915_gem_dmabuf.c +1
-1-err_unpin: +err_put_pages:
Quoted 1 line
i915_gem_object_unpin_pages(obj);
I do not quite follow the logic leading to rename in here.
Quoted 19 lines
drivers/gpu/drm/i915/i915_gem_dmabuf.c +1
-1@@ -177,15 +155,22 @@ static int i915_gem_begin_cpu_access(struct dma_buf *dma_buf, enum dma_data_dire { struct drm_i915_gem_object *obj = dma_buf_to_obj(dma_buf); struct drm_device *dev = obj->base.dev; -int ret; bool write = (direction == DMA_BIDIRECTIONAL || direction == DMA_TO_DEVICE); +int ret;(quoted diff) +1
-1- ret = i915_mutex_lock_interruptible(dev); + ret = i915_gem_object_pin_pages(obj);if (ret)
return ret;drivers/gpu/drm/i915/i915_gem_dmabuf.c +4
-0+ ret =i915_mutex_lock_interruptible (dev); +if (ret) + goto err; +ret =
i915_gem_object_set_to_cpu_domain(obj, write);mutex_unlock(&dev->struct_mutex);
Review
Quoted 2 lines
(quoted diff) +2
-0+ +err:
combined path, 'out:'
Quoted 15 lines
drivers/gpu/drm/i915/i915_gem_dmabuf.c +0
-0@@ -195,13 +180,19 @@ static int i915_gem_end_cpu_access(struct dma_buf *dma_buf, enum dma_data_direct struct drm_device *dev = obj->base.dev; int ret;(quoted diff) +1
-1- ret = i915_mutex_lock_interruptible(dev); + ret = i915_gem_object_pin_pages(obj);if (ret)
return ret;drivers/gpu/drm/i915/i915_gem_dmabuf.c +4
-0+ ret =i915_mutex_lock_interruptible (dev); +if (ret) + goto err; +ret = i915_gem_object_set_to_gtt_domain(obj, false); mutex_unlock(&dev->struct_mutex);
Quoted 2 lines
+err:
Ditto.
With above fixed;
Regards, Joonas
Signature · 7 lines
Joonas Lahtinen Open Source Technology Center Intel Corporation _______________________________________________ Intel-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH 20/42 drm/i915: Implement pread without struct-mutex
In reply to Chris WilsonWe only need struct_mutex within pread for a brief window where we need to serialise with rendering and control our cache domains. Elsewhere we can rely on the backing storage being pinned, and forgive userspace any races against us.
--- drivers/gpu/drm/i915/i915_gem.c | 364 +++++++++++++++++----------------------- 1 file changed, 158 insertions(+), 206 deletions(-)
drivers/gpu/drm/i915/i915_gem.c +158 -206
@@ -63,13 +63,13 @@ static bool cpu_write_needs_clflush(struct drm_i915_gem_object *obj) }static int -insert_mappable_node (struct drm_i915_private *i915, +insert_mappable_node (struct i915_ggtt *ggtt,struct drm_mm_node *node,u32 size) {memset (node,0 ,sizeof (*node)); -return drm_mm_insert_node_in_range_generic (&i915->ggtt.base.mm, node, - size,0 ,0 ,0 , - i915->ggtt.mappable_end, +return drm_mm_insert_node_in_range_generic (&ggtt->base.mm, node, + size,0 ,-1 , +0 , ggtt->mappable_end,DRM_MM_SEARCH_DEFAULT ,DRM_MM_CREATE_DEFAULT ); }
@@ -820,32 +820,6 @@ err_unpin:return ret; } -/* Per-page copy function for the shmem pread fastpath. -* Flushes invalid cachelines before reading the target if -* needs_clflush is set. */ -static int -shmem_pread_fast (struct page *page,int shmem_page_offset,int page_length, -char __user *user_data, -bool page_do_bit17_swizzling,bool needs_clflush) -{ -char *vaddr; -int ret; - -if (unlikely (page_do_bit17_swizzling)) -return -EINVAL ; - - vaddr =kmap_atomic (page); -if (needs_clflush) -drm_clflush_virt_range (vaddr + shmem_page_offset, - page_length); - ret =__copy_to_user_inatomic (user_data, - vaddr + shmem_page_offset, - page_length); -kunmap_atomic (vaddr); - -return ret ? -EFAULT :0 ; -} -static void shmem_clflush_swizzled_range (char *addr,unsigned long length,bool swizzled)
@@ -871,7 +845,7 @@ shmem_clflush_swizzled_range(char *addr, unsigned long length,/* Only difference to the fast-path function is that this can handle bit17 * and uses non-atomic copy and kmap functions. */ static int -shmem_pread_slow (struct page *page,int shmem_page_offset,int page_length, +shmem_pread_slow (struct page *page,int offset,int length,char __user *user_data,bool page_do_bit17_swizzling,bool needs_clflush) {
@@ -880,60 +854,132 @@ shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length, vaddr =kmap (page);if (needs_clflush) -shmem_clflush_swizzled_range (vaddr + shmem_page_offset, - page_length, +shmem_clflush_swizzled_range (vaddr + offset, length, page_do_bit17_swizzling);if (page_do_bit17_swizzling) - ret =__copy_to_user_swizzled (user_data, - vaddr, shmem_page_offset, - page_length); + ret =__copy_to_user_swizzled (user_data, vaddr, offset, length);else - ret =__copy_to_user (user_data, - vaddr + shmem_page_offset, - page_length); + ret =__copy_to_user (user_data, vaddr + offset, length);kunmap (page);return ret ? -EFAULT :0 ; } -static inline unsigned long -slow_user_access (struct io_mapping *mapping, -uint64_t page_base,int page_offset, -char __user *user_data, -unsigned long length,bool pwrite) +static int +shmem_pread (struct page *page,int offset,int length,char __user *user_data, +bool page_do_bit17_swizzling,bool needs_clflush) { -void __iomem *ioaddr; +int ret; + + ret = -ENODEV ; +if (!page_do_bit17_swizzling) { +char *vaddr =kmap_atomic (page); + +if (needs_clflush) +drm_clflush_virt_range (vaddr + offset, length); + ret =__copy_to_user_inatomic (user_data, vaddr + offset, length); +kunmap_atomic (vaddr); + } +if (ret ==0 ) +return 0 ; + +return shmem_pread_slow (page, offset, length, user_data, + page_do_bit17_swizzling, needs_clflush); +} + +static int +i915_gem_shmem_pread (struct drm_i915_gem_object *obj, +struct drm_i915_gem_pread *args) +{ +char __user *user_data; +u64 remain; +unsigned int obj_do_bit17_swizzling; +unsigned int needs_clflush; +unsigned int idx, offset; +int ret; + + obj_do_bit17_swizzling =0 ; +if (i915_gem_object_needs_bit17_swizzle (obj)) + obj_do_bit17_swizzling =1 <<17 ; + + ret =mutex_lock_interruptible (&obj->base.dev->struct_mutex); +if (ret) +return ret; + + ret =i915_gem_obj_prepare_shmem_read (obj, &needs_clflush); +mutex_unlock (&obj->base.dev->struct_mutex); +if (ret) +return ret; + + remain = args->size; + user_data =u64_to_user_ptr (args->data_ptr); + offset =offset_in_page (args->offset); +for (idx = args->offset >>PAGE_SHIFT ; remain; idx++) { +struct page *page =i915_gem_object_get_page (obj, idx); +int length; + + length = remain; +if (offset + length >PAGE_SIZE ) + length =PAGE_SIZE - offset; + + ret =shmem_pread (page, offset, length, user_data, +page_to_phys (page) & obj_do_bit17_swizzling, + needs_clflush); +if (ret) +break ; + + remain -= length; + user_data += length; + offset =0 ; + } + +i915_gem_obj_finish_shmem_access (obj); +return ret; +} + +static inline int +gtt_user_read (struct io_mapping *mapping, +loff_t page_base,int page_offset, +char __user *user_data, +int length) +{ +void __iomem *vaddr_atomic;void *vaddr; -uint64_t unwritten; +unsigned long unwritten; - ioaddr =io_mapping_map_wc (mapping, page_base,PAGE_SIZE ); + vaddr_atomic =io_mapping_map_atomic_wc (mapping, page_base);/* We can use the cpu mem copy function because this is X86. */ - vaddr = (void __force *)ioaddr + page_offset; -if (pwrite) - unwritten =__copy_from_user (vaddr, user_data, length); -else - unwritten =__copy_to_user (user_data, vaddr, length); - -io_mapping_unmap (ioaddr); + vaddr = (void __force *)vaddr_atomic + page_offset; + unwritten =__copy_to_user_inatomic (user_data, vaddr, length); +io_mapping_unmap_atomic (vaddr_atomic); +if (unwritten) { + vaddr_atomic =io_mapping_map_wc (mapping, page_base,PAGE_SIZE ); +/* We can use the cpu mem copy function because this is X86. */ + vaddr = (void __force *)vaddr_atomic + page_offset; + unwritten =copy_to_user (user_data, vaddr, length); +io_mapping_unmap (vaddr_atomic); + }return unwritten; }static int -i915_gem_gtt_pread (struct drm_device *dev, -struct drm_i915_gem_object *obj,uint64_t size, -uint64_t data_offset,uint64_t data_ptr) +i915_gem_gtt_pread (struct drm_i915_gem_object *obj, +const struct drm_i915_gem_pread *args) { -struct drm_i915_private *dev_priv =to_i915 (dev); -struct i915_ggtt *ggtt = &dev_priv->ggtt; -struct i915_vma *vma; +struct i915_ggtt *ggtt = &to_i915 (obj->base.dev)->ggtt;struct drm_mm_node node; -char __user *user_data; -uint64_t remain; -uint64_t offset; +struct i915_vma *vma; +void __user *user_data; +u64 remain, offset;int ret; - vma =i915_gem_object_ggtt_pin (obj,NULL ,0 ,0 ,PIN_MAPPABLE ); + ret =mutex_lock_interruptible (&obj->base.dev->struct_mutex); +if (ret) +return ret; + + vma =i915_gem_object_ggtt_pin (obj,NULL ,0 ,0 , +PIN_MAPPABLE |PIN_NONBLOCK );if (!IS_ERR (vma)) { node.start =i915_ggtt_offset (vma); node.allocated = false;
@@ -944,33 +990,21 @@ i915_gem_gtt_pread(struct drm_device *dev, } }if (IS_ERR (vma)) { - ret =insert_mappable_node (dev_priv, &node,PAGE_SIZE ); + ret =insert_mappable_node (ggtt, &node,PAGE_SIZE );if (ret) - goto out; - - ret =i915_gem_object_pin_pages (obj); -if (ret) { -remove_mappable_node (&node); - goto out; - } + goto out_unlock; +GEM_BUG_ON (!node.allocated); } ret =i915_gem_object_set_to_gtt_domain (obj, false);if (ret) goto out_unpin; - user_data =u64_to_user_ptr (data_ptr); - remain = size; - offset = data_offset; +mutex_unlock (&obj->base.dev->struct_mutex); -mutex_unlock (&dev->struct_mutex); -if (likely (!i915.prefault_disable)) { - ret =fault_in_multipages_writeable (user_data, remain); -if (ret) { -mutex_lock (&dev->struct_mutex); - goto out_unpin; - } - } + user_data =u64_to_user_ptr (args->data_ptr); + remain = args->size; + offset = args->offset;while (remain >0 ) {/* Operation in this page
@@ -987,19 +1021,14 @@ i915_gem_gtt_pread(struct drm_device *dev,wmb (); ggtt->base.insert_page (&ggtt->base,i915_gem_object_get_dma_address (obj, offset >>PAGE_SHIFT ), - node.start, -I915_CACHE_NONE ,0 ); + node.start,I915_CACHE_NONE ,0 );wmb (); }else { page_base += offset &PAGE_MASK ; } -/* This is a slow read/write as it tries to read from -* and write to user memory which may result into page -* faults, and so we cannot perform this under struct_mutex. -*/ -if (slow_user_access (&ggtt->mappable, page_base, - page_offset, user_data, - page_length, false)) { + +if (gtt_user_read (&ggtt->mappable, page_base, page_offset, + user_data, page_length)) { ret = -EFAULT ;break ; }
@@ -1009,111 +1038,19 @@ i915_gem_gtt_pread(struct drm_device *dev, offset += page_length; } -mutex_lock (&dev->struct_mutex); -if (ret ==0 && (obj->base.read_domains &I915_GEM_DOMAIN_GTT ) ==0 ) { -/* The user has modified the object whilst we tried -* reading from it, and we now have no idea what domain -* the pages should be in. As we have just been touching -* them directly, flush everything back to the GTT -* domain. -*/ - ret =i915_gem_object_set_to_gtt_domain (obj, false); - } - +mutex_lock (&obj->base.dev->struct_mutex); out_unpin:if (node.allocated) {wmb (); ggtt->base.clear_range (&ggtt->base, node.start, node.size, true); -i915_gem_object_unpin_pages (obj);remove_mappable_node (&node); }else {i915_vma_unpin (vma); } -out: -return ret; -} - -static int -i915_gem_shmem_pread (struct drm_device *dev, -struct drm_i915_gem_object *obj, -struct drm_i915_gem_pread *args, -struct drm_file *file) -{ -char __user *user_data; -ssize_t remain; -loff_t offset; -int shmem_page_offset, page_length, ret =0 ; -int obj_do_bit17_swizzling, page_do_bit17_swizzling; -int prefaulted =0 ; -int needs_clflush =0 ; -struct sg_page_iter sg_iter; - - ret =i915_gem_obj_prepare_shmem_read (obj, &needs_clflush); -if (ret) -return ret; - - obj_do_bit17_swizzling =i915_gem_object_needs_bit17_swizzle (obj); - user_data =u64_to_user_ptr (args->data_ptr); - offset = args->offset; - remain = args->size; - -for_each_sg_page (obj->mm.pages->sgl, &sg_iter, obj->mm.pages->nents, - offset >>PAGE_SHIFT ) { -struct page *page =sg_page_iter_page (&sg_iter); - -if (remain <=0 ) -break ; - -/* Operation in this page -* -* shmem_page_offset = offset within page in shmem file -* page_length = bytes to copy for this page -*/ - shmem_page_offset =offset_in_page (offset); - page_length = remain; -if ((shmem_page_offset + page_length) >PAGE_SIZE ) - page_length =PAGE_SIZE - shmem_page_offset; - - page_do_bit17_swizzling = obj_do_bit17_swizzling && - (page_to_phys (page) & (1 <<17 )) !=0 ; - - ret =shmem_pread_fast (page, shmem_page_offset, page_length, - user_data, page_do_bit17_swizzling, - needs_clflush); -if (ret ==0 ) - goto next_page; - -mutex_unlock (&dev->struct_mutex); - -if (likely (!i915.prefault_disable) && !prefaulted) { - ret =fault_in_multipages_writeable (user_data, remain); -/* Userspace is tricking us, but we've already clobbered -* its pages with the prefault and promised to write the -* data up to the first fault. Hence ignore any errors -* and just continue. */ - (void )ret; - prefaulted =1 ; - } - - ret =shmem_pread_slow (page, shmem_page_offset, page_length, - user_data, page_do_bit17_swizzling, - needs_clflush); - -mutex_lock (&dev->struct_mutex); - -if (ret) - goto out; - -next_page: - remain -= page_length; - user_data += page_length; - offset += page_length; - } - -out: -i915_gem_obj_finish_shmem_access (obj); +out_unlock: +mutex_unlock (&obj->base.dev->struct_mutex);return ret; }
@@ -1132,7 +1069,7 @@ i915_gem_pread_ioctl(struct drm_device *dev, void *data, {struct drm_i915_gem_pread *args = data;struct drm_i915_gem_object *obj; -int ret =0 ; +int ret;if (args->size ==0 )return 0 ;
@@ -1150,7 +1087,7 @@ i915_gem_pread_ioctl(struct drm_device *dev, void *data,if (args->offset > obj->base.size || args->size > obj->base.size - args->offset) { ret = -EINVAL ; - goto err; + goto out; }trace_i915_gem_object_pread (obj, args->offset, args->size);
@@ -1160,28 +1097,21 @@ i915_gem_pread_ioctl(struct drm_device *dev, void *data,MAX_SCHEDULE_TIMEOUT , to_rps_client(file));if (ret) - goto err; + goto out; - ret =i915_mutex_lock_interruptible (dev); + ret =i915_gem_object_pin_pages (obj);if (ret) - goto err; - - ret =i915_gem_shmem_pread (dev, obj, args, file); + goto out; -/* pread for non shmem backed objects */ + ret =i915_gem_shmem_pread (obj, args);if (ret == -EFAULT || ret == -ENODEV ) {intel_runtime_pm_get (to_i915 (dev)); - ret =i915_gem_gtt_pread (dev, obj, args->size, - args->offset, args->data_ptr); + ret =i915_gem_gtt_pread (obj, args);intel_runtime_pm_put (to_i915 (dev)); } -i915_gem_object_put (obj); -mutex_unlock (&dev->struct_mutex); - -return ret; - -err: +i915_gem_object_unpin_pages (obj); +out:i915_gem_object_put_unlocked (obj);return ret; }
@@ -1209,6 +1139,28 @@ fast_user_write(struct io_mapping *mapping,return unwritten; } +static inline unsigned long +slow_user_access (struct io_mapping *mapping, +unsigned long page_base,int page_offset, +char __user *user_data, +unsigned long length,bool pwrite) +{ +void __iomem *ioaddr; +void *vaddr; +unsigned long unwritten; + + ioaddr =io_mapping_map_wc (mapping, page_base,PAGE_SIZE ); +/* We can use the cpu mem copy function because this is X86. */ + vaddr = (void __force *)ioaddr + page_offset; +if (pwrite) + unwritten =__copy_from_user (vaddr, user_data, length); +else + unwritten =__copy_to_user (user_data, vaddr, length); + +io_mapping_unmap (ioaddr); +return unwritten; +} +/** * This is the fast pwrite path, where we copy the data directly from the * user into the GTT, uncached.
@@ -1247,7 +1199,7 @@ i915_gem_gtt_pwrite_fast(struct drm_i915_private *i915, } }if (IS_ERR (vma)) { - ret =insert_mappable_node (i915, &node,PAGE_SIZE ); + ret =insert_mappable_node (ggtt, &node,PAGE_SIZE );if (ret) goto out;
@@ -1276,8 +1228,8 @@ i915_gem_gtt_pwrite_fast(struct drm_i915_private *i915,* page_length = bytes to copy for this page */ u32 page_base = node.start; -unsigned page_offset =offset_in_page (offset); -unsigned page_length =PAGE_SIZE - page_offset; +unsigned int page_offset =offset_in_page (offset); +unsigned int page_length =PAGE_SIZE - page_offset; page_length = remain < page_length ? remain : page_length;if (node.allocated) {wmb ();/* flush the write before we modify the GGTT */
Signature · 6 lines
2.9.3 _______________________________________________ Intel-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/intel-gfx
Re: [PATCH 20/42] drm/i915: Implement pread without struct-mutex
drivers/gpu/drm/i915/i915_gem.c shmem_clflush_swizzled_range(char *addr, unsigned long length, · line 848
Quoted 6 lines · Chris Wilson
drivers/gpu/drm/i915/i915_gem.c +0
-0@@ -871,7 +845,7 @@ shmem_clflush_swizzled_range(char *addr, unsigned long length, /* Only difference to the fast-path function is that this can handle bit17* and uses non-atomic copy and kmap functions. */ static int
drivers/gpu/drm/i915/i915_gem.c +1
-1-shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length, +shmem_pread_slow(struct page *page, int offset, int length,
Lets maybe call this shmem_pread_cpu? I understand it's been faster on recent generations even for swizzling. Can be added as follow-up.
Quoted 4 lines
drivers/gpu/drm/i915/i915_gem.c +4
-0+static int +i915_gem_shmem_pread (struct drm_i915_gem_object *obj, + struct drm_i915_gem_pread *args) +{
drivers/gpu/drm/i915/i915_gem.c shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length, · line 904
Quoted 10 lines
drivers/gpu/drm/i915/i915_gem.c +10
-0+char __user *user_data; +u64 remain; +unsigned int obj_do_bit17_swizzling; +unsigned int needs_clflush; +unsigned int idx, offset; +int ret; + + obj_do_bit17_swizzling =0 ; +if (i915_gem_object_needs_bit17_swizzle (obj)) + obj_do_bit17_swizzling =1 <<17 ;
Could use BIT(17) to make it super explicit.
Regards, Joonas
PS. Something like C context-aware diff would be super, would make reading these much more fun. Or maybe patchwork 2-way diff view.
Signature · 7 lines
Joonas Lahtinen Open Source Technology Center Intel Corporation _______________________________________________ Intel-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH 21/42 drm/i915: Implement pwrite without struct-mutex
In reply to Chris WilsonWe only need struct_mutex within pwrite for a brief window where we need to serialise with rendering and control our cache domains. Elsewhere we can rely on the backing storage being pinned, and forgive userspace any races against us.
--- drivers/gpu/drm/i915/i915_gem.c | 346 ++++++++++++++-------------------------- 1 file changed, 120 insertions(+), 226 deletions(-)
drivers/gpu/drm/i915/i915_gem.c +120 -226
@@ -1121,10 +1121,10 @@ out:*/ static inline int -fast_user_write (struct io_mapping *mapping, -loff_t page_base,int page_offset, -char __user *user_data, -int length) +ggtt_write (struct io_mapping *mapping, +loff_t page_base,int page_offset, +char __user *user_data, +int length) {void __iomem *vaddr_atomic;void *vaddr;
@@ -1132,60 +1132,42 @@ fast_user_write(struct io_mapping *mapping, vaddr_atomic =io_mapping_map_atomic_wc (mapping, page_base);/* We can use the cpu mem copy function because this is X86. */ - vaddr = (void __force *)vaddr_atomic + page_offset; + vaddr = (void __force *)vaddr_atomic + page_offset; unwritten =__copy_from_user_inatomic_nocache (vaddr, user_data, length);io_mapping_unmap_atomic (vaddr_atomic); -return unwritten; -} -static inline unsigned long -slow_user_access (struct io_mapping *mapping, -unsigned long page_base,int page_offset, -char __user *user_data, -unsigned long length,bool pwrite) -{ -void __iomem *ioaddr; -void *vaddr; -unsigned long unwritten; - - ioaddr =io_mapping_map_wc (mapping, page_base,PAGE_SIZE ); -/* We can use the cpu mem copy function because this is X86. */ - vaddr = (void __force *)ioaddr + page_offset; -if (pwrite) - unwritten =__copy_from_user (vaddr, user_data, length); -else - unwritten =__copy_to_user (user_data, vaddr, length); +if (unwritten) { + vaddr_atomic =io_mapping_map_wc (mapping, page_base,PAGE_SIZE ); +/* We can use the cpu mem copy function because this is X86. */ + vaddr = (void __force *)vaddr_atomic + page_offset; + unwritten =copy_from_user (vaddr, user_data, length); +io_mapping_unmap (vaddr_atomic); + } -io_mapping_unmap (ioaddr);return unwritten; }/** * This is the fast pwrite path, where we copy the data directly from the * user into the GTT, uncached. -* @i915: i915 device private data -* @obj: i915 gem object +* @obj: i915 GEM object * @args: pwrite arguments structure -* @file: drm file pointer */ static int -i915_gem_gtt_pwrite_fast (struct drm_i915_private *i915, -struct drm_i915_gem_object *obj, -struct drm_i915_gem_pwrite *args, -struct drm_file *file) +i915_gem_gtt_pwrite_fast (struct drm_i915_gem_object *obj, +const struct drm_i915_gem_pwrite *args) { -struct i915_ggtt *ggtt = &i915->ggtt; -struct drm_device *dev = obj->base.dev; -struct i915_vma *vma; +struct i915_ggtt *ggtt = &to_i915 (obj->base.dev)->ggtt;struct drm_mm_node node; -uint64_t remain, offset; -char __user *user_data; +struct i915_vma *vma; +u64 remain, offset; +void __user *user_data;int ret; -bool hit_slow_path = false; -if (i915_gem_object_is_tiled (obj)) -return -EFAULT ; + ret =mutex_lock_interruptible (&obj->base.dev->struct_mutex); +if (ret) +return ret; vma =i915_gem_object_ggtt_pin (obj,NULL ,0 ,0 ,PIN_MAPPABLE |PIN_NONBLOCK );
@@ -1201,21 +1183,17 @@ i915_gem_gtt_pwrite_fast(struct drm_i915_private *i915,if (IS_ERR (vma)) { ret =insert_mappable_node (ggtt, &node,PAGE_SIZE );if (ret) - goto out; - - ret =i915_gem_object_pin_pages (obj); -if (ret) { -remove_mappable_node (&node); - goto out; - } + goto out_unlock; +GEM_BUG_ON (!node.allocated); } ret =i915_gem_object_set_to_gtt_domain (obj, true);if (ret) goto out_unpin; +mutex_unlock (&obj->base.dev->struct_mutex); +intel_fb_obj_invalidate (obj,ORIGIN_CPU ); - obj->mm.dirty = true; user_data =u64_to_user_ptr (args->data_ptr); offset = args->offset;
@@ -1246,92 +1224,36 @@ i915_gem_gtt_pwrite_fast(struct drm_i915_private *i915,* If the object is non-shmem backed, we retry again with the * path that handles page fault. */ -if (fast_user_write (&ggtt->mappable, page_base, - page_offset, user_data, page_length)) { - hit_slow_path = true; -mutex_unlock (&dev->struct_mutex); -if (slow_user_access (&ggtt->mappable, - page_base, - page_offset, user_data, - page_length, true)) { - ret = -EFAULT ; -mutex_lock (&dev->struct_mutex); - goto out_flush; - } - -mutex_lock (&dev->struct_mutex); +if (ggtt_write (&ggtt->mappable, page_base, page_offset, + user_data, page_length)) { + ret = -EFAULT ; +break ; } remain -= page_length; user_data += page_length; offset += page_length; } - -out_flush: -if (hit_slow_path) { -if (ret ==0 && - (obj->base.read_domains &I915_GEM_DOMAIN_GTT ) ==0 ) { -/* The user has modified the object whilst we tried -* reading from it, and we now have no idea what domain -* the pages should be in. As we have just been touching -* them directly, flush everything back to the GTT -* domain. -*/ - ret =i915_gem_object_set_to_gtt_domain (obj, false); - } - } -intel_fb_obj_flush (obj, false,ORIGIN_CPU ); + +mutex_lock (&obj->base.dev->struct_mutex); out_unpin:if (node.allocated) {wmb (); ggtt->base.clear_range (&ggtt->base, node.start, node.size, true); -i915_gem_object_unpin_pages (obj);remove_mappable_node (&node); }else {i915_vma_unpin (vma); } -out: +out_unlock: +mutex_unlock (&obj->base.dev->struct_mutex);return ret; } -/* Per-page copy function for the shmem pwrite fastpath. -* Flushes invalid cachelines before writing to the target if -* needs_clflush_before is set and flushes out any written cachelines after -* writing if needs_clflush is set. */ -static int -shmem_pwrite_fast (struct page *page,int shmem_page_offset,int page_length, -char __user *user_data, -bool page_do_bit17_swizzling, -bool needs_clflush_before, -bool needs_clflush_after) -{ -char *vaddr; -int ret; - -if (unlikely (page_do_bit17_swizzling)) -return -EINVAL ; - - vaddr =kmap_atomic (page); -if (needs_clflush_before) -drm_clflush_virt_range (vaddr + shmem_page_offset, - page_length); - ret =__copy_from_user_inatomic (vaddr + shmem_page_offset, - user_data, page_length); -if (needs_clflush_after) -drm_clflush_virt_range (vaddr + shmem_page_offset, - page_length); -kunmap_atomic (vaddr); - -return ret ? -EFAULT :0 ; -} - -/* Only difference to the fast-path function is that this can handle bit17 -* and uses non-atomic copy and kmap functions. */ static int -shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length, +shmem_pwrite_slow(struct page *page, int offset, int length, char __user *user_data, bool page_do_bit17_swizzling, bool needs_clflush_before,
@@ -1342,124 +1264,113 @@ shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length, vaddr =kmap (page);if (unlikely (needs_clflush_before || page_do_bit17_swizzling)) -shmem_clflush_swizzled_range (vaddr + shmem_page_offset, - page_length, +shmem_clflush_swizzled_range (vaddr + offset, length, page_do_bit17_swizzling);if (page_do_bit17_swizzling) - ret =__copy_from_user_swizzled (vaddr, shmem_page_offset, - user_data, - page_length); + ret =__copy_from_user_swizzled (vaddr, offset, user_data, + length);else - ret =__copy_from_user (vaddr + shmem_page_offset, - user_data, - page_length); + ret =__copy_from_user (vaddr + offset, user_data, length);if (needs_clflush_after) -shmem_clflush_swizzled_range (vaddr + shmem_page_offset, - page_length, +shmem_clflush_swizzled_range (vaddr + offset, length, page_do_bit17_swizzling);kunmap (page);return ret ? -EFAULT :0 ; } +/* Per-page copy function for the shmem pwrite fastpath. +* Flushes invalid cachelines before writing to the target if +* needs_clflush_before is set and flushes out any written cachelines after +* writing if needs_clflush is set. +*/ static int -i915_gem_shmem_pwrite (struct drm_device *dev, -struct drm_i915_gem_object *obj, -struct drm_i915_gem_pwrite *args, -struct drm_file *file) +shmem_pwrite (struct page *page,int offset,int len,char __user *user_data, +bool page_do_bit17_swizzling, +bool needs_clflush_before, +bool needs_clflush_after) { -ssize_t remain; -loff_t offset; -char __user *user_data; -int shmem_page_offset, page_length, ret =0 ; -int obj_do_bit17_swizzling, page_do_bit17_swizzling; -int hit_slowpath =0 ; +int ret; + + ret = -ENODEV ; +if (!page_do_bit17_swizzling) { +char *vaddr =kmap_atomic (page); + +if (needs_clflush_before) +drm_clflush_virt_range (vaddr + offset, len); + ret =__copy_from_user_inatomic (vaddr + offset, user_data, len); +if (needs_clflush_after) +drm_clflush_virt_range (vaddr + offset, len); + +kunmap_atomic (vaddr); + } +if (ret ==0 ) +return ret; + +return shmem_pwrite_slow (page, offset, len, user_data, + page_do_bit17_swizzling, + needs_clflush_before, + needs_clflush_after); +} + +static int +i915_gem_shmem_pwrite (struct drm_i915_gem_object *obj, +const struct drm_i915_gem_pwrite *args) +{ +void __user *user_data; +u64 remain; +unsigned int obj_do_bit17_swizzling; +unsigned int partial_cacheline_write;unsigned int needs_clflush; -struct sg_page_iter sg_iter; +unsigned int offset, idx; +int ret; - ret =i915_gem_obj_prepare_shmem_write (obj, &needs_clflush); + ret =mutex_lock_interruptible (&obj->base.dev->struct_mutex);if (ret)return ret; - obj_do_bit17_swizzling =i915_gem_object_needs_bit17_swizzle (obj); - user_data =u64_to_user_ptr (args->data_ptr); - offset = args->offset; - remain = args->size; + ret =i915_gem_obj_prepare_shmem_write (obj, &needs_clflush); +mutex_unlock (&obj->base.dev->struct_mutex); +if (ret) +return ret; -for_each_sg_page (obj->mm.pages->sgl, &sg_iter, obj->mm.pages->nents, - offset >>PAGE_SHIFT ) { -struct page *page =sg_page_iter_page (&sg_iter); -int partial_cacheline_write; + obj_do_bit17_swizzling =0 ; +if (i915_gem_object_needs_bit17_swizzle (obj)) + obj_do_bit17_swizzling =1 <<17 ; -if (remain <=0 ) -break ; +/* If we don't overwrite a cacheline completely we need to be +* careful to have up-to-date data by first clflushing. Don't +* overcomplicate things and flush the entire patch. +*/ + partial_cacheline_write =0 ; +if (needs_clflush &CLFLUSH_BEFORE ) + partial_cacheline_write = boot_cpu_data.x86_clflush_size -1 ; -/* Operation in this page -* -* shmem_page_offset = offset within page in shmem file -* page_length = bytes to copy for this page -*/ - shmem_page_offset =offset_in_page (offset); - - page_length = remain; -if ((shmem_page_offset + page_length) >PAGE_SIZE ) - page_length =PAGE_SIZE - shmem_page_offset; - -/* If we don't overwrite a cacheline completely we need to be -* careful to have up-to-date data by first clflushing. Don't -* overcomplicate things and flush the entire patch. */ - partial_cacheline_write = needs_clflush &CLFLUSH_BEFORE && - ((shmem_page_offset | page_length) - & (boot_cpu_data.x86_clflush_size -1 )); - - page_do_bit17_swizzling = obj_do_bit17_swizzling && - (page_to_phys (page) & (1 <<17 )) !=0 ; - - ret =shmem_pwrite_fast (page, shmem_page_offset, page_length, - user_data, page_do_bit17_swizzling, - partial_cacheline_write, - needs_clflush &CLFLUSH_AFTER ); -if (ret ==0 ) - goto next_page; - - hit_slowpath =1 ; -mutex_unlock (&dev->struct_mutex); - ret =shmem_pwrite_slow (page, shmem_page_offset, page_length, - user_data, page_do_bit17_swizzling, - partial_cacheline_write, - needs_clflush &CLFLUSH_AFTER ); + user_data =u64_to_user_ptr (args->data_ptr); + remain = args->size; + offset =offset_in_page (args->offset); +for (idx = args->offset >>PAGE_SHIFT ; remain; idx++) { +struct page *page =i915_gem_object_get_page (obj, idx); +int length; -mutex_lock (&dev->struct_mutex); + length = remain; +if (offset + length >PAGE_SIZE ) + length =PAGE_SIZE - offset; + ret =shmem_pwrite (page, offset, length, user_data, +page_to_phys (page) & obj_do_bit17_swizzling, + (offset | length) & partial_cacheline_write, + needs_clflush &CLFLUSH_AFTER );if (ret) - goto out; - -next_page: - remain -= page_length; - user_data += page_length; - offset += page_length; - } - -out: -i915_gem_obj_finish_shmem_access (obj); +break ; -if (hit_slowpath) { -/* -* Fixup: Flush cpu caches in case we didn't flush the dirty -* cachelines in-line while writing and the object moved -* out of the cpu write domain while we've dropped the lock. -*/ -if (!(needs_clflush &CLFLUSH_AFTER ) && - obj->base.write_domain !=I915_GEM_DOMAIN_CPU ) { -if (i915_gem_clflush_object (obj, obj->pin_display)) - needs_clflush |=CLFLUSH_AFTER ; - } + remain -= length; + user_data += length; + offset =0 ; } -if (needs_clflush &CLFLUSH_AFTER ) -i915_gem_chipset_flush (to_i915 (dev)); -intel_fb_obj_flush (obj, false,ORIGIN_CPU ); +i915_gem_obj_finish_shmem_access (obj);return ret; }
@@ -1475,7 +1386,6 @@ int i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, struct drm_file *file) { - struct drm_i915_private *dev_priv = to_i915(dev); struct drm_i915_gem_pwrite *args = data;struct drm_i915_gem_object *obj;int ret;
@@ -1488,13 +1398,6 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, args->size))return -EFAULT ; -if (likely (!i915.prefault_disable)) { - ret =fault_in_multipages_readable (u64_to_user_ptr (args->data_ptr), - args->size); -if (ret) -return -EFAULT ; - } - obj =i915_gem_object_lookup (file, args->handle);if (!obj)return -ENOENT ;
@@ -1516,11 +1419,9 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,if (ret) goto err; -intel_runtime_pm_get (dev_priv); - - ret =i915_mutex_lock_interruptible (dev); + ret =i915_gem_object_pin_pages (obj);if (ret) - goto err_rpm; + goto err; ret = -EFAULT ;/* We can only do the GTT pwrite on untiled buffers, as otherwise
@@ -1531,7 +1432,7 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,*/ if (!i915_gem_object_has_struct_page (obj) ||cpu_write_needs_clflush (obj)) { - ret =i915_gem_gtt_pwrite_fast (dev_priv, obj, args, file); + ret =i915_gem_gtt_pwrite_fast (obj, args);/* Note that the gtt paths might fail with non-page-backed user * pointers (e.g. gtt mappings when moving data between * textures). Fallback to the shmem path in that case. */
@@ -1541,17 +1442,10 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,if (obj->phys_handle) ret =i915_gem_phys_pwrite (obj, args, file);else - ret =i915_gem_shmem_pwrite (dev, obj, args, file); + ret =i915_gem_shmem_pwrite (obj, args); } -i915_gem_object_put (obj); -mutex_unlock (&dev->struct_mutex); -intel_runtime_pm_put (dev_priv); - -return ret; - -err_rpm: -intel_runtime_pm_put (dev_priv); +i915_gem_object_unpin_pages (obj); err:i915_gem_object_put_unlocked (obj);return ret;
Signature · 6 lines
2.9.3 _______________________________________________ Intel-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/intel-gfx
Re: [PATCH 21/42] drm/i915: Implement pwrite without struct-mutex
Quoted 6 lines · Chris Wilson
+/* Per-page copy function for the shmem pwrite fastpath.
+ * Flushes invalid cachelines before writing to the target if
+ *needs_clflush_beforeis set and flushes out any written cachelines after
+ * writing ifneeds_clflushis set.
+ */
static int
drivers/gpu/drm/i915/i915_gem.c shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length, · line 1291
Quoted 8 lines
drivers/gpu/drm/i915/i915_gem.c +4
-4-i915_gem_shmem_pwrite(struct drm_device *dev, - struct drm_i915_gem_object *obj, - struct drm_i915_gem_pwrite *args, - struct drm_file *file) +shmem_pwrite(struct page *page, int offset, int len, char __user *user_data, + bool page_do_bit17_swizzling, + bool needs_clflush_before, + bool needs_clflush_after)
I remember having complaints of two bool arguments in same func. Could these tree be fixed while mangling the code otherwise too? Or as a follow-up.
drivers/gpu/drm/i915/i915_gem.c shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length, · line 1323
Quoted 8 lines
drivers/gpu/drm/i915/i915_gem.c +8
-0+static int +i915_gem_shmem_pwrite (struct drm_i915_gem_object *obj, + const struct drm_i915_gem_pwrite *args) +{ +void __user *user_data; +u64 remain; +unsigned int obj_do_bit17_swizzling; +unsigned int partial_cacheline_write;
partial_cacheline_mask might be more descriptive?
Regards, Joonas
Signature · 7 lines
Joonas Lahtinen Open Source Technology Center Intel Corporation _______________________________________________ Intel-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/intel-gfx
Re: [PATCH 21/42] drm/i915: Implement pwrite without struct-mutex
Quoted 11 lines · Joonas Lahtinen
Quoted 10 lines · Chris Wilson
+/* Per-page copy function for the shmem pwrite fastpath.
+ * Flushes invalid cachelines before writing to the target if
+ *needs_clflush_beforeis set and flushes out any written cachelines after
+ * writing ifneeds_clflushis set.
+ */
static intdrivers/gpu/drm/i915/i915_gem.c +0
-4-i915_gem_shmem_pwrite(struct drm_device *dev, - struct drm_i915_gem_object *obj, - struct drm_i915_gem_pwrite *args, - struct drm_file *file)
Quoted 8 lines
drivers/gpu/drm/i915/i915_gem.c shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length, · line 1291
Quoted 4 lines
drivers/gpu/drm/i915/i915_gem.c +4
-0+shmem_pwrite(struct page *page, int offset, int len, char __user *user_data, + bool page_do_bit17_swizzling, + bool needs_clflush_before, + bool needs_clflush_after)I remember having complaints of two bool arguments in same func. Could these tree be fixed while mangling the code otherwise too? Or as a follow-up.
They are three different evaluations on each loop. Looks to be bit of a hassle to amalgamate them into one in the caller. Though
(pages_to_phys() & BIT(17)) |
((offset | length) & 63) |
(needs_clflush & 2)
almost works (just need to choose another bit for needs_clflush). I'm not convinced you'd like the unpacking in shmem_pwrite() ;) Something like:
static int
shmem_pwrite_slow(struct page *page, int offset, int length,
char __user *user_data,
unsigned flags)
{
char *vaddr;
int ret;
vaddr = kmap(page);
if (unlikely(flags & (BIT(17) | 63)))
shmem_clflush_swizzled_range(vaddr + offset, length, flags);
if (flags & BIT(17))
ret = __copy_from_user_swizzled(vaddr, offset, user_data,
length);
else
ret = __copy_from_user(vaddr + offset, user_data, length);
if (flags & BIT(16))
shmem_clflush_swizzled_range(vaddr + offset, length, flags));
kunmap(page);
return ret ? -EFAULT : 0;
}Quoted 10 lines
drivers/gpu/drm/i915/i915_gem.c shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length, · line 1323
Quoted 8 lines
drivers/gpu/drm/i915/i915_gem.c +8
-0+static int +i915_gem_shmem_pwrite (struct drm_i915_gem_object *obj, + const struct drm_i915_gem_pwrite *args) +{ +void __user *user_data; +u64 remain; +unsigned int obj_do_bit17_swizzling; +unsigned int partial_cacheline_write;
partial_cacheline_maskmight be more descriptive?
To me that says we want to mask off the partial cacheline, or use them constructively. Imho,
len & partial_cacheline_write
reads more clearly as a boolean question than
len & partial_cacheline_mask.
-Chris
Signature · 5 lines
Chris Wilson, Intel Open Source Technology Centre _______________________________________________ Intel-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/intel-gfx
Joonas Lahtinen
maintainerRe: [PATCH 21/42] drm/i915: Implement pwrite without struct-mutex
Quoted 8 lines · Chris Wilson
To me that says we want to mask off the partial cacheline, or use them constructively. Imho,
len &
partial_cacheline_writereads more clearly as a boolean question than
len &
partial_cacheline_mask.
For me they convey the same value in this context, but alone, partial_cacheline_write sounds like bool. You can combine it too into write_mask as an option, too. But not a big deal after all.
Regards, joonas
Signature · 7 lines
Joonas Lahtinen Open Source Technology Center Intel Corporation _______________________________________________ Intel-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH 22/42 drm/i915: Acquire the backing storage outside of struct_mutex in set-domain
In reply to Chris WilsonAs we can locklessly (well struct_mutex-lessly) acquire the backing storage, do so in set-domain-ioctl to reduce the contention on the struct_mutex.
--- drivers/gpu/drm/i915/i915_gem.c | 83 ++++++++++++++++++++++++++--------------- 1 file changed, 53 insertions(+), 30 deletions(-)
drivers/gpu/drm/i915/i915_gem.c +53 -30
@@ -1458,6 +1458,30 @@ write_origin(struct drm_i915_gem_object *obj, unsigned domain) obj->frontbuffer_ggtt_origin : ORIGIN_CPU); } +static void i915_gem_object_bump_inactive_ggtt (struct drm_i915_gem_object *obj) +{ +struct drm_i915_private *i915; +struct list_head *list; +struct i915_vma *vma; + +list_for_each_entry (vma, &obj->vma_list, obj_link) { +if (!i915_vma_is_ggtt (vma)) +continue ; + +if (i915_vma_is_active (vma)) +continue ; + +if (!drm_mm_node_allocated (&vma->node)) +continue ; + +list_move_tail (&vma->vm_link, &vma->vm->inactive_list); + } + + i915 =to_i915 (obj->base.dev); + list = obj->bind_count ? &i915->mm.bound_list : &i915->mm.unbound_list; +list_move_tail (&obj->global_list, list); +} +/** * Called when user space prepares to use an object with the CPU, either * through the mmap ioctl's mapping or a GTT mapping.
@@ -1499,25 +1523,40 @@ i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,MAX_SCHEDULE_TIMEOUT , to_rps_client(file));if (ret) - goto err; + goto err_unlocked; + +/* Flush and acquire obj->pages so that we are coherent through +* direct access in memory with previous cached writes through +* shmemfs and that our cache domain tracking remains valid. +* For example, if the obj->filp was moved to swap without us +* being notified and releasing the pages, we would mistakenly +* continue to assume that the obj remained out of the CPU cached +* domain. +*/ + ret =i915_gem_object_pin_pages (obj); +if (ret) + goto err_unlocked; ret =i915_mutex_lock_interruptible (dev);if (ret) - goto err; + goto err_pages;if (read_domains &I915_GEM_DOMAIN_GTT ) ret =i915_gem_object_set_to_gtt_domain (obj, write_domain !=0 );else ret =i915_gem_object_set_to_cpu_domain (obj, write_domain !=0 ); -if (write_domain !=0 ) -intel_fb_obj_invalidate (obj,write_origin (obj, write_domain)); +/* And bump the LRU for this access */ +i915_gem_object_bump_inactive_ggtt (obj); -i915_gem_object_put (obj);mutex_unlock (&dev->struct_mutex); -return ret; -err: +if (write_domain !=0 ) +intel_fb_obj_invalidate (obj,write_origin (obj, write_domain)); + +err_pages: +i915_gem_object_unpin_pages (obj); +err_unlocked:i915_gem_object_put_unlocked (obj);return ret; }
@@ -1740,6 +1779,10 @@ int i915_gem_fault(struct vm_area_struct *area, struct vm_fault *vmf)if (ret) goto err; + ret =i915_gem_object_pin_pages (obj); +if (ret) + goto err; +intel_runtime_pm_get (dev_priv); ret =i915_mutex_lock_interruptible (dev);
@@ -1821,6 +1864,7 @@ err_unlock:mutex_unlock (&dev->struct_mutex); err_rpm:intel_runtime_pm_put (dev_priv); +i915_gem_object_unpin_pages (obj); err:switch (ret) {case -EIO :
@@ -3214,24 +3258,6 @@ i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj) I915_GEM_DOMAIN_CPU); } -static void i915_gem_object_bump_inactive_ggtt (struct drm_i915_gem_object *obj) -{ -struct i915_vma *vma; - -list_for_each_entry (vma, &obj->vma_list, obj_link) { -if (!i915_vma_is_ggtt (vma)) -continue ; - -if (i915_vma_is_active (vma)) -continue ; - -if (!drm_mm_node_allocated (&vma->node)) -continue ; - -list_move_tail (&vma->vm_link, &vma->vm->inactive_list); - } -} -/** * Moves a single object to the GTT read, and possibly write domain. * @obj: object to act on
@@ -3287,7 +3313,7 @@ i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)/* It should now be out of any other write domains, and we can update * the domain values for our changes. */ -BUG_ON ((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT ) !=0 ); +GEM_BUG_ON ((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT ) !=0 ); obj->base.read_domains |=I915_GEM_DOMAIN_GTT ;if (write) { obj->base.read_domains =I915_GEM_DOMAIN_GTT ;
@@ -3299,10 +3325,7 @@ i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write) old_read_domains, old_write_domain); -/* And bump the LRU for this access */ -i915_gem_object_bump_inactive_ggtt (obj);i915_gem_object_unpin_pages (obj); -return 0 ; }
@@ -3666,7 +3689,7 @@ i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)/* It should now be out of any other write domains, and we can update * the domain values for our changes. */ -BUG_ON ((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU ) !=0 ); +GEM_BUG_ON ((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU ) !=0 );/* If we're writing through the CPU, then the GPU read domains will * need to be invalidated at next use.
Signature · 6 lines
2.9.3 _______________________________________________ Intel-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/intel-gfx
Re: [PATCH 22/42] drm/i915: Acquire the backing storage outside of struct_mutex in set-domain
Quoted 38 lines · Chris Wilson
drivers/gpu/drm/i915/i915_gem.c +13
-1@@ -1499,25 +1523,40 @@ i915_gem_set_domain_ioctl(struct drm_device *dev, void *data, MAX_SCHEDULE_TIMEOUT, to_rps_client(file)); if (ret) - goto err; + goto err_unlocked; + +/* Flush and acquire obj->pages so that we are coherent through +* direct access in memory with previous cached writes through +* shmemfs and that our cache domain tracking remains valid. +* For example, if the obj->filp was moved to swap without us +* being notified and releasing the pages, we would mistakenly +* continue to assume that the obj remained out of the CPU cached +* domain. +*/ + ret =i915_gem_object_pin_pages (obj); +if (ret) + goto err_unlocked;
ret =i915_mutex_lock_interruptible(dev);
if (ret)(quoted diff) +1
-1- goto err; + goto err_pages;
if (read_domains&I915_GEM_DOMAIN_GTT)
ret =i915_gem_object_set_to_gtt_domain(obj, write_domain != 0); else
ret =i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);drivers/gpu/drm/i915/i915_gem.c +2
-2-if (write_domain !=0 ) -intel_fb_obj_invalidate (obj,write_origin (obj, write_domain)); +/* And bump the LRU for this access */ +i915_gem_object_bump_inactive_ggtt (obj);drivers/gpu/drm/i915/i915_gem.c +0
-1-i915_gem_object_put (obj);
mutex_unlock(&dev->struct_mutex);drivers/gpu/drm/i915/i915_gem.c +0
-1-return ret;
drivers/gpu/drm/i915/i915_gem.c i915_gem_set_domain_ioctl(struct drm_device *dev, void *data, · line 1557
Quoted 5 lines
drivers/gpu/drm/i915/i915_gem.c +4
-1-err: +if (write_domain !=0 ) +intel_fb_obj_invalidate (obj,write_origin (obj, write_domain)); + +err_pages:
out_pages: as it's a shared path now.
Code motion would still be great as separate patches.
Regards, Joonas
Signature · 7 lines
Joonas Lahtinen Open Source Technology Center Intel Corporation _______________________________________________ Intel-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH 23/42 drm/i915: Move object release to a freelist + worker
In reply to Chris WilsonWe want to hide the latency of releasing objects and their backing storage from the submission, so we move the actual free to a worker. This allows us to switch to struct_mutex freeing of the object in the next patch.
Furthermore, if we know that the object we are dereferencing remains valid for the duration of our access, we can forgo the usual synchronisation barriers and atomic reference counting. To ensure this we defer freeing an object til after an RCU grace period, such that any lookup of the object within an RCU read critical section will remain valid until after we exit that critical section. We also employ this delay for rate-limiting the serialisation on reallocation - we have to slow down object creation in order to prevent resource starvation (in particular, files).
--- drivers/gpu/drm/i915/i915_debugfs.c | 15 ++- drivers/gpu/drm/i915/i915_drv.c | 3 + drivers/gpu/drm/i915/i915_drv.h | 44 +++++++- drivers/gpu/drm/i915/i915_gem.c | 166 +++++++++++++++++++++---------- drivers/gpu/drm/i915/i915_gem_shrinker.c | 14 ++- drivers/gpu/drm/i915/i915_gem_tiling.c | 21 ++-- 6 files changed, 193 insertions(+), 70 deletions(-)
drivers/gpu/drm/i915/i915_debugfs.c +11 -4
@@ -4872,10 +4872,12 @@ DEFINE_SIMPLE_ATTRIBUTE(i915_ring_test_irq_fops,#define DROP_BOUND 0x2#define DROP_RETIRE 0x4#define DROP_ACTIVE 0x8 -#define DROP_ALL (DROP_UNBOUND | \ - DROP_BOUND | \ - DROP_RETIRE | \ - DROP_ACTIVE) +#define DROP_FREED 0x10 +#define DROP_ALL (DROP_UNBOUND | \ + DROP_BOUND | \ + DROP_RETIRE | \ + DROP_ACTIVE | \ + DROP_FREED)static int i915_drop_caches_get (void *data,u64 *val) {
@@ -4919,6 +4921,11 @@ i915_drop_caches_set(void *data, u64 val) unlock:mutex_unlock (&dev->struct_mutex); +if (val &DROP_FREED ) { +synchronize_rcu (); +flush_work (&dev_priv->mm.free_work); + } +return ret; }
drivers/gpu/drm/i915/i915_drv.c +3 -0
@@ -563,6 +563,9 @@ static void i915_gem_fini(struct drm_device *dev)i915_gem_context_fini (dev);mutex_unlock (&dev->struct_mutex); +synchronize_rcu (); +flush_work (&dev_priv->mm.free_work); +WARN_ON (!list_empty (&to_i915 (dev)->context_list)); }
drivers/gpu/drm/i915/i915_drv.h +41 -3
68 lines not on this page · raw message
drivers/gpu/drm/i915/i915_gem.c +116 -50
253 lines not on this page · raw message
drivers/gpu/drm/i915/i915_gem_shrinker.c +10 -4
@@ -201,6 +201,10 @@ i915_gem_shrink(struct drm_i915_private *dev_priv, typeof(*obj), global_list))) {list_move_tail (&obj->global_list, &still_in_list); +if (!obj->mm.pages) { +list_del_init (&obj->global_list); +continue ; + }if (flags &I915_SHRINK_PURGEABLE && obj->mm.madv !=I915_MADV_DONTNEED )
@@ -217,8 +221,6 @@ i915_gem_shrink(struct drm_i915_private *dev_priv,if (!can_release_pages (obj))continue ; -i915_gem_object_get (obj); -if (unsafe_drop_pages (obj)) {mutex_lock (&obj->mm.lock);if (!obj->mm.pages) {
@@ -227,8 +229,6 @@ i915_gem_shrink(struct drm_i915_private *dev_priv, }mutex_unlock (&obj->mm.lock); } - -i915_gem_object_put (obj); }list_splice (&still_in_list, phase->list); }
@@ -395,12 +395,18 @@ i915_gem_shrinker_oom(struct notifier_block *nb, unsigned long event, void *ptr)*/ unbound = bound = unevictable =0 ;list_for_each_entry (obj, &dev_priv->mm.unbound_list, global_list) { +if (!obj->mm.pages) +continue ; +if (!can_release_pages (obj)) unevictable += obj->base.size >>PAGE_SHIFT ;else unbound += obj->base.size >>PAGE_SHIFT ; }list_for_each_entry (obj, &dev_priv->mm.bound_list, global_list) { +if (!obj->mm.pages) +continue ; +if (!can_release_pages (obj)) unevictable += obj->base.size >>PAGE_SHIFT ;else
drivers/gpu/drm/i915/i915_gem_tiling.c +12 -9
@@ -328,12 +328,17 @@ i915_gem_get_tiling(struct drm_device *dev, void *data,struct drm_i915_gem_get_tiling *args = data;struct drm_i915_private *dev_priv =to_i915 (dev);struct drm_i915_gem_object *obj; +int err = -ENOENT ; + +rcu_read_lock (); + obj =i915_gem_object_lookup_rcu (file, args->handle); +if (obj) { + args->tiling_mode = +READ_ONCE (obj->tiling_and_stride) &TILING_MASK ; + err =0 ; + } +rcu_read_unlock (); - obj =i915_gem_object_lookup (file, args->handle); -if (!obj) -return -ENOENT ; - - args->tiling_mode =READ_ONCE (obj->tiling_and_stride) &TILING_MASK ;switch (args->tiling_mode) {case I915_TILING_X : args->swizzle_mode = dev_priv->mm.bit_6_swizzle_x;
@@ -341,11 +346,10 @@ i915_gem_get_tiling(struct drm_device *dev, void *data,case I915_TILING_Y : args->swizzle_mode = dev_priv->mm.bit_6_swizzle_y;break ; +default :case I915_TILING_NONE : args->swizzle_mode =I915_BIT_6_SWIZZLE_NONE ;break ; -default : -DRM_ERROR ("unknown tiling mode\n" ); }/* Hide bit 17 from the user -- see comment in i915_gem_set_tiling */
@@ -358,6 +362,5 @@ i915_gem_get_tiling(struct drm_device *dev, void *data,if (args->swizzle_mode ==I915_BIT_6_SWIZZLE_9_10_17 ) args->swizzle_mode =I915_BIT_6_SWIZZLE_9_10 ; -i915_gem_object_put_unlocked (obj); -return 0 ; +return err; }
Signature · 6 lines
2.9.3 _______________________________________________ Intel-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/intel-gfx
Re: [PATCH 23/42] drm/i915: Move object release to a freelist + worker
Quoted 477 lines · Chris Wilson
We want to hide the latency of releasing objects and their backing storage from the submission, so we move the actual free to a worker. This allows us to switch to
struct_mutexfreeing of the object in the next patch.Furthermore, if we know that the object we are dereferencing remains valid for the duration of our access, we can forgo the usual synchronisation barriers and atomic reference counting. To ensure this we defer freeing an object til after an RCU grace period, such that any lookup of the object within an RCU read critical section will remain valid until after we exit that critical section. We also employ this delay for rate-limiting the serialisation on reallocation - we have to slow down object creation in order to prevent resource starvation (in particular, files).
Signed-off-by Chris Wilson <[email protected]>--- drivers/gpu/drm/i915/i915_debugfs.c | 15 ++- drivers/gpu/drm/i915/i915_drv.c | 3 + drivers/gpu/drm/i915/i915_drv.h | 44 +++++++- drivers/gpu/drm/i915/i915_gem.c | 166 +++++++++++++++++++++---------- drivers/gpu/drm/i915/i915_gem_shrinker.c | 14 ++- drivers/gpu/drm/i915/i915_gem_tiling.c | 21 ++-- 6 files changed, 193 insertions(+), 70 deletions(-)drivers/gpu/drm/i915/i915_debugfs.c +11
-4@@ -4872,10 +4872,12 @@ DEFINE_SIMPLE_ATTRIBUTE(i915_ring_test_irq_fops,#define DROP_BOUND 0x2#define DROP_RETIRE 0x4#define DROP_ACTIVE 0x8 -#define DROP_ALL (DROP_UNBOUND | \ - DROP_BOUND | \ - DROP_RETIRE | \ - DROP_ACTIVE) +#define DROP_FREED 0x10 +#define DROP_ALL (DROP_UNBOUND | \ + DROP_BOUND | \ + DROP_RETIRE | \ + DROP_ACTIVE | \ + DROP_FREED)static int i915_drop_caches_get (void *data,u64 *val) {@@ -4919,6 +4921,11 @@ i915_drop_caches_set(void *data, u64 val) unlock:mutex_unlock (&dev->struct_mutex); +if (val &DROP_FREED ) { +synchronize_rcu (); +flush_work (&dev_priv->mm.free_work); + } +return ret; }drivers/gpu/drm/i915/i915_drv.c +3
-0@@ -563,6 +563,9 @@ static void i915_gem_fini(struct drm_device *dev)i915_gem_context_fini (dev);mutex_unlock (&dev->struct_mutex); +synchronize_rcu (); +flush_work (&dev_priv->mm.free_work); +WARN_ON (!list_empty (&to_i915 (dev)->context_list)); }drivers/gpu/drm/i915/i915_drv.h +41
-3@@ -1354,11 +1354,17 @@ struct i915_gem_mm {struct list_head bound_list;/** * List of objects which are not bound to the GTT (thus -* are idle and not used by the GPU) but still have -* (presumably uncached) pages still attached. +* are idle and not used by the GPU). These objects may or may +* not actually have any pages attached. */ struct list_head unbound_list; +/** +* List of objects which are pending destruction. +*/ +struct llist_head free_list; +struct work_struct free_work; +/** Usable portion of the GTT for GEM */ unsigned long stolen_base;/* limited to low memory (32-bit) */ @@ -2214,6 +2220,10 @@ struct drm_i915_gem_object {/** Stolen memory for this object, instead of being backed by shmem. */ struct drm_mm_node *stolen;struct list_head global_list; +union { +struct rcu_head rcu; +struct llist_node freed; + };/** Used in execbuf to temporarily hold a ref */ struct list_head obj_exec_link;@@ -2333,10 +2343,38 @@ to_intel_bo(struct drm_gem_object *gem) return container_of(gem, struct drm_i915_gem_object, base); } +/** +* i915_gem_object_lookup_rcu - look up a temporary GEM object from its handle +* @filp: DRM file private date +* @handle: userspace handle +* +* Returns: +* +* A pointer to the object named by the handle if such exists on @filp, NULL +* otherwise. This object is only valid whilst under the RCU read lock, and +* note carefully the object may be in the process of being destroyed. +*/ +static inline struct drm_i915_gem_object * +i915_gem_object_lookup_rcu (struct drm_file *file,u32 handle) +{ +#ifdef CONFIG_LOCKDEP +WARN_ON (!lock_is_held (&rcu_lock_map)); +#endif +return idr_find (&file->object_idr, handle); +} +static inline struct drm_i915_gem_object *i915_gem_object_lookup (struct drm_file *file,u32 handle) { -return to_intel_bo (drm_gem_object_lookup (file, handle)); +struct drm_i915_gem_object *obj; + +rcu_read_lock (); + obj =i915_gem_object_lookup_rcu (file, handle); +if (obj && !kref_get_unless_zero (&obj->base.refcount)) + obj =NULL ; +rcu_read_unlock (); + +return obj; }__deprecated drivers/gpu/drm/i915/i915_gem.c +116
-50@@ -42,6 +42,7 @@#include <linux/pci.h> #include <linux/dma-buf.h> +static void i915_gem_flush_free_objects (struct drm_i915_private *i915);static void i915_gem_object_flush_gtt_write_domain (struct drm_i915_gem_object *obj);static void i915_gem_object_flush_cpu_write_domain (struct drm_i915_gem_object *obj);@@ -647,6 +648,8 @@ i915_gem_create_ioctl(struct drm_device *dev, void *data, {struct drm_i915_gem_create *args = data; +i915_gem_flush_free_objects (to_i915 (dev)); +return i915_gem_create (file, dev, args->size, &args->handle); }@@ -3469,10 +3472,14 @@ int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data, {struct drm_i915_gem_caching *args = data;struct drm_i915_gem_object *obj; +int err =0 ; - obj =i915_gem_object_lookup (file, args->handle); -if (!obj) -return -ENOENT ; +rcu_read_lock (); + obj =i915_gem_object_lookup_rcu (file, args->handle); +if (!obj) { + err = -ENOENT ; + goto out; + }switch (obj->cache_level) {case I915_CACHE_LLC :@@ -3488,9 +3495,9 @@ int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data, args->caching =I915_CACHING_NONE ;break ; } - -i915_gem_object_put_unlocked (obj); -return 0 ; +out: +rcu_read_unlock (); +return err; } int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data,@@ -4001,10 +4008,14 @@ i915_gem_busy_ioctl(struct drm_device *dev, void *data,struct drm_i915_gem_busy *args = data;struct drm_i915_gem_object *obj;unsigned long active; +int err; - obj =i915_gem_object_lookup (file, args->handle); -if (!obj) -return -ENOENT ; +rcu_read_lock (); + obj =i915_gem_object_lookup_rcu (file, args->handle); +if (!obj) { + err = -ENOENT ; + goto out; + } args->busy =0 ; active =__I915_BO_ACTIVE (obj);@@ -4034,7 +4045,6 @@ i915_gem_busy_ioctl(struct drm_device *dev, void *data,* are busy is not completely reliable - we only guarantee * that the object was busy. */ -rcu_read_lock ();for_each_active (active, idx) args->busy |=busy_check_reader (&obj->last_read[idx]);@@ -4052,12 +4062,11 @@ i915_gem_busy_ioctl(struct drm_device *dev, void *data,* the result. */ args->busy |=busy_check_writer (&obj->last_write); - -rcu_read_unlock (); } -i915_gem_object_put_unlocked (obj); -return 0 ; +out: +rcu_read_unlock (); +return err; }int @@ -4204,7 +4213,6 @@ struct drm_i915_gem_object *i915_gem_object_create(struct drm_device *dev, fail:i915_gem_object_free (obj); -return ERR_PTR (ret); }@@ -4232,16 +4240,69 @@ static bool discard_backing_storage(struct drm_i915_gem_object *obj)return atomic_long_read (&obj->base.filp->f_count) ==1 ; } -void i915_gem_free_object (struct drm_gem_object *gem_obj) +static void __i915_gem_free_objects (struct drm_i915_private *i915, +struct llist_node *freed) { -struct drm_i915_gem_object *obj =to_intel_bo (gem_obj); -struct drm_device *dev = obj->base.dev; -struct drm_i915_private *dev_priv =to_i915 (dev); -struct i915_vma *vma, *next; +struct drm_i915_gem_object *obj, *on; -intel_runtime_pm_get (dev_priv); +mutex_lock (&i915->drm.struct_mutex); +intel_runtime_pm_get (i915); +llist_for_each_entry (obj, freed, freed) { +struct i915_vma *vma, *vn; + +trace_i915_gem_object_destroy (obj); + +GEM_BUG_ON (i915_gem_object_is_active (obj)); +list_for_each_entry_safe (vma, vn, + &obj->vma_list, obj_link) { +GEM_BUG_ON (!i915_vma_is_ggtt (vma)); +GEM_BUG_ON (i915_vma_is_active (vma)); + vma->flags &= ~I915_VMA_PIN_MASK ; +i915_vma_close (vma); + } + +list_del (&obj->global_list); + } +intel_runtime_pm_put (i915); +mutex_unlock (&i915->drm.struct_mutex); + +llist_for_each_entry_safe (obj, on, freed, freed) { +GEM_BUG_ON (obj->bind_count); +GEM_BUG_ON (atomic_read (&obj->frontbuffer_bits)); + +if (obj->ops->release) + obj->ops->release (obj); + +if (WARN_ON (i915_gem_object_has_pinned_pages (obj))) +atomic_set (&obj->mm.pages_pin_count,0 ); +__i915_gem_object_put_pages (obj); +GEM_BUG_ON (obj->mm.pages); + +if (obj->base.import_attach) +drm_prime_gem_destroy (&obj->base,NULL ); + +drm_gem_object_release (&obj->base); +i915_gem_info_remove_obj (i915, obj->base.size); + +kfree (obj->bit_17); +i915_gem_object_free (obj); + } +} + +static void i915_gem_flush_free_objects (struct drm_i915_private *i915) +{ +struct llist_node *freed; -trace_i915_gem_object_destroy (obj); + freed =llist_del_all (&i915->mm.free_list); +if (unlikely (freed)) +__i915_gem_free_objects (i915, freed); +} + +static void __i915_gem_free_work (struct work_struct *work) +{ +struct drm_i915_private *i915 = + container_of(work, struct drm_i915_private, mm.free_work); +struct llist_node *freed;/* All file-owned VMA should have been released by this point through * i915_gem_close_object(), or earlier by i915_gem_context_close(). @@ -4250,42 +4311,44 @@ void i915_gem_free_object(struct drm_gem_object *gem_obj)* the GTT either for the user or for scanout). Those VMA still need to * unbound now. */ -list_for_each_entry_safe (vma, next, &obj->vma_list, obj_link) { -GEM_BUG_ON (!i915_vma_is_ggtt (vma)); -GEM_BUG_ON (i915_vma_is_active (vma)); - vma->flags &= ~I915_VMA_PIN_MASK ; -i915_vma_close (vma); - } -GEM_BUG_ON (obj->bind_count); -WARN_ON (atomic_read (&obj->frontbuffer_bits)); +while ((freed =llist_del_all (&i915->mm.free_list))) +__i915_gem_free_objects (i915, freed); +} -if (obj->mm.pages && obj->mm.madv ==I915_MADV_WILLNEED && - dev_priv->quirks &QUIRK_PIN_SWIZZLED_PAGES && -i915_gem_object_is_tiled (obj)) -__i915_gem_object_unpin_pages (obj); +static void __i915_gem_free_object_rcu (struct rcu_head *head) +{ +struct drm_i915_gem_object *obj = +container_of (head,typeof (*obj), rcu); +struct drm_i915_private *i915 =to_i915 (obj->base.dev); + +/* We can't simply use call_rcu() from i915_gem_free_object() +* as we need to block whilst unbinding, and the call_rcu +* task may be called from softirq context. So we take a +* detour through a worker. +*/ +if (llist_add (&obj->freed, &i915->mm.free_list)) +schedule_work (&i915->mm.free_work); +} -if (obj->ops->release) - obj->ops->release (obj); +void i915_gem_free_object (struct drm_gem_object *gem_obj) +{ +struct drm_i915_gem_object *obj =to_intel_bo (gem_obj); -if (WARN_ON (i915_gem_object_has_pinned_pages (obj))) -atomic_set (&obj->mm.pages_pin_count,0 );if (discard_backing_storage (obj)) obj->mm.madv =I915_MADV_DONTNEED ; -__i915_gem_object_put_pages (obj); -GEM_BUG_ON (obj->mm.pages); - -if (obj->base.import_attach) -drm_prime_gem_destroy (&obj->base,NULL ); - -drm_gem_object_release (&obj->base); -i915_gem_info_remove_obj (dev_priv, obj->base.size); - -kfree (obj->bit_17); -i915_gem_object_free (obj); +if (obj->mm.pages && obj->mm.madv ==I915_MADV_WILLNEED && +to_i915 (obj->base.dev)->quirks &QUIRK_PIN_SWIZZLED_PAGES && +i915_gem_object_is_tiled (obj)) +__i915_gem_object_unpin_pages (obj); -intel_runtime_pm_put (dev_priv); +/* Before we free the object, make sure any pure RCU-only +* read-side critical sections are complete, e.g. +* i915_gem_busy_ioctl(). For the corresponding synchronized +* lookup see i915_gem_object_lookup_rcu(). +*/ +call_rcu (&obj->rcu, __i915_gem_free_object_rcu); }void __i915_gem_object_release_unless_active (struct drm_i915_gem_object *obj)@@ -4334,6 +4397,7 @@ int i915_gem_suspend(struct drm_device *dev)cancel_delayed_work_sync (&dev_priv->gpu_error.hangcheck_work);cancel_delayed_work_sync (&dev_priv->gt.retire_work);flush_delayed_work (&dev_priv->gt.idle_work); +flush_work (&dev_priv->mm.free_work);/* Assert that we sucessfully flushed all the work and * reset the GPU back to its idle, low power state. @@ -4630,6 +4694,8 @@ i915_gem_load_init(struct drm_device *dev) NULL);INIT_LIST_HEAD (&dev_priv->context_list); +INIT_WORK (&dev_priv->mm.free_work, __i915_gem_free_work); +init_llist_head (&dev_priv->mm.free_list);INIT_LIST_HEAD (&dev_priv->mm.unbound_list);INIT_LIST_HEAD (&dev_priv->mm.bound_list);INIT_LIST_HEAD (&dev_priv->mm.fence_list);drivers/gpu/drm/i915/i915_gem_shrinker.c +10
-4@@ -201,6 +201,10 @@ i915_gem_shrink(struct drm_i915_private *dev_priv, typeof(*obj), global_list))) {list_move_tail (&obj->global_list, &still_in_list); +if (!obj->mm.pages) { +list_del_init (&obj->global_list); +continue ; + }if (flags &I915_SHRINK_PURGEABLE && obj->mm.madv !=I915_MADV_DONTNEED )@@ -217,8 +221,6 @@ i915_gem_shrink(struct drm_i915_private *dev_priv,if (!can_release_pages (obj))continue ; -i915_gem_object_get (obj); -if (unsafe_drop_pages (obj)) {mutex_lock (&obj->mm.lock);if (!obj->mm.pages) {@@ -227,8 +229,6 @@ i915_gem_shrink(struct drm_i915_private *dev_priv, }mutex_unlock (&obj->mm.lock); } - -i915_gem_object_put (obj); }list_splice (&still_in_list, phase->list); }@@ -395,12 +395,18 @@ i915_gem_shrinker_oom(struct notifier_block *nb, unsigned long event, void *ptr)*/ unbound = bound = unevictable =0 ;list_for_each_entry (obj, &dev_priv->mm.unbound_list, global_list) { +if (!obj->mm.pages) +continue ; +if (!can_release_pages (obj)) unevictable += obj->base.size >>PAGE_SHIFT ;else unbound += obj->base.size >>PAGE_SHIFT ; }list_for_each_entry (obj, &dev_priv->mm.bound_list, global_list) { +if (!obj->mm.pages) +continue ; +if (!can_release_pages (obj)) unevictable += obj->base.size >>PAGE_SHIFT ;else drivers/gpu/drm/i915/i915_gem_tiling.c +1
-0@@ -328,12 +328,17 @@ i915_gem_get_tiling(struct drm_device *dev, void *data,struct drm_i915_gem_get_tiling *args = data;struct drm_i915_private *dev_priv =to_i915 (dev);struct drm_i915_gem_object *obj; +int err = -ENOENT ;
drivers/gpu/drm/i915/i915_gem_tiling.c i915_gem_get_tiling(struct drm_device *dev, void *data, · line 341
Quoted 10 lines
drivers/gpu/drm/i915/i915_gem_tiling.c +9
-0+ +rcu_read_lock (); + obj =i915_gem_object_lookup_rcu (file, args->handle); +if (obj) { + args->tiling_mode = +READ_ONCE (obj->tiling_and_stride) &TILING_MASK ; + err =0 ; + } +rcu_read_unlock ();
Should it not just return -ENOENT here if !obj rather than going through the rest of the function and having to modify that to ignore unknown tiling modes (because it is doing the switch on uninitialised data)?
Quoted 22 lines
drivers/gpu/drm/i915/i915_gem_tiling.c +1
-7- obj =i915_gem_object_lookup (file, args->handle); -if (!obj) -return -ENOENT ; - - args->tiling_mode =READ_ONCE (obj->tiling_and_stride) &TILING_MASK ;switch (args->tiling_mode) {case I915_TILING_X : args->swizzle_mode = dev_priv->mm.bit_6_swizzle_x;@@ -341,11 +346,10 @@ i915_gem_get_tiling(struct drm_device *dev, void *data,case I915_TILING_Y : args->swizzle_mode = dev_priv->mm.bit_6_swizzle_y;break ; +default :case I915_TILING_NONE : args->swizzle_mode =I915_BIT_6_SWIZZLE_NONE ;break ; -default : -DRM_ERROR ("unknown tiling mode\n" ); }/* Hide bit 17 from the user -- see comment in i915_gem_set_tiling */
drivers/gpu/drm/i915/i915_gem_tiling.c i915_gem_get_tiling(struct drm_device *dev, void *data, · line 366
Quoted 7 lines
drivers/gpu/drm/i915/i915_gem_tiling.c +1
-2@@ -358,6 +362,5 @@ i915_gem_get_tiling(struct drm_device *dev, void *data,if (args->swizzle_mode ==I915_BIT_6_SWIZZLE_9_10_17 ) args->swizzle_mode =I915_BIT_6_SWIZZLE_9_10 ; -i915_gem_object_put_unlocked (obj); -return 0 ; +return err; }
_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx
PATCH 24/42 drm/i915: Treat a framebuffer reference as an active reference whilst shrinking
In reply to Chris WilsonTreat a framebuffer reference with the same priority as an active reference whilst shrinking. Framebuffers are likely to be reused and typically cost more to migrate to and from GPU memory (on LLC architectures we need to clflush), so defer the temptation to purge them during a kswapd run until we have run out of cheap buffers.
--- drivers/gpu/drm/i915/i915_gem_shrinker.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-)
drivers/gpu/drm/i915/i915_gem_shrinker.c +3 -2
@@ -214,8 +214,9 @@ i915_gem_shrink(struct drm_i915_private *dev_priv, !is_vmalloc_addr(obj->mm.mapping))continue ; -if ((flags &I915_SHRINK_ACTIVE ) ==0 && -i915_gem_object_is_active (obj)) +if (!(flags &I915_SHRINK_ACTIVE ) && + (i915_gem_object_is_active (obj) || + obj->framebuffer_references))continue ;if (!can_release_pages (obj))
Signature · 6 lines
2.9.3 _______________________________________________ Intel-gfx mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/intel-gfx
Re: [PATCH 24/42] drm/i915: Treat a framebuffer reference as an active reference whilst shrinking
Quoted 17 lines · Chris Wilson
Treat a framebuffer reference with the same priority as an active reference whilst shrinking. Framebuffers are likely to be reused and typically cost more to migrate to and from GPU memory (on LLC architectures we need to clflush), so defer the temptation to purge them during a kswapd run until we have run out of cheap buffers.
Signed-off-by Chris Wilson <[email protected]>--- drivers/gpu/drm/i915/i915_gem_shrinker.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-)drivers/gpu/drm/i915/i915_gem_shrinker.c +0
-0@@ -214,8 +214,9 @@ i915_gem_shrink(struct drm_i915_private *dev_priv, !is_vmalloc_addr(obj->mm.mapping))
drivers/gpu/drm/i915/i915_gem_shrinker.c i915_gem_shrink(struct drm_i915_private *dev_priv, · line 222
Quoted 10 lines
drivers/gpu/drm/i915/i915_gem_shrinker.c +3
-2continue ; -if ((flags &I915_SHRINK_ACTIVE ) ==0 && -i915_gem_object_is_active (obj)) +if (!(flags &I915_SHRINK_ACTIVE ) && + (i915_gem_object_is_active (obj) || + obj->framebuffer_references))continue ;if (!can_release_pages (obj))
_______________________________________________
Intel-gfx mailing list
[email protected]
https://lists.freedesktop.org/mailman/listinfo/intel-gfx