drm/i915: Allocate atomically in execbuf path
Subsystems (2)
PATCH 1/1 drm/i915: Allocate atomically in execbuf path
If we end up calling the shrinker, which in turn requires the OOM killer, we may end up infinitely waiting for a process to die if the OOM chooses. The case that this prevents occurs in execbuf. The forked variants of gem_evict_everything is a good way to hit it. This is exacerbated by Daniel's recent patch to give OOM precedence to the GEM tests.
It's a twisted form of a deadlock.
What occurs is the following (assume just 2 procs)
1. proc A gets to execbuf while out of memory, gets struct_mutex.
2. OOM killer comes in and chooses proc B
3. proc B closes it's fds, which requires struct mutex, blocks 4, OOM killer waits for B to die before killing another process (this part is speculative)
--- drivers/gpu/drm/i915/i915_gem.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
drivers/gpu/drm/i915/i915_gem.c +2 -2
@@ -1842,12 +1842,12 @@ i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)BUG_ON (obj->base.read_domains &I915_GEM_GPU_DOMAINS );BUG_ON (obj->base.write_domain &I915_GEM_GPU_DOMAINS ); - st =kmalloc (sizeof (*st),GFP_KERNEL ); + st =kmalloc (sizeof (*st),GFP_NOWAIT );if (st ==NULL )return -ENOMEM ; page_count = obj->base.size /PAGE_SIZE ; -if (sg_alloc_table (st, page_count,GFP_KERNEL )) { +if (sg_alloc_table (st, page_count,GFP_NOWAIT )) {kfree (st);return -ENOMEM ; }
Signature · 1 line
1.8.4.2
Quoted 7 lines · Ben Widawsky
If we end up calling the shrinker, which in turn requires the OOM killer, we may end up infinitely waiting for a process to die if the OOM chooses. The case that this prevents occurs in execbuf. The forked variants of
gem_evict_everythingis a good way to hit it. This is exacerbated by Daniel's recent patch to give OOM precedence to the GEM tests.
Quoted 9 lines
It's a twisted form of a deadlock.
What occurs is the following (assume just 2 procs)
1. proc A gets to execbuf while out of memory, getsstruct_mutex.
2. OOM killer comes in and chooses proc B
3. proc B closes it's fds, which requires struct mutex, blocks 4, OOM killer waits for B to die before killing another process (this part is speculative)
It appears that by itself, this patch is insufficient to prevent the failure when run in piglit. Before I go on a wild goose chase of finding all allocations, maybe I'll give people a chance to chime in. The symptom is the same always, OOM, procs can't die because struct_mutex is held.
Quoted 29 lines
Cc Daniel Vetter <[email protected]>Cc Chris Wilson <[email protected]>Signed-off-by Ben Widawsky <[email protected]>--- drivers/gpu/drm/i915/i915_gem.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)drivers/gpu/drm/i915/i915_gem.c +2
-2@@ -1842,12 +1842,12 @@ i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)BUG_ON (obj->base.read_domains &I915_GEM_GPU_DOMAINS );BUG_ON (obj->base.write_domain &I915_GEM_GPU_DOMAINS ); - st =kmalloc (sizeof (*st),GFP_KERNEL ); + st =kmalloc (sizeof (*st),GFP_NOWAIT );if (st ==NULL )return -ENOMEM ; page_count = obj->base.size /PAGE_SIZE ; -if (sg_alloc_table (st, page_count,GFP_KERNEL )) { +if (sg_alloc_table (st, page_count,GFP_NOWAIT )) {kfree (st);return -ENOMEM ; }Signature · 1 line
1.8.4.2
Signature · 1 line
Ben Widawsky, Intel Open Source Technology Center
Quoted 18 lines · Ben Widawsky
Quoted 7 lines · Ben Widawsky
If we end up calling the shrinker, which in turn requires the OOM killer, we may end up infinitely waiting for a process to die if the OOM chooses. The case that this prevents occurs in execbuf. The forked variants of
gem_evict_everythingis a good way to hit it. This is exacerbated by Daniel's recent patch to give OOM precedence to the GEM tests.Quoted 9 lines
It's a twisted form of a deadlock.
What occurs is the following (assume just 2 procs)
1. proc A gets to execbuf while out of memory, getsstruct_mutex.
2. OOM killer comes in and chooses proc B
3. proc B closes it's fds, which requires struct mutex, blocks 4, OOM killer waits for B to die before killing another process (this part is speculative)
Quoted 6 lines
It appears that by itself, this patch is insufficient to prevent the failure when run in piglit. Before I go on a wild goose chase of finding all allocations, maybe I'll give people a chance to chime in. The symptom is the same always, OOM, procs can't die because
struct_mutexis held.
too late on the goose chase. vma allocation was the missing bit.
Quoted 36 lines
Quoted 29 lines
Cc Daniel Vetter <[email protected]>Cc Chris Wilson <[email protected]>Signed-off-by Ben Widawsky <[email protected]>--- drivers/gpu/drm/i915/i915_gem.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)drivers/gpu/drm/i915/i915_gem.c +2
-2@@ -1842,12 +1842,12 @@ i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)BUG_ON (obj->base.read_domains &I915_GEM_GPU_DOMAINS );BUG_ON (obj->base.write_domain &I915_GEM_GPU_DOMAINS ); - st =kmalloc (sizeof (*st),GFP_KERNEL ); + st =kmalloc (sizeof (*st),GFP_NOWAIT );if (st ==NULL )return -ENOMEM ; page_count = obj->base.size /PAGE_SIZE ; -if (sg_alloc_table (st, page_count,GFP_KERNEL )) { +if (sg_alloc_table (st, page_count,GFP_NOWAIT )) {kfree (st);return -ENOMEM ; }Signature · 1 line
1.8.4.2Signature · 5 lines
Ben Widawsky, Intel Open Source Technology Center _______________________________________________ Intel-gfx mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/intel-gfx
Signature · 1 line
Ben Widawsky, Intel Open Source Technology Center
Quoted 16 lines · Ben Widawsky
If we end up calling the shrinker, which in turn requires the OOM killer, we may end up infinitely waiting for a process to die if the OOM chooses. The case that this prevents occurs in execbuf. The forked variants of
gem_evict_everythingis a good way to hit it. This is exacerbated by Daniel's recent patch to give OOM precedence to the GEM tests.It's a twisted form of a deadlock.
What occurs is the following (assume just 2 procs)
1. proc A gets to execbuf while out of memory, getsstruct_mutex.
2. OOM killer comes in and chooses proc B
3. proc B closes it's fds, which requires struct mutex, blocks 4, OOM killer waits for B to die before killing another process (this part is speculative)
Quoted 3 lines
Cc Daniel Vetter <[email protected]>Cc Chris Wilson <[email protected]>Signed-off-by Ben Widawsky <[email protected]>
I'd still like to know if I am crazy, but I'm now trying to defer the stuff we do on file close without using any allocs. Just an update...
Signature · 1 line
Ben Widawsky, Intel Open Source Technology Center
Quoted 17 lines · Ben Widawsky
Quoted 16 lines · Ben Widawsky
If we end up calling the shrinker, which in turn requires the OOM killer, we may end up infinitely waiting for a process to die if the OOM chooses. The case that this prevents occurs in execbuf. The forked variants of
gem_evict_everythingis a good way to hit it. This is exacerbated by Daniel's recent patch to give OOM precedence to the GEM tests.It's a twisted form of a deadlock.
What occurs is the following (assume just 2 procs)
1. proc A gets to execbuf while out of memory, getsstruct_mutex.
2. OOM killer comes in and chooses proc B
3. proc B closes it's fds, which requires struct mutex, blocks 4, OOM killer waits for B to die before killing another process (this part is speculative)
Quoted 6 lines
Quoted 3 lines
Cc Daniel Vetter <[email protected]>Cc Chris Wilson <[email protected]>Signed-off-by Ben Widawsky <[email protected]>I'd still like to know if I am crazy, but I'm now trying to defer the stuff we do on file close without using any allocs. Just an update...
Sound's intrigueing, but tbh I don't really have clue about things. What about adding the relevant stuck task backtraces to the patch and submitting this to a wider audience (lkml, mm-devel) as an akpm-probe? The more botched the patch, the better the probe usually. -Daniel
Signature · 3 lines
Daniel Vetter Software Engineer, Intel Corporation +41 (0) 79 365 57 48 - http://blog.ffwll.ch
Quoted 17 lines · Ben Widawsky
Quoted 16 lines · Ben Widawsky
If we end up calling the shrinker, which in turn requires the OOM killer, we may end up infinitely waiting for a process to die if the OOM chooses. The case that this prevents occurs in execbuf. The forked variants of
gem_evict_everythingis a good way to hit it. This is exacerbated by Daniel's recent patch to give OOM precedence to the GEM tests.It's a twisted form of a deadlock.
What occurs is the following (assume just 2 procs)
1. proc A gets to execbuf while out of memory, getsstruct_mutex.
2. OOM killer comes in and chooses proc B
3. proc B closes it's fds, which requires struct mutex, blocks 4, OOM killer waits for B to die before killing another process (this part is speculative)
Quoted 7 lines
Quoted 3 lines
Cc Daniel Vetter <[email protected]>Cc Chris Wilson <[email protected]>Signed-off-by Ben Widawsky <[email protected]>I'd still like to know if I am crazy, but I'm now trying to defer the stuff we do on file close without using any allocs. Just an update...
workqueue still has similar problems. It could be because deferring the context cleanup means we don't actually free much space, and so the OOM isn't enough, or [more likely] something else is going on.
Maybe it's my bug. I am really out of ideas at the moment. The system just becomes unresponsive after all threads end up blocked waiting for struct mutex. I know I'd seen such problems in the past with gem_evict_everything, but this time around I seem to be the sole cause (and not all my machines hit it).
Sorry for the noise - just totally burning out on this one.
Signature · 1 line
Ben Widawsky, Intel Open Source Technology Center
PATCH 1/1 drm/i915: Optionally defer context closing
In reply to Ben WidawskyThis patch doesn't seem to be the panacea that I had hoped, although I've yet to thoroughly test if it actually improves some of the other tests which caused trouble.
What occurs is the following (assume just 2 procs)
1. proc A gets to execbuf while out of memory, gets struct_mutex.
2. OOM killer comes in and chooses proc B
3. proc B closes it's fds, which requires struct mutex, blocks 4, OOM killer waits for B to die before killing another process (this part is speculative)
In order to let the OOM complete, we defer processing the context destruction parts which are those that require struct_mutex.
This patch has not really been cleaned up, I am only posting it for posterity. (Several of the hunks are only relevant to full PPGTT patches which are not merged).
--- drivers/gpu/drm/i915/i915_debugfs.c | 6 ++- drivers/gpu/drm/i915/i915_drv.h | 9 +++-- drivers/gpu/drm/i915/i915_gem.c | 2 +- drivers/gpu/drm/i915/i915_gem_context.c | 68 ++++++++++++++++++++++++--------- 4 files changed, 61 insertions(+), 24 deletions(-)
drivers/gpu/drm/i915/i915_debugfs.c +4 -2
@@ -1717,11 +1717,13 @@ static void gen6_ppgtt_info(struct seq_file *m, struct drm_device *dev)struct drm_i915_file_private *file_priv = file->driver_priv;struct i915_hw_ppgtt *pvt_ppgtt; - pvt_ppgtt =ctx_to_ppgtt (file_priv->private_default_ctx); + pvt_ppgtt = +ctx_to_ppgtt (file_priv->ctx_info->private_default_ctx);seq_printf (m,"proc: %s\n" ,get_pid_task (file->pid,PIDTYPE_PID )->comm);seq_puts (m," default context:\n" ); -idr_for_each (&file_priv->context_idr, per_file_ctx, m); +idr_for_each (&file_priv->ctx_info->context_idr, per_file_ctx, + m); }seq_printf (m,"ECOCHK: 0x%08x\n" ,I915_READ (GAM_ECOCHK )); }
drivers/gpu/drm/i915/i915_drv.h +6 -3
@@ -1735,15 +1735,18 @@ struct drm_i915_gem_request {struct drm_i915_file_private {struct drm_i915_private *dev_priv; -struct {spinlock_t lock;struct list_head request_list;struct delayed_work idle_work; } mm; -struct idr context_idr; +struct i915_gem_ctx_info { +struct drm_i915_private *dev_priv; +struct idr context_idr; +struct work_struct close_work; +struct i915_hw_context *private_default_ctx; + } *ctx_info; -struct i915_hw_context *private_default_ctx;atomic_t rps_wait_boost; };
drivers/gpu/drm/i915/i915_gem.c +1 -1
@@ -2326,7 +2326,7 @@ static void i915_set_reset_status(struct intel_ring_buffer *ring,if (request->ctx && request->ctx->id !=DEFAULT_CONTEXT_ID ) hs = &request->ctx->hang_stats;else if (request->file_priv) - hs = &request->file_priv->private_default_ctx->hang_stats; + hs = &request->file_priv->ctx_info->private_default_ctx->hang_stats;if (hs) {if (guilty) {
drivers/gpu/drm/i915/i915_gem_context.c +50 -18
121 lines not on this page · raw message
Signature · 1 line
1.8.4.2