drm/imagination: Add async VM_BIND with sparse mappings
Subsystems (6)
v2 cover drm/imagination: Add async VM_BIND with sparse mappings
Userspace MR:
https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/43794
This series is for Vulkan sparse binding.
- Patches 1-2 switch the GPUVM to immediate mode.
- Patch 3 moves the existing map and unmap paths onto a single bind entry point.
- Patch 4 adds the VM_BIND ioctl.
- Patch 5 adds asynchronous requests.
- Patch 6 adds sparse mappings.
Patches 1-2 were sent separately [1]. They conflict with drm-misc-next, so they are rebased and included here.
Testing:
Tested on BeagleY-AI.
- The existing IGT pvr_vm_map passes without regressions.
- dEQP-VK.sparse_resources passes for the sparse features the Mesa MR above exposes.
[1] https://lore.kernel.org/r/[email protected]
---
Changes in v2:
- New patch 3, "Route map and unmap through pvr_vm_bind()".
- Split v1 3/4 into the ioctl (4) and the asynchronous mode (5).
- Bump the driver interface version to 1.2, as 1.1 is already taken by another patch.
- Link to v1: https://lore.kernel.org/r/[email protected]
---
Gyeyoung Baek (6):
drm/imagination: Switch the GPUVM to immediate mode drm/imagination: Stop locking reservations around map and unmap drm/imagination: Route map and unmap through pvr_vm_bind() drm/imagination: Add VM_BIND ioctl
drm/imagination: Add asynchronous VM_BIND drm/imagination: Add MAP_SPARSE to VM_BIND
Documentation/gpu/imagination/uapi.rst | 15 + drivers/gpu/drm/imagination/pvr_device.h | 10 + drivers/gpu/drm/imagination/pvr_drv.c | 331 ++++++++++++ drivers/gpu/drm/imagination/pvr_drv.h | 4 +- drivers/gpu/drm/imagination/pvr_fw_meta.c | 6 +- drivers/gpu/drm/imagination/pvr_fw_riscv.c | 6 +- drivers/gpu/drm/imagination/pvr_job.c | 6 + drivers/gpu/drm/imagination/pvr_mmu.c | 2 +- drivers/gpu/drm/imagination/pvr_mmu.h | 1 + drivers/gpu/drm/imagination/pvr_queue.c | 3 + drivers/gpu/drm/imagination/pvr_sync.c | 146 +++++- drivers/gpu/drm/imagination/pvr_sync.h | 13 + drivers/gpu/drm/imagination/pvr_vm.c | 781 ++++++++++++++++++++++------- drivers/gpu/drm/imagination/pvr_vm.h | 63 ++- include/uapi/drm/pvr_drm.h | 129 +++++ 15 files changed, 1303 insertions(+), 213 deletions(-)
---
base-commit: 2d5a8313d9cf989b8f53a5689a96e891bf99fb2c
change-id: 20260817-pvr-vm-bind-1ce4f09a1774
prerequisite-message-id: <[email protected]>
prerequisite-patch-id: 5188d944e90112e4cf2678e75255c7a91434ad09
Best regards,
Signature · 1 line
Gyeyoung Baek <[email protected]>
PATCH v2 1/6 drm/imagination: Switch the GPUVM to immediate mode
Use DRM_GPUVM_IMMEDIATE_MODE so that each GEM's GPUVA list is protected by its gpuva.lock instead of its dma_resv. This prepares for asynchronous VM_BIND, whose run_job() will update the GPUVA lists before signalling its fence. That section imposes three rules:
1. No dma_resv: another thread may hold it while waiting for the job's fence, directly or through a dependency.
-> Protect the lists with gpuva.lock, and take it around
drm_gpuva_link().
2. No allocation under a lock run_job() takes: reclaim may wait for a fence that depends on the job.
-> Allocate the vm_bo before taking gpuva.lock:
drm_gpuvm_bo_obtain_locked() becomes drm_gpuvm_bo_create() + drm_gpuvm_bo_obtain_prealloc().
3. No freeing of GEM objects: that takes their dma_resv, and under gpuva.lock would free the held lock too.
-> Defer it: drm_gpuva_unlink() and drm_gpuvm_bo_put() become
drm_gpuva_unlink_defer() and drm_gpuvm_bo_put_deferred(), and drm_gpuvm_bo_deferred_cleanup() frees once the locks are dropped.
This also fixes an unlocked GPUVA list update: when a map fails, pvr_vm_bind_op_fini() drops the last vm_bo reference without holding the object's dma_resv.
Failing VM_MAP allocations with fail-nth shows
it on a lockdep kernel (real allocation failures under memory pressure take the same path):
WARNING: drivers/gpu/drm/drm_gpuvm.c:1661 at drm_gpuvm_bo_put+0x2ec/0x660 [drm_gpuvm] Call trace: drm_gpuvm_bo_put pvr_vm_bind_op_fini pvr_vm_map pvr_ioctl_vm_map
--- drivers/gpu/drm/imagination/pvr_vm.c | 39 +++++++++++++++++++++++++++--------- 1 file changed, 30 insertions(+), 9 deletions(-)
drivers/gpu/drm/imagination/pvr_vm.c +30 -9
@@ -214,7 +214,7 @@ static int pvr_vm_bind_op_exec(struct pvr_vm_bind_op *bind_op)static void pvr_vm_bind_op_fini (struct pvr_vm_bind_op *bind_op) { -drm_gpuvm_bo_put (bind_op->gpuvm_bo); +drm_gpuvm_bo_put_deferred (bind_op->gpuvm_bo);kfree (bind_op->new_va);kfree (bind_op->prev_va);
@@ -255,11 +255,11 @@ pvr_vm_bind_op_map_init(struct pvr_vm_bind_op *bind_op, bind_op->type =PVR_VM_BIND_TYPE_MAP ; -dma_resv_lock (obj->resv,NULL ); - bind_op->gpuvm_bo =drm_gpuvm_bo_obtain_locked (&vm_ctx->gpuvm_mgr, obj); -dma_resv_unlock (obj->resv); -if (IS_ERR (bind_op->gpuvm_bo)) -return PTR_ERR (bind_op->gpuvm_bo); + bind_op->gpuvm_bo =drm_gpuvm_bo_create (&vm_ctx->gpuvm_mgr, obj); +if (!bind_op->gpuvm_bo) +return -ENOMEM ; + + bind_op->gpuvm_bo =drm_gpuvm_bo_obtain_prealloc (bind_op->gpuvm_bo); bind_op->new_va =kzalloc_obj (*bind_op->new_va); bind_op->prev_va =kzalloc_obj (*bind_op->prev_va);
@@ -293,6 +293,8 @@ pvr_vm_bind_op_map_init(struct pvr_vm_bind_op *bind_op, err_bind_op_fini:pvr_vm_bind_op_fini (bind_op); +/* Drain what fini() deferred. */ +drm_gpuvm_bo_deferred_cleanup (&vm_ctx->gpuvm_mgr);return err; }
@@ -366,7 +368,11 @@ pvr_vm_gpuva_map(struct drm_gpuva_op *op, void *op_ctx)return err;drm_gpuva_map (&ctx->vm_ctx->gpuvm_mgr, &ctx->new_va->base, &op->map); + +mutex_lock (&op->map.gem.obj->gpuva.lock);drm_gpuva_link (&ctx->new_va->base, ctx->gpuvm_bo); +mutex_unlock (&op->map.gem.obj->gpuva.lock); + ctx->new_va =NULL ;return 0 ;
@@ -396,7 +402,7 @@ pvr_vm_gpuva_unmap(struct drm_gpuva_op *op, void *op_ctx)return err;drm_gpuva_unmap (&op->unmap); -drm_gpuva_unlink (op->unmap.va); +drm_gpuva_unlink_defer (op->unmap.va);kfree (to_pvr_vm_gpuva (op->unmap.va));return 0 ;
@@ -418,6 +424,7 @@ pvr_vm_gpuva_unmap(struct drm_gpuva_op *op, void *op_ctx)static int pvr_vm_gpuva_remap (struct drm_gpuva_op *op,void *op_ctx) { +struct drm_gem_object *obj = op->remap.unmap->va->gem.obj;/* The split parts belong to the object of the mapping being split. */ struct drm_gpuvm_bo *vm_bo = op->remap.unmap->va->vm_bo;struct pvr_vm_bind_op *ctx = op_ctx;
@@ -435,16 +442,20 @@ pvr_vm_gpuva_remap(struct drm_gpuva_op *op, void *op_ctx)drm_gpuva_remap (&ctx->prev_va->base, &ctx->next_va->base, &op->remap);if (op->remap.prev) { +mutex_lock (&obj->gpuva.lock);drm_gpuva_link (&ctx->prev_va->base, vm_bo); +mutex_unlock (&obj->gpuva.lock); ctx->prev_va =NULL ; }if (op->remap.next) { +mutex_lock (&obj->gpuva.lock);drm_gpuva_link (&ctx->next_va->base, vm_bo); +mutex_unlock (&obj->gpuva.lock); ctx->next_va =NULL ; } -drm_gpuva_unlink (op->remap.unmap->va); +drm_gpuva_unlink_defer (op->remap.unmap->va);kfree (to_pvr_vm_gpuva (op->remap.unmap->va));return 0 ;
@@ -568,7 +579,8 @@ pvr_vm_create_context(struct pvr_device *pvr_dev, bool is_userspace_context)drm_gem_private_object_init (&pvr_dev->base, &vm_ctx->dummy_gem,0 );drm_gpuvm_init (&vm_ctx->gpuvm_mgr, is_userspace_context ?"PowerVR-user-VM" :"PowerVR-FW-VM" , -0 , &pvr_dev->base, &vm_ctx->dummy_gem, +DRM_GPUVM_IMMEDIATE_MODE , &pvr_dev->base, + &vm_ctx->dummy_gem,0 ,PVR_PAGE_TABLE_ADDR_SPACE_SIZE ,0 ,0 , &pvr_vm_gpuva_ops);mutex_init (&vm_ctx->lock);
@@ -602,6 +614,7 @@ pvr_vm_context_release(struct kref *ref_count)pvr_fw_object_destroy (vm_ctx->fw_mem_ctx_obj);pvr_vm_unmap_all (vm_ctx); +drm_gpuvm_bo_deferred_cleanup (&vm_ctx->gpuvm_mgr);pvr_mmu_context_destroy (vm_ctx->mmu_ctx);drm_gem_private_object_fini (&vm_ctx->dummy_gem);
@@ -738,7 +751,9 @@ pvr_vm_map(struct pvr_vm_context *vm_ctx, struct pvr_gem_object *pvr_obj, err_cleanup:mutex_unlock (&vm_ctx->lock); +pvr_vm_bind_op_fini (&bind_op); +drm_gpuvm_bo_deferred_cleanup (&vm_ctx->gpuvm_mgr);return err; }
@@ -820,6 +835,8 @@ pvr_vm_unmap_obj(struct pvr_vm_context *vm_ctx, struct pvr_gem_object *pvr_obj, err =pvr_vm_unmap_obj_locked (vm_ctx, pvr_obj, device_addr, size);mutex_unlock (&vm_ctx->lock); +drm_gpuvm_bo_deferred_cleanup (&vm_ctx->gpuvm_mgr); +return err; }
@@ -854,6 +871,8 @@ pvr_vm_unmap(struct pvr_vm_context *vm_ctx, u64 device_addr, u64 size)mutex_unlock (&vm_ctx->lock); +drm_gpuvm_bo_deferred_cleanup (&vm_ctx->gpuvm_mgr); +return err; }
@@ -889,6 +908,8 @@ pvr_vm_unmap_all(struct pvr_vm_context *vm_ctx) }mutex_unlock (&vm_ctx->lock); + +drm_gpuvm_bo_deferred_cleanup (&vm_ctx->gpuvm_mgr); }/* Static data areas are determined by firmware. */
Signature · 1 line
2.43.0
PATCH v2 2/6 drm/imagination: Stop locking reservations around map and unmap
In reply to Gyeyoung BaekThe GPUVA lists are now protected by gpuva.lock instead of the reservations, so map and unmap no longer need to lock the reservations. Nothing else on that path relies on them.
--- drivers/gpu/drm/imagination/pvr_vm.c | 44 ------------------------------------ 1 file changed, 44 deletions(-)
drivers/gpu/drm/imagination/pvr_vm.c +0 -44
@@ -10,7 +10,6 @@#include "pvr_rogue_fwif.h" #include "pvr_rogue_heap_config.h" -#include <drm/drm_exec.h> #include <drm/drm_gem.h> #include <drm/drm_gpuvm.h> #include <drm/drm_print.h>
@@ -684,16 +683,6 @@ void pvr_destroy_vm_contexts_for_file(struct pvr_file *pvr_file) } } -static int -pvr_vm_lock_extra (struct drm_gpuvm_exec *vm_exec) -{ -struct pvr_vm_bind_op *bind_op = vm_exec->extra.priv; -struct pvr_gem_object *pvr_obj = bind_op->pvr_obj; - -/* Acquire lock on the GEM object being mapped/unmapped. */ -return drm_exec_lock_obj (&vm_exec->exec,gem_from_pvr_gem (pvr_obj)); -} -/** * pvr_vm_map() - Map a section of physical memory into a section of * device-virtual memory.
@@ -721,15 +710,6 @@ pvr_vm_map(struct pvr_vm_context *vm_ctx, struct pvr_gem_object *pvr_obj, u64 pvr_obj_offset, u64 device_addr, u64 size) { struct pvr_vm_bind_op bind_op = {0}; -struct drm_gpuvm_exec vm_exec = { - .vm = &vm_ctx->gpuvm_mgr, - .flags =DRM_EXEC_INTERRUPTIBLE_WAIT | -DRM_EXEC_IGNORE_DUPLICATES , - .extra = { - .fn = pvr_vm_lock_extra, - .priv = &bind_op, - }, - }; int err = pvr_vm_bind_op_map_init(&bind_op, vm_ctx, pvr_obj, pvr_obj_offset, device_addr,
@@ -741,15 +721,7 @@ pvr_vm_map(struct pvr_vm_context *vm_ctx, struct pvr_gem_object *pvr_obj,pvr_gem_object_get (pvr_obj);mutex_lock (&vm_ctx->lock); - err =drm_gpuvm_exec_lock (&vm_exec); -if (err) - goto err_cleanup; - err =pvr_vm_bind_op_exec (&bind_op); - -drm_gpuvm_exec_unlock (&vm_exec); - -err_cleanup:mutex_unlock (&vm_ctx->lock);pvr_vm_bind_op_fini (&bind_op);
@@ -782,15 +754,6 @@ pvr_vm_unmap_obj_locked(struct pvr_vm_context *vm_ctx, u64 device_addr, u64 size) {struct pvr_vm_bind_op bind_op = {0 }; -struct drm_gpuvm_exec vm_exec = { - .vm = &vm_ctx->gpuvm_mgr, - .flags =DRM_EXEC_INTERRUPTIBLE_WAIT | -DRM_EXEC_IGNORE_DUPLICATES , - .extra = { - .fn = pvr_vm_lock_extra, - .priv = &bind_op, - }, - };int err =pvr_vm_bind_op_unmap_init (&bind_op, vm_ctx, pvr_obj, device_addr, size);
@@ -799,15 +762,8 @@ pvr_vm_unmap_obj_locked(struct pvr_vm_context *vm_ctx,pvr_gem_object_get (pvr_obj); - err =drm_gpuvm_exec_lock (&vm_exec); -if (err) - goto err_cleanup; - err =pvr_vm_bind_op_exec (&bind_op); -drm_gpuvm_exec_unlock (&vm_exec); - -err_cleanup:pvr_vm_bind_op_fini (&bind_op);return err;
Signature · 1 line
2.43.0
PATCH v2 3/6 drm/imagination: Route map and unmap through pvr_vm_bind()
In reply to Gyeyoung BaekGive the VM layer a single kernel-side bind entry point, and move every existing map and unmap path onto it.
FW mappings and the VM_MAP/VM_UNMAP ioctls each had their own copy of the same sequence (prepare, lock, execute, clean up). Fold them into one pvr_vm_bind() taking an array of struct pvr_vm_bind_desc. The VM_BIND ioctl that follows uses the same entry point.
Before:
pvr_meta_vm_map() ----+
pvr_riscv_vm_map() ---+--> pvr_vm_map()
pvr_ioctl_vm_map() ---+ map_init -> lock -> exec -> unlock -> fini
pvr_meta_vm_unmap() --+
pvr_riscv_vm_unmap() -+--> pvr_vm_unmap_obj()
unmap_init -> lock -> exec -> unlock -> fini
pvr_ioctl_vm_unmap() ----> pvr_vm_unmap()
unmap_init -> lock -> find -> exec
-> unlock -> finiAfter:
pvr_meta_vm_map() ----+
pvr_riscv_vm_map() ---+--> pvr_vm_map() ---+
pvr_ioctl_vm_map() ---+ |
+--> pvr_vm_bind() pvr_meta_vm_unmap() --+ | init -> lock -> exec pvr_riscv_vm_unmap() -+--> pvr_vm_unmap() -+ -> unlock -> fini
pvr_ioctl_vm_unmap() -+
- pvr_vm_unmap() sets the exact flag of its descriptor, since VM_UNMAP only removes a mapping that exactly matches the range. An unmap through pvr_vm_bind() otherwise removes everything in the range, because the VM_BIND ioctl that follows takes an arbitrary range, like Vulkan sparse unbinding, rather than a previous binding.
- pvr_vm_unmap_all() is the exception and does not go through pvr_vm_bind().
- pvr_vm_bind(): with the asynchronous mode the VM_BIND ioctl gains afterwards, a queued request that fails part way through leaves the address space in a state nobody can reason about. The VM context is then marked unusable, and pvr_vm_bind() refuses it.
- pvr_vm_unmap_all(): it empties a VM context, so it has to succeed whatever state the context is in.
- The only userspace-visible change: VM_UNMAP returns -EINVAL instead of -ENOENT for an invalid address or size.
--- drivers/gpu/drm/imagination/pvr_fw_meta.c | 6 +- drivers/gpu/drm/imagination/pvr_fw_riscv.c | 6 +- drivers/gpu/drm/imagination/pvr_vm.c | 304 +++++++++++++++-------------- drivers/gpu/drm/imagination/pvr_vm.h | 46 ++++- 4 files changed, 205 insertions(+), 157 deletions(-)
drivers/gpu/drm/imagination/pvr_fw_meta.c +2 -4
@@ -542,10 +542,8 @@ pvr_meta_vm_map(struct pvr_device *pvr_dev, struct pvr_fw_object *fw_obj)static void pvr_meta_vm_unmap (struct pvr_device *pvr_dev,struct pvr_fw_object *fw_obj) { -struct pvr_gem_object *pvr_obj = fw_obj->gem; - -pvr_vm_unmap_obj (pvr_dev->kernel_vm_ctx, pvr_obj, - fw_obj->fw_mm_node.start, fw_obj->fw_mm_node.size); +pvr_vm_unmap (pvr_dev->kernel_vm_ctx, fw_obj->fw_mm_node.start, + fw_obj->fw_mm_node.size); } static bool
drivers/gpu/drm/imagination/pvr_fw_riscv.c +2 -4
@@ -149,10 +149,8 @@ pvr_riscv_vm_map(struct pvr_device *pvr_dev, struct pvr_fw_object *fw_obj)static void pvr_riscv_vm_unmap (struct pvr_device *pvr_dev,struct pvr_fw_object *fw_obj) { -struct pvr_gem_object *pvr_obj = fw_obj->gem; - -pvr_vm_unmap_obj (pvr_dev->kernel_vm_ctx, pvr_obj, - fw_obj->fw_mm_node.start, fw_obj->fw_mm_node.size); +pvr_vm_unmap (pvr_dev->kernel_vm_ctx, fw_obj->fw_mm_node.start, + fw_obj->fw_mm_node.size); } static bool
drivers/gpu/drm/imagination/pvr_vm.c +158 -146
454 lines not on this page · raw message
drivers/gpu/drm/imagination/pvr_vm.h +43 -3
53 lines not on this page · raw message
Signature · 1 line
2.43.0
[email protected]
AI reviewRe: [PATCH v2 3/6] drm/imagination: Route map and unmap through pvr_vm_bind()
Thank you for your contribution! Sashiko AI review found 1 potential issue to consider: - [High] drm/imagination: UAF and sync ordering bug in pvr_vm_bind() --
Thank you for your contribution! Sashiko AI review found 1 potential issue to consider:
- [High] drm/imagination: UAF and sync ordering bug in pvr_vm_bind() --
commit 858616fa9e0f9c862d2be468e58d23d170048080
Author: Gyeyoung Baek <[email protected]>
drm/imagination: Route map and unmap through pvr_vm_bind()
This commit consolidates map and unmap paths into a single pvr_vm_bind() entry point taking an array of struct pvr_vm_bind_desc.
Quoted 4 lines
drivers/gpu/drm/imagination/pvr_vm.c +0
-0
[ ... ]
Quoted 1 line
drivers/gpu/drm/imagination/pvr_vm.c +0
-0@@ -683,188 +692,191 @@ void pvr_destroy_vm_contexts_for_file(struct pvr_file *pvr_file)
[ ... ]
Quoted 87 lines
drivers/gpu/drm/imagination/pvr_vm.c +40
-26-static int -pvr_vm_unmap_obj_locked (struct pvr_vm_context *vm_ctx, -struct pvr_gem_object *pvr_obj, -u64 device_addr,u64 size) +static struct pvr_vm_bind_op * +pvr_vm_bind_ops_create (struct pvr_vm_context *vm_ctx, +const struct pvr_vm_bind_desc *descs,u32 desc_count) { -struct pvr_vm_bind_op bind_op = {0 }; - -int err =pvr_vm_bind_op_unmap_init (&bind_op, vm_ctx, pvr_obj, - device_addr, size); -if (err) -return err; +struct pvr_vm_bind_op *bind_ops; +int err; -pvr_gem_object_get (pvr_obj); + bind_ops =kvzalloc_objs (*bind_ops, desc_count,GFP_KERNEL_ACCOUNT ); +if (!bind_ops) +return ERR_PTR (-ENOMEM ); - err =pvr_vm_bind_op_exec (&bind_op); +for (u32 prepared =0 ; prepared < desc_count; prepared++) { + err =pvr_vm_bind_op_init (&bind_ops[prepared], vm_ctx, + &descs[prepared]); +if (err) { +pvr_vm_bind_ops_free (bind_ops, prepared); +/* Release the vm_bos that fini() deferred. */ +drm_gpuvm_bo_deferred_cleanup (&vm_ctx->gpuvm_mgr); +return ERR_PTR (err); + } -pvr_vm_bind_op_fini (&bind_op); +cond_resched (); + } -return err; +return bind_ops; }/** -* pvr_vm_unmap_obj() - Unmap an already mapped section of device-virtual -* memory. +* pvr_vm_bind() - Apply a batch of bind operations to a VM context. * @vm_ctx: Target VM context. -* @pvr_obj: Target PowerVR memory object. -* @device_addr: Virtual device address at the start of the target mapping. -* @size: Size of the target mapping. +* @descs: Operations to apply, in array order. +* @desc_count: Number of entries in @descs. +* +* Not atomic: if an operation fails, the earlier ones stay applied. * * Return: -* * 0 on success, -* * Any error encountered by pvr_vm_unmap_obj_locked. +* * 0 on success, or +* * A negative error code on failure. */ -int -pvr_vm_unmap_obj (struct pvr_vm_context *vm_ctx,struct pvr_gem_object *pvr_obj, -u64 device_addr,u64 size) +int pvr_vm_bind (struct pvr_vm_context *vm_ctx, +const struct pvr_vm_bind_desc *descs,u32 desc_count) { -int err; +struct pvr_vm_bind_op *bind_ops; +int err =0 ; + + bind_ops =pvr_vm_bind_ops_create (vm_ctx, descs, desc_count); +if (IS_ERR (bind_ops)) +return PTR_ERR (bind_ops);mutex_lock (&vm_ctx->lock); - err =pvr_vm_unmap_obj_locked (vm_ctx, pvr_obj, device_addr, size); + +for (u32 i =0 ; !err && i < desc_count; i++) + err =pvr_vm_bind_op_exec (&bind_ops[i]); +mutex_unlock (&vm_ctx->lock); +pvr_vm_bind_ops_free (bind_ops, desc_count);drm_gpuvm_bo_deferred_cleanup (&vm_ctx->gpuvm_mgr);return err; }
[ ... ]
Quoted 30 lines
drivers/gpu/drm/imagination/pvr_vm.c +6
-19@@ -862,25 +862,21 @@ pvr_vm_unmap(struct pvr_vm_context *vm_ctx, u64 device_addr, u64 size)void pvr_vm_unmap_all (struct pvr_vm_context *vm_ctx) { -u64 addr = vm_ctx->gpuvm_mgr.mm_start; -u64 end = vm_ctx->gpuvm_mgr.mm_start + vm_ctx->gpuvm_mgr.mm_range; -u64 va_addr, va_range; - -mutex_lock (&vm_ctx->lock); - -for (; addr < end; addr = va_addr + va_range) { -struct pvr_gem_object *pvr_obj; -struct drm_gpuva *va; - - va =drm_gpuva_find_first (&vm_ctx->gpuvm_mgr, addr, end - addr); -if (!va) -break ; - - pvr_obj =gem_to_pvr_gem (va->gem.obj); - va_addr = va->va.addr; - va_range = va->va.range; +const struct pvr_vm_bind_desc desc = { + .type =PVR_VM_BIND_TYPE_UNMAP , + .device_addr = vm_ctx->gpuvm_mgr.mm_start, + .size = vm_ctx->gpuvm_mgr.mm_range, + }; +struct pvr_vm_bind_op bind_op = {0 }; -WARN_ON (pvr_vm_unmap_obj_locked (vm_ctx, pvr_obj, va_addr, va_range)); - }
drivers/gpu/drm/imagination/pvr_vm.c pvr_vm_unmap(struct pvr_vm_context *vm_ctx, u64 device_addr, u64 size) · line 881
Quoted 10 lines
drivers/gpu/drm/imagination/pvr_vm.c +5
-0+if (WARN_ON (pvr_vm_bind_op_unmap_init (&bind_op, vm_ctx, &desc))) +return ; +mutex_lock (&vm_ctx->lock); +WARN_ON (pvr_vm_bind_op_exec (&bind_op));mutex_unlock (&vm_ctx->lock); +pvr_vm_bind_op_fini (&bind_op);drm_gpuvm_bo_deferred_cleanup (&vm_ctx->gpuvm_mgr); }
[Severity: High]
Can calling pvr_vm_bind_ops_free() and pvr_vm_bind_op_fini() after dropping vm_ctx->lock cause a use-after-free on op_ctx->curr_page.l0_table and l1_table?
During pvr_vm_bind_op_exec(), pvr_mmu_map() and pvr_mmu_unmap() leave raw pointers to the active page tables in op_ctx->curr_page.l0_table and l1_table with pending syncs recorded in op_ctx->sync_level_required. Previously, pvr_vm_unmap_obj_locked() ran pvr_vm_bind_op_fini() -> pvr_mmu_op_context_destroy() -> pvr_mmu_op_context_sync() while holding vm_ctx->lock. With vm_ctx->lock dropped before pvr_vm_bind_ops_free(), a second thread can unmap the remaining entries in the same L0 or L1 table and free it before the first thread syncs:
CPU0 CPU1pvr_vm_bind()
mutex_lock(&vm_ctx->lock)
pvr_vm_bind_op_exec(&bind_ops[0])
/* leaves curr_page.l0_table = T0,
T0->entry_count == 1 */
mutex_unlock(&vm_ctx->lock)
pvr_vm_bind()
mutex_lock(&vm_ctx->lock)
pvr_vm_bind_op_exec(...)
/* unmaps last entry in T0,
queues T0 on l0_free_tables */
mutex_unlock(&vm_ctx->lock)
pvr_vm_bind_ops_free()
...
pvr_page_table_l0_free(T0)
pvr_vm_bind_ops_free()
pvr_vm_bind_op_fini(&bind_ops[0])
pvr_mmu_op_context_destroy()
...
pvr_page_table_l0_sync(T0) <-- use-after-freeIn addition, when desc_count > 1 in pvr_vm_bind(), deferring pvr_mmu_op_context_sync() until pvr_vm_bind_ops_free() runs after the loop breaks the leaf-to-root (L0 -> L1 -> L2) DMA sync order across operations in the same batch (and a later unmap in the batch can free a table still referenced by an earlier op's curr_page). For instance, if bind_ops[0] allocates an L1 table T1 (requiring a level 2 sync) and bind_ops[1] attaches a new L0 table T0_b to T1, freeing bind_ops[0] first syncs T1 and L2 before freeing bind_ops[1] syncs T0_b.
Signature · 1 line
Sashiko AI review · https://sashiko.dev/#/patchset/[email protected]?part=3
PATCH v2 4/6 drm/imagination: Add VM_BIND ioctl
In reply to Gyeyoung BaekAdd DRM_IOCTL_PVR_VM_BIND, a batched version of VM_MAP and VM_UNMAP.
Problem with VM_MAP and VM_UNMAP:
vkQueueBindSparse hands over operations in bulk, which VM_MAP and VM_UNMAP turn into hundreds of ioctls.
Difference from VM_UNMAP:
VM_UNMAP only removes a mapping that exactly matches the range. VM_BIND's UNMAP removes everything in the range, splitting mappings that cross its boundaries, as Vulkan sparse unbinding takes an arbitrary range rather than a previous binding.
The flags and sync_ops fields are reserved for asynchronous requests and must be zero and empty until then, which fixes the layout.
Bump the driver interface minor version to 2.
--- Documentation/gpu/imagination/uapi.rst | 12 +++ drivers/gpu/drm/imagination/pvr_drv.c | 163 +++++++++++++++++++++++++++++++++ drivers/gpu/drm/imagination/pvr_drv.h | 4 +- include/uapi/drm/pvr_drm.h | 90 ++++++++++++++++++ 4 files changed, 268 insertions(+), 1 deletion(-)
Documentation/gpu/imagination/uapi.rst +12 -0
@@ -87,6 +87,18 @@ VM_MAP and VM_UNMAP :identifiers: drm_pvr_ioctl_vm_map_args drm_pvr_ioctl_vm_unmap_args +VM_BIND +------- +.. kernel-doc:: include/uapi/drm/pvr_drm.h + :doc: PowerVR IOCTL VM_BIND interface + +.. kernel-doc:: include/uapi/drm/pvr_drm.h + :doc: Flags for VM_BIND operations. + +.. kernel-doc:: include/uapi/drm/pvr_drm.h + :identifiers: drm_pvr_vm_bind_op + drm_pvr_ioctl_vm_bind_args + CREATE_CONTEXT and DESTROY_CONTEXT ---------------------------------- .. kernel-doc:: include/uapi/drm/pvr_drm.h
drivers/gpu/drm/imagination/pvr_drv.c +163 -0
175 lines not on this page · raw message
drivers/gpu/drm/imagination/pvr_drv.h +3 -1
@@ -14,9 +14,10 @@* Driver interface version: * - 1.0: Initial interface * - 1.1: Add DRM_PVR_DEV_QUERY_FW_INFO_GET query +* - 1.2: Add DRM_IOCTL_PVR_VM_BIND */ #define PVR_DRIVER_MAJOR 1 -#define PVR_DRIVER_MINOR 1 +#define PVR_DRIVER_MINOR 2#define PVR_DRIVER_PATCHLEVEL 0int pvr_get_uobj (u64 usr_ptr,u32 usr_size,u32 min_size,u32 obj_size,void *out);
@@ -61,6 +62,7 @@ int pvr_set_uobj_array(const struct drm_pvr_obj_array *out, u32 min_stride, u32#define PVR_UOBJ_MIN_SIZE (_obj_name) _Generic(_obj_name \ PVR_UOBJ_DECL(struct drm_pvr_job, hwrt) \ PVR_UOBJ_DECL(struct drm_pvr_sync_op, value) \ + PVR_UOBJ_DECL(struct drm_pvr_vm_bind_op, size) \ PVR_UOBJ_DECL(struct drm_pvr_dev_query_gpu_info, num_phantoms) \ PVR_UOBJ_DECL(struct drm_pvr_dev_query_runtime_info, cdm_max_local_mem_size_regs) \ PVR_UOBJ_DECL(struct drm_pvr_dev_query_quirks, _padding_c) \
include/uapi/drm/pvr_drm.h +90 -0
102 lines not on this page · raw message
Signature · 1 line
2.43.0
[email protected]
AI reviewRe: [PATCH v2 4/6] drm/imagination: Add VM_BIND ioctl
Thank you for your contribution! Sashiko AI review found 1 potential issue to consider: - [Critical] drm/imagination: UAF on page tables in pvr_vm_bind() after unlock --
Thank you for your contribution! Sashiko AI review found 1 potential issue to consider:
- [Critical] drm/imagination: UAF on page tables in pvr_vm_bind() after unlock
--
commit d5bbe10d571fc12d872392309205546a1d09b850
Author: Gyeyoung Baek <[email protected]>
drm/imagination: Add VM_BIND ioctl
This commit adds the DRM_IOCTL_PVR_VM_BIND ioctl to apply a batch of map and unmap operations to a VM context, and bumps the driver minor version to 2.
Quoted 5 lines
drivers/gpu/drm/imagination/pvr_drv.c +0
-0@@ -1135,6 +1135,168 @@ pvr_ioctl_vm_unmap(struct drm_device *drm_dev, void *raw_args,
[ ... ]
Quoted 4 lines
drivers/gpu/drm/imagination/pvr_drv.c +4
-0+static int +pvr_ioctl_vm_bind (struct drm_device *drm_dev,void *raw_args, +struct drm_file *file) +{
[ ... ]
drivers/gpu/drm/imagination/pvr_drv.c pvr_ioctl_vm_unmap(struct drm_device *drm_dev, void *raw_args, · line 1287
Quoted 8 lines
drivers/gpu/drm/imagination/pvr_drv.c +8
-0+ descs =pvr_vm_bind_descs_from_uapi (pvr_file, uapi_ops, args->ops.count); +kvfree (uapi_ops); +if (IS_ERR (descs)) { + err =PTR_ERR (descs); + goto err_put_vm_context; + } + + err =pvr_vm_bind (vm_ctx, descs, args->ops.count);
[Severity: Critical]
Can concurrent VM operations on the same vm_ctx cause a use-after-free on op_ctx->curr_page.l0_table and l1_table in pvr_vm_bind()?
drivers/gpu/drm/imagination/pvr_vm.c:pvr_vm_bind() {
...
mutex_lock(&vm_ctx->lock);
for (u32 i = 0; !err && i < desc_count; i++)
err = pvr_vm_bind_op_exec(&bind_ops[i]);
mutex_unlock(&vm_ctx->lock);
pvr_vm_bind_ops_free(bind_ops, desc_count);
...
}pvr_vm_bind_op_exec() caches raw pointers to L0 and L1 page tables in op_ctx->curr_page.l0_table and l1_table and defers the final DMA sync until pvr_vm_bind_ops_free() -> pvr_vm_bind_op_fini() -> pvr_mmu_op_context_destroy() -> pvr_mmu_op_context_sync(), which runs after vm_ctx->lock has been dropped:
CPU0 CPU1pvr_ioctl_vm_bind()
pvr_vm_bind()
mutex_lock(&vm_ctx->lock)
pvr_vm_bind_op_exec()
...
op_ctx->curr_page.l0_table = ...
mutex_unlock(&vm_ctx->lock)
pvr_ioctl_vm_bind() / pvr_ioctl_vm_unmap()
pvr_vm_bind()
mutex_lock(&vm_ctx->lock)
pvr_vm_bind_op_exec()
...
pvr_page_table_l1_remove()
mutex_unlock(&vm_ctx->lock)
pvr_vm_bind_ops_free()
...
pvr_page_table_l0_free()
pvr_vm_bind_ops_free()
...
pvr_mmu_op_context_sync_manual()
pvr_page_table_l0_sync(op_ctx->curr_page.l0_table)
<-- use-after-freeOnce CPU0 drops vm_ctx->lock, CPU1 can unmap the remaining entries in those page tables and free them before CPU0 runs pvr_mmu_op_context_sync_manual(), causing CPU0 to dereference freed page table structs and sync unmapped backing pages.
A later patch in this series, commit "drm/imagination: Add asynchronous VM_BIND", calls pvr_mmu_op_context_sync() and pvr_mmu_flush_exec() under vm_ctx->lock in pvr_vm_bind_run_job(), but synchronous pvr_vm_bind() still leaves them outside the lock.
Signature · 1 line
Sashiko AI review · https://sashiko.dev/#/patchset/[email protected]?part=4
PATCH v2 5/6 drm/imagination: Add asynchronous VM_BIND
In reply to Gyeyoung BaekAdd DRM_PVR_VM_BIND_ASYNC, which runs a VM_BIND request asynchronously. sync_ops can only be used with this flag.
Each VM context has one drm_sched queue for bind requests. A request with the flag set is queued there as a job and runs in submission order. The job runs on the CPU, so it waits for completion fences rather than scheduled ones, and syncs the page tables itself. pvr_mmu_op_context_sync() is made public for this.
A request that fails part way through makes its VM context unusable:
further binds and job submissions on it fail with -ECANCELED.
Converting a wait operation into a fence is split out into pvr_sync_wait_op_get_fence() and shared with pvr_sync_add_deps_to_job(), which behaves as before.
--- Documentation/gpu/imagination/uapi.rst | 3 + drivers/gpu/drm/imagination/pvr_drv.c | 127 ++++++++++- drivers/gpu/drm/imagination/pvr_job.c | 6 + drivers/gpu/drm/imagination/pvr_mmu.c | 2 +- drivers/gpu/drm/imagination/pvr_mmu.h | 1 + drivers/gpu/drm/imagination/pvr_queue.c | 3 + drivers/gpu/drm/imagination/pvr_sync.c | 146 +++++++++++-- drivers/gpu/drm/imagination/pvr_sync.h | 13 ++ drivers/gpu/drm/imagination/pvr_vm.c | 359 +++++++++++++++++++++++++++++++- drivers/gpu/drm/imagination/pvr_vm.h | 9 + include/uapi/drm/pvr_drm.h | 34 ++- 11 files changed, 673 insertions(+), 30 deletions(-)
Documentation/gpu/imagination/uapi.rst +3 -0
@@ -95,6 +95,9 @@ VM_BIND .. kernel-doc:: include/uapi/drm/pvr_drm.h :doc: Flags for VM_BIND operations. +.. kernel-doc:: include/uapi/drm/pvr_drm.h + :doc: Flags for the VM_BIND ioctl. + .. kernel-doc:: include/uapi/drm/pvr_drm.h :identifiers: drm_pvr_vm_bind_op drm_pvr_ioctl_vm_bind_args
drivers/gpu/drm/imagination/pvr_drv.c +120 -7
175 lines not on this page · raw message
drivers/gpu/drm/imagination/pvr_job.c +6 -0
@@ -15,6 +15,7 @@#include "pvr_stream_defs.h" #include "pvr_sync.h" #include "pvr_trace.h" +#include "pvr_vm.h" #include <drm/drm_exec.h> #include <drm/drm_gem.h>
@@ -434,6 +435,11 @@ create_job(struct pvr_device *pvr_dev, goto err_put_job; } +if (pvr_vm_context_is_unusable (job->ctx->vm_ctx)) { + err = -ECANCELED ; + goto err_put_job; + } +if (args->hwrt.set_handle) { job->hwrt =pvr_hwrt_data_lookup (pvr_file, args->hwrt.set_handle, args->hwrt.data_index);
drivers/gpu/drm/imagination/pvr_mmu.c +1 -1
@@ -2017,7 +2017,7 @@ pvr_mmu_op_context_sync_partial(struct pvr_mmu_op_context *op_ctx,* that subsequent calls to this function will be no-ops unless @op_ctx is * otherwise updated. */ -static void +void pvr_mmu_op_context_sync (struct pvr_mmu_op_context *op_ctx) {pvr_mmu_op_context_sync_manual (op_ctx, op_ctx->sync_level_required);
drivers/gpu/drm/imagination/pvr_mmu.h +1 -0
@@ -97,6 +97,7 @@ void pvr_mmu_context_destroy(struct pvr_mmu_context *ctx);dma_addr_t pvr_mmu_get_root_table_dma_addr (struct pvr_mmu_context *ctx);void pvr_mmu_op_context_destroy (struct pvr_mmu_op_context *op_ctx); +void pvr_mmu_op_context_sync (struct pvr_mmu_op_context *op_ctx);struct pvr_mmu_op_context *pvr_mmu_op_context_create (struct pvr_mmu_context *ctx,struct sg_table *sgt,u64 device_addr,u64 sgt_offset,u64 size);
drivers/gpu/drm/imagination/pvr_queue.c +3 -0
@@ -759,6 +759,9 @@ static struct dma_fence *pvr_queue_run_job(struct drm_sched_job *sched_job)return dma_fence_get (job->done_fence); } +if (pvr_vm_context_is_unusable (job->ctx->vm_ctx)) +return ERR_PTR (-ECANCELED ); +/* The only kind of jobs that can be paired are geometry and fragment, and * we bail out early if we see a fragment job that's paired with a geometry job. * Paired jobs must also target the same context and point to the same HWRT.
drivers/gpu/drm/imagination/pvr_sync.c +130 -16
175 lines not on this page · raw message
drivers/gpu/drm/imagination/pvr_sync.h +13 -0
@@ -9,6 +9,9 @@/* Forward declaration from <linux/xarray.h>. */ struct xarray ; +/* Forward declaration from <linux/dma-fence.h>. */ +struct dma_fence ; +/* Forward declaration from <drm/drm_file.h>. */ struct drm_file ;
@@ -72,6 +75,9 @@ pvr_sync_signal_array_update_fences(struct xarray *array,const struct drm_pvr_sync_op *sync_ops, struct dma_fence *finished_fence); +void +pvr_sync_signal_array_set_fence (struct xarray *array,struct dma_fence *fence); +void pvr_sync_signal_array_push_fences (struct xarray *array);
@@ -81,4 +87,11 @@ pvr_sync_add_deps_to_job(struct pvr_file *pvr_file, struct drm_sched_job *job,const struct drm_pvr_sync_op *sync_ops, struct xarray *signal_array); +void +pvr_sync_put_fences (struct dma_fence **fences,u32 count); + +struct dma_fence ** +pvr_sync_get_wait_fences (struct drm_file *file,u32 sync_op_count, +const struct drm_pvr_sync_op *sync_ops,u32 *count); +#endif /* PVR_SYNC_H */
drivers/gpu/drm/imagination/pvr_vm.c +357 -2
419 lines not on this page · raw message
drivers/gpu/drm/imagination/pvr_vm.h +9 -0
@@ -23,6 +23,9 @@ struct pvr_vm_context;/* Forward declaration from <uapi/drm/pvr_drm.h> */ struct drm_pvr_ioctl_get_heap_info_args ; +/* Forward declaration from <linux/dma-fence.h> */ +struct dma_fence ; +/* Forward declaration from <drm/drm_exec.h> */ struct drm_exec ;
@@ -81,9 +84,15 @@ struct pvr_vm_bind_desc {int pvr_vm_bind (struct pvr_vm_context *vm_ctx,const struct pvr_vm_bind_desc *descs,u32 desc_count); +struct dma_fence * +pvr_vm_bind_async (struct pvr_vm_context *vm_ctx, +const struct pvr_vm_bind_desc *descs,u32 desc_count, +struct dma_fence **deps,u32 dep_count,u64 client_id);void pvr_vm_unmap_all (struct pvr_vm_context *vm_ctx); +bool pvr_vm_context_is_unusable (struct pvr_vm_context *vm_ctx); +dma_addr_t pvr_vm_get_page_table_root_addr (struct pvr_vm_context *vm_ctx);struct dma_resv *pvr_vm_get_dma_resv (struct pvr_vm_context *vm_ctx);
include/uapi/drm/pvr_drm.h +30 -4
@@ -1314,12 +1314,20 @@ struct drm_pvr_ioctl_submit_jobs_args {* DOC: PowerVR IOCTL VM_BIND interface * * %DRM_IOCTL_PVR_VM_BIND applies a batch of map and/or unmap operations to a -* single VM context before the IOCTL returns. +* single VM context, synchronously or, with %DRM_PVR_VM_BIND_ASYNC, from a +* queue. * -* Operations within a request are applied in array order. +* Operations within a request are applied in array order, and queued requests +* in submission order per VM context. A synchronous request does not wait for +* the queued ones. * * A request that fails part way through may leave the address space in an * undefined state; how much of it was applied is not reported. +* +* If a queued request fails, its VM context becomes permanently unusable: +* further %DRM_IOCTL_PVR_VM_BIND and %DRM_IOCTL_PVR_SUBMIT_JOBS calls on it +* fail with -%ECANCELED, and jobs already queued on it complete with an error. +* A new VM context has to be created. */ /**
@@ -1379,6 +1387,17 @@ struct drm_pvr_vm_bind_op {__u64 size; }; +/** +* DOC: Flags for the VM_BIND ioctl. +* +* .. c:macro:: DRM_PVR_VM_BIND_ASYNC +* +* Queue the request instead of applying it synchronously. +*/ +#define DRM_PVR_VM_BIND_ASYNC _BITUL(0) + +#define DRM_PVR_VM_BIND_FLAGS_MASK DRM_PVR_VM_BIND_ASYNC +/** * struct drm_pvr_ioctl_vm_bind_args - Arguments for %DRM_IOCTL_PVR_VM_BIND. */
@@ -1389,13 +1408,20 @@ struct drm_pvr_ioctl_vm_bind_args {*/ __u32 vm_context_handle; -/** @flags: [IN] Reserved. MBZ. */ +/** @flags: [IN] Combination of ``DRM_PVR_VM_BIND_`` flags. */ __u32 flags;/** @ops: [IN] Array of &struct drm_pvr_vm_bind_op to apply. */ struct drm_pvr_obj_array ops; -/** @sync_ops: [IN] Reserved. Must be empty. */ +/** +* @sync_ops: [IN] Sync operations applied once to the request as a +* whole: waits before any of @ops, signals after all of them. Must be +* empty unless %DRM_PVR_VM_BIND_ASYNC is set in @flags. +* +* @ops may be empty, in which case the request only places a +* synchronisation point on the queue. +*/ struct drm_pvr_obj_array sync_ops; };
Signature · 1 line
2.43.0
[email protected]
AI reviewRe: [PATCH v2 5/6] drm/imagination: Add asynchronous VM_BIND
Thank you for your contribution! Sashiko AI review found 7 potential issues to consider: - [High] drm/imagination: use-after-free of vm_ctx->sched via exported bind fence - [High] drm/imagination…
Thank you for your contribution! Sashiko AI review found 7 potential issues to consider:
- [High] drm/imagination: use-after-free of vm_ctx->sched via exported bind fence
- [High] drm/imagination: KCCB slot leak in pvr_queue_run_job() on unusable VM
- [High] drm/imagination: paired fragment job hang in pvr_queue_run_job() on cancel
- [High] drm/imagination: vm_ctx reference cycle with queued jobs in pvr_vm_bind_async()
- [High] drm/imagination: dma_fence signalling violation in pvr_vm_bind_run_job()
- [High] drm/imagination: premature BO free via concurrent drm_gpuvm_bo_deferred_cleanup()
- [Medium] drm/imagination: unlocked arm and push in pvr_vm_bind_job_push()
--
commit 6a1b0d839030cd6b32797f1866d0043e0dd0721f
Author: Gyeyoung Baek <[email protected]>
drm/imagination: Add asynchronous VM_BIND
This commit adds DRM_PVR_VM_BIND_ASYNC to queue VM_BIND operations on a per-VM-context drm_sched queue with sync_ops wait and signal support, and marks the VM context unusable if an asynchronous bind fails.
Quoted 4 lines
drivers/gpu/drm/imagination/pvr_drv.c +0
-0
[ ... ]
Quoted 13 lines
drivers/gpu/drm/imagination/pvr_drv.c +8
-1@@ -1284,8 +1372,33 @@ pvr_ioctl_vm_bind(struct drm_device *drm_dev, void *raw_args, goto err_put_vm_context; } - err =pvr_vm_bind (vm_ctx, descs, args->ops.count); +/* Empty in synchronous mode, rejected above otherwise. */ + err =pvr_vm_bind_syncs_from_uapi (&syncs, file, &args->sync_ops); +if (err) + goto err_descs_release; + +if (args->flags &DRM_PVR_VM_BIND_ASYNC ) { +struct dma_fence *fence; +
drivers/gpu/drm/imagination/pvr_drv.c pvr_ioctl_vm_bind(struct drm_device *drm_dev, void *raw_args, · line 1392
Quoted 10 lines
drivers/gpu/drm/imagination/pvr_drv.c +10
-0+ fence =pvr_vm_bind_async (vm_ctx, descs, args->ops.count, + syncs.waits, syncs.wait_count, + file->client_id); +if (IS_ERR (fence)) { + err =PTR_ERR (fence); + goto err_syncs_release; + } + +pvr_sync_signal_array_set_fence (&syncs.signals, fence); +pvr_sync_signal_array_push_fences (&syncs.signals);
[Severity: High]
Can exporting the bind job's finished fence into syncs.signals cause a use-after-free of vm_ctx->sched?
pvr_vm_bind_async() returns &job->base.s_fence->finished, whose sched_fence->sched points to &vm_ctx->sched embedded in struct pvr_vm_context. Because pvr_vm_bind_run_job() returns NULL (or an ERR_PTR), s_fence->parent is NULL and s_fence holds no reference to vm_ctx. Once the bind job finishes and pvr_vm_bind_job_destroy() drops job->vm_ctx, userspace can destroy the VM context while the fence remains installed in a drm_syncobj. A later DRM_IOCTL_PVR_SUBMIT_JOBS waiting on that syncobj then dereferences the freed vm_ctx->sched:
drivers/gpu/drm/imagination/pvr_queue.c:pvr_queue_fence_is_native() {
...
if (sched_fence &&
sched_fence->sched->ops == &pvr_queue_sched_ops)
return true;
...
}[ ... ]
drivers/gpu/drm/imagination/pvr_queue.c static struct dma_fence *pvr_queue_run_job(struct drm_sched_job *sched_job) · line 764
Quoted 11 lines
drivers/gpu/drm/imagination/pvr_queue.c +3
-0@@ -759,6 +759,9 @@ static struct dma_fence *pvr_queue_run_job(struct drm_sched_job *sched_job)return dma_fence_get (job->done_fence); } +if (pvr_vm_context_is_unusable (job->ctx->vm_ctx)) +return ERR_PTR (-ECANCELED ); +
[Severity: High]
Does returning ERR_PTR(-ECANCELED) here leak a reserved KCCB slot?
Before pvr_queue_run_job() runs, pvr_queue_prepare_job() -> pvr_queue_get_job_kccb_fence() calls pvr_kccb_reserve_slot() and sets job->kccb_fence = NULL. Returning early without calling pvr_kccb_release_slot(pvr_dev) leaves pvr_dev->kccb.reserved_count incremented; after pvr_kccb_capacity(pvr_dev) jobs fail this way, all KCCB slots on the device are exhausted and further KCCB submissions deadlock.
[Severity: High]
Also, if a paired geometry job fails with -ECANCELED here, will its paired fragment job hang the fragment queue?
During the geometry job's prepare_job(), pvr_queue_get_paired_frag_job_dep() already initializes frag_job->done_fence->ops:
drivers/gpu/drm/imagination/pvr_queue.c:pvr_queue_get_paired_frag_job_dep() {
...
/* Initialize the paired fragment job's done_fence, so we can signal it. */
pvr_queue_job_fence_init(frag_job->done_fence, frag_queue);
...
}When the geometry job returns ERR_PTR(-ECANCELED) without submitting the paired kick to the firmware, the fragment job is still scheduled onto frag_queue. In pvr_queue_run_job(), the fragment job hits the earlier job->done_fence->ops check before pvr_vm_context_is_unusable() and returns the unsignaled done_fence, hanging frag_queue until pvr_queue_timedout_job() fires WARN_ON(atomic_read(&queue->in_flight_job_count) != job_count). Checking pvr_vm_context_is_unusable() before the paired fragment check (or checking job->has_pm_ref instead of job->done_fence->ops) would avoid this.
Quoted 4 lines
drivers/gpu/drm/imagination/pvr_vm.c +0
-0
[ ... ]
drivers/gpu/drm/imagination/pvr_vm.c pvr_vm_context_release(struct kref *ref_count) · line 659
Quoted 9 lines
drivers/gpu/drm/imagination/pvr_vm.c +2
-0@@ -618,6 +652,8 @@ pvr_vm_context_release(struct kref *ref_count)struct pvr_vm_context *vm_ctx = container_of(ref_count, struct pvr_vm_context, ref_count); +pvr_vm_bind_sched_fini (vm_ctx); +if (vm_ctx->fw_mem_ctx_obj)pvr_fw_object_destroy (vm_ctx->fw_mem_ctx_obj);
[Severity: High]
Does calling pvr_vm_bind_sched_fini() only from pvr_vm_context_release() create a reference cycle when bind jobs are queued?
Each queued pvr_vm_bind_job holds a reference on vm_ctx via job->vm_ctx = pvr_vm_context_get(vm_ctx) until the job finishes or is killed by drm_sched_entity_destroy(&vm_ctx->entity). Because neither pvr_ioctl_destroy_vm_context() nor pvr_destroy_vm_contexts_for_file() destroys vm_ctx->entity, vm_ctx->ref_count cannot reach zero while any bind job is waiting on an unsignaled dependency fence, leaking vm_ctx, the job, and its pinned GEM objects if the process is killed. On file close, pvr_destroy_contexts_for_file() also calls pvr_vm_unmap_all(ctx->vm_ctx) while pending async bind jobs on vm_ctx->entity can still run later and map buffers back into the VM context.
[ ... ]
Quoted 52 lines
drivers/gpu/drm/imagination/pvr_vm.c +52
-0+static void pvr_vm_bind_job_destroy (struct pvr_vm_bind_job *job) +{ +if (!job) +return ; + +pvr_vm_bind_ops_free (job->bind_ops, job->bind_op_count); + +if (job->vm_ctx) { +drm_gpuvm_bo_deferred_cleanup (&job->vm_ctx->gpuvm_mgr); +pvr_vm_context_put (job->vm_ctx); + } + +kfree (job); +} + +static void pvr_vm_bind_job_cleanup_work (struct work_struct *work) +{ +struct pvr_vm_bind_job *job = + container_of(work, struct pvr_vm_bind_job, cleanup_work); + +pvr_vm_bind_job_destroy (job); +} + +/* +* Runs in the dma-fence signalling path: nothing here may allocate or take a +* dma_resv. pvr_vm_bind_job_create() prepares everything beforehand. +*/ +static struct dma_fence * +pvr_vm_bind_run_job (struct drm_sched_job *sched_job) +{ +struct pvr_vm_bind_job *job =to_pvr_vm_bind_job (sched_job); +struct pvr_vm_context *vm_ctx = job->vm_ctx; +bool wait = false; +int flush_err; +int err =0 ; +bool cookie; + +if (pvr_vm_context_is_unusable (vm_ctx)) +return ERR_PTR (-ECANCELED ); + + cookie =dma_fence_begin_signalling (); + +mutex_lock (&vm_ctx->lock); + +/* +* Sync the page tables and flush the MMU before signalling, including +* tables a failed operation has already changed. +*/ +for (u32 i =0 ; i < job->bind_op_count; i++) { +struct pvr_vm_bind_op *bind_op = &job->bind_ops[i]; + + err =pvr_vm_bind_op_exec (bind_op);
drivers/gpu/drm/imagination/pvr_vm.c pvr_vm_get_fw_mem_context(struct pvr_vm_context *vm_ctx) · line 1311
Quoted 10 lines
drivers/gpu/drm/imagination/pvr_vm.c +10
-0+ +pvr_mmu_op_context_sync (bind_op->mmu_op_ctx); +if (bind_op->type ==PVR_VM_BIND_TYPE_UNMAP ) + wait = true; + +if (err) +break ; + } + + flush_err =pvr_mmu_flush_exec (vm_ctx->pvr_dev, wait);
[Severity: High]
Can calling pvr_mmu_flush_exec() inside dma_fence_begin_signalling() violate dma_fence signalling rules?
If KCCB submission or completion fails in pvr_mmu_flush_exec(), its err_reset_and_retry path calls pvr_power_reset(pvr_dev, true) -> pvr_fw_hard_reset() -> pvr_fw_object_vmap() -> pvr_gem_object_vmap(), which takes dma_resv_lock() and allocates virtual address space with GFP_KERNEL via drm_gem_shmem_vmap_locked() while inside the dma_fence signalling section and holding vm_ctx->lock. Even on the normal path, pvr_mmu_flush_exec() -> pvr_kccb_send_cmd() -> pvr_power_get() calls pm_runtime_resume_and_get() inside the signalling section.
[Severity: High]
Can a concurrent drm_gpuvm_bo_deferred_cleanup() free an unmapped BO's pages before pvr_vm_bind_run_job() flushes the MMU?
During pvr_vm_bind_op_exec(), pvr_vm_gpuva_unmap() and pvr_vm_gpuva_remap() call drm_gpuva_unlink_defer(), queuing unlinked drm_gpuvm_bo entries onto
the lockless vm_ctx->gpuvm_mgr.bo_defer list before pvr_mmu_op_context_sync()
and pvr_mmu_flush_exec() complete:
kworker 0 kworker 1pvr_vm_bind_run_job()
mutex_lock(&vm_ctx->lock)
pvr_vm_bind_op_exec()
...
drm_gpuva_unlink_defer()
pvr_vm_bind_job_cleanup_work()
pvr_vm_bind_job_destroy()
drm_gpuvm_bo_deferred_cleanup()
/* drops last BO ref, frees pages */ pvr_mmu_op_context_sync()
pvr_mmu_flush_exec() <-- GPU TLB still cached freed pages
Because pvr_dev->sched_wq is an unbound multi-threaded workqueue and drm_gpuvm_bo_deferred_cleanup() runs without vm_ctx->lock in pvr_vm_bind_job_destroy() (as well as in pvr_vm_bind() and pvr_vm_bind_ops_create()), the backing pages can be freed while the GPU MMU can still access them.
[ ... ]
drivers/gpu/drm/imagination/pvr_vm.c pvr_vm_get_fw_mem_context(struct pvr_vm_context *vm_ctx) · line 1485
Quoted 13 lines
drivers/gpu/drm/imagination/pvr_vm.c +13
-0+static struct dma_fence *pvr_vm_bind_job_push (struct pvr_vm_bind_job *job) +{ +struct dma_fence *fence; + +drm_sched_job_arm (&job->base); + +/* Pushing may free @job, so take the fence first. */ + fence =dma_fence_get (&job->base.s_fence->finished); + +drm_sched_entity_push_job (&job->base); + +return fence; +}
[Severity: Medium]
Should drm_sched_job_arm() and drm_sched_entity_push_job() be serialized by a lock on vm_ctx?
Without a lock held across both calls, two concurrent DRM_PVR_VM_BIND_ASYNC ioctls on the same vm_ctx can arm jobs in one order (assigning fence seqnos N and N+1 on vm_ctx->entity) and push them to vm_ctx->entity in the reverse order:
CPU0 CPU1pvr_vm_bind_job_push(jobA) pvr_vm_bind_job_push(jobB)
drm_sched_job_arm(jobA)
/* gets seqno N */
drm_sched_job_arm(jobB)
/* gets seqno N+1 */
drm_sched_entity_push_job(jobB) drm_sched_entity_push_job(jobA) <-- seqno N+1 runs before seqno N
This breaks dma_fence seqno ordering on entity->fence_context and causes drm_sched_job_add_dependency() to drop a dependency on unsignaled fence N when deduplicating against fence N+1.
Signature · 1 line
Sashiko AI review · https://sashiko.dev/#/patchset/[email protected]?part=5
PATCH v2 6/6 drm/imagination: Add MAP_SPARSE to VM_BIND
In reply to Gyeyoung BaekVulkan sparse residency requires that an access to an unbound part of a sparse resource does not fault, so a range has to be mappable without naming any buffer object.
Add DRM_PVR_VM_BIND_OP_MAP_SPARSE, which maps one dummy object over and over across the range.
To save memory, one dummy is shared by all sparse mappings, but per file rather than per device: writes to a sparse range land in the dummy, so per file keeps clients isolated from each other.
The dummy is 2MiB, so pvr_mmu_map() runs once per 2MiB, but the mapping itself is still done per page.
--- drivers/gpu/drm/imagination/pvr_device.h | 10 ++++ drivers/gpu/drm/imagination/pvr_drv.c | 63 +++++++++++++++++++++++-- drivers/gpu/drm/imagination/pvr_vm.c | 81 +++++++++++++++++++++++++++++--- drivers/gpu/drm/imagination/pvr_vm.h | 8 ++++ include/uapi/drm/pvr_drm.h | 21 +++++++-- 5 files changed, 169 insertions(+), 14 deletions(-)
drivers/gpu/drm/imagination/pvr_device.h +10 -0
@@ -380,6 +380,16 @@ struct pvr_file {*/ struct xarray hwrt_handles; +/** +* @sparse_dummy_bo: Object the file's sparse mappings repeat. Pins its +* pages, so it is created on first use, under @sparse_dummy_lock. Per +* file, so that sparse writes stay out of other clients' view. +*/ +struct pvr_gem_object *sparse_dummy_bo; + +/** @sparse_dummy_lock: Serialises the creation of @sparse_dummy_bo. */ +struct mutex sparse_dummy_lock; +/** * @vm_ctx_handles: Array of VM contexts belonging to this file. Array * members are of type "struct pvr_vm_context *".
drivers/gpu/drm/imagination/pvr_drv.c +59 -4
102 lines not on this page · raw message
drivers/gpu/drm/imagination/pvr_vm.c +75 -6
128 lines not on this page · raw message
drivers/gpu/drm/imagination/pvr_vm.h +8 -0
@@ -80,6 +80,12 @@ struct pvr_vm_bind_desc {/** @size: Size of the range. */ u64 size; + +/** +* @sparse: Repeat @obj over the range rather than covering it once. +* Map only. +*/ +bool sparse; }; int pvr_vm_bind(struct pvr_vm_context *vm_ctx,
@@ -93,6 +99,8 @@ void pvr_vm_unmap_all(struct pvr_vm_context *vm_ctx);bool pvr_vm_context_is_unusable (struct pvr_vm_context *vm_ctx); +struct pvr_gem_object *pvr_vm_sparse_dummy_create (struct pvr_device *pvr_dev); +dma_addr_t pvr_vm_get_page_table_root_addr (struct pvr_vm_context *vm_ctx);struct dma_resv *pvr_vm_get_dma_resv (struct pvr_vm_context *vm_ctx);
include/uapi/drm/pvr_drm.h +17 -4
@@ -1333,7 +1333,17 @@ struct drm_pvr_ioctl_submit_jobs_args {/** * DOC: Flags for VM_BIND operations. * -* Set exactly one operation type in &drm_pvr_vm_bind_op.flags. +* Set exactly one operation type in &drm_pvr_vm_bind_op.flags, optionally +* ORed with modifiers. +* +* .. c:macro:: DRM_PVR_VM_BIND_OP_MAP_SPARSE +* +* Sparsely map a range without naming a buffer object. Only valid with +* %DRM_PVR_VM_BIND_OP_TYPE_MAP. +* +* GPU accesses to the range do not fault. Nothing else is guaranteed: +* reads return undefined values, and writes may become visible through +* other sparse mappings. * * .. c:macro:: DRM_PVR_VM_BIND_OP_TYPE_MAP *
@@ -1349,11 +1359,14 @@ struct drm_pvr_ioctl_submit_jobs_args {* * Mask used to extract the operation type. */ +#define DRM_PVR_VM_BIND_OP_MAP_SPARSE _BITUL(0) +#define DRM_PVR_VM_BIND_OP_TYPE_MAP (0u << 28)#define DRM_PVR_VM_BIND_OP_TYPE_UNMAP (1u << 28)#define DRM_PVR_VM_BIND_OP_TYPE_MASK (0xfu << 28) -#define DRM_PVR_VM_BIND_OP_FLAGS_MASK DRM_PVR_VM_BIND_OP_TYPE_MASK +#define DRM_PVR_VM_BIND_OP_FLAGS_MASK \ + (DRM_PVR_VM_BIND_OP_MAP_SPARSE | DRM_PVR_VM_BIND_OP_TYPE_MASK)/** * struct drm_pvr_vm_bind_op - A single VM bind operation.
@@ -1366,13 +1379,13 @@ struct drm_pvr_vm_bind_op {* @handle: [IN] Handle of the target buffer object. * * Must be a valid handle returned by %DRM_IOCTL_PVR_CREATE_BO for map -* operations. MBZ for unmap operations. +* operations. MBZ for unmap and sparse map operations. */ __u32 handle;/** * @offset: [IN] Offset into the target buffer object from which to -* begin the mapping. MBZ for unmap operations. +* begin the mapping. MBZ for unmap and sparse map operations. */ __u64 offset;
Signature · 1 line
2.43.0