Enable SVM atomics in Xe / GPU SVM
Subsystems (4)
Matthew Brost
maintainerv7 cover Enable SVM atomics in Xe / GPU SVM
Minimal set of patches to enable compute UMD SVM atomics.
Collaboration with Himal.
Matt
Himal Prasad Ghimiray (1):
drm/gpusvm: Introduce devmem_only flag for allocation
Matthew Brost (4):
drm/xe: Strict migration policy for atomic SVM faults drm/gpusvm: Add timeslicing support to GPU SVM drm/xe: Timeslice GPU on atomic SVM fault drm/xe: Add atomic_svm_timeslice_ms debugfs entry
drivers/gpu/drm/drm_gpusvm.c | 37 +++++++-- drivers/gpu/drm/xe/xe_debugfs.c | 38 +++++++++ drivers/gpu/drm/xe/xe_device.c | 1 + drivers/gpu/drm/xe/xe_device_types.h | 3 + drivers/gpu/drm/xe/xe_module.c | 3 - drivers/gpu/drm/xe/xe_module.h | 1 - drivers/gpu/drm/xe/xe_pt.c | 14 +++- drivers/gpu/drm/xe/xe_svm.c | 116 +++++++++++++++++++++------ drivers/gpu/drm/xe/xe_svm.h | 5 -- include/drm/drm_gpusvm.h | 47 +++++++---- 10 files changed, 208 insertions(+), 57 deletions(-)
Signature · 1 line
2.34.1
Matthew Brost
maintainerPATCH v7 1/5 drm/gpusvm: Introduce devmem_only flag for allocation
From: Himal Prasad Ghimiray <[email protected]>
This commit adds a new flag, devmem_only, to the drm_gpusvm structure. The purpose of this flag is to ensure that the get_pages function allocates memory exclusively from the device's memory. If the allocation from device memory fails, the function will return an -EFAULT error.
Required for shared CPU and GPU atomics on certain devices.
v3:
- s/vram_only/devmem_only/
--- drivers/gpu/drm/drm_gpusvm.c | 5 +++++ include/drm/drm_gpusvm.h | 2 ++ 2 files changed, 7 insertions(+)
drivers/gpu/drm/drm_gpusvm.c +5 -0
@@ -1454,6 +1454,11 @@ int drm_gpusvm_range_get_pages(struct drm_gpusvm *gpusvm, goto err_unmap; } +if (ctx->devmem_only) { + err = -EFAULT ; + goto err_unmap; + } + addr = dma_map_page(gpusvm->drm->dev, page, 0, PAGE_SIZE << order,
include/drm/drm_gpusvm.h +2 -0
@@ -286,6 +286,7 @@ struct drm_gpusvm {* @in_notifier: entering from a MMU notifier * @read_only: operating on read-only memory * @devmem_possible: possible to use device memory +* @devmem_only: use only device memory * * Context that is DRM GPUSVM is operating in (i.e. user arguments). */
@@ -294,6 +295,7 @@ struct drm_gpusvm_ctx { unsigned int in_notifier :1; unsigned int read_only :1; unsigned int devmem_possible :1; + unsigned int devmem_only :1; }; int drm_gpusvm_init(struct drm_gpusvm *gpusvm,
Signature · 1 line
2.34.1
Matthew Brost
maintainerPATCH v7 2/5 drm/xe: Strict migration policy for atomic SVM faults
In reply to Matthew BrostMixing GPU and CPU atomics does not work unless a strict migration policy of GPU atomics must be device memory. Enforce a policy of must be in VRAM with a retry loop of 3 attempts, if retry loop fails abort fault.
Removing always_migrate_to_vram modparam as we now have real migration policy.
v2:
- Only retry migration on atomics
- Drop alway migrate modparam
v3:
- Only set vram_only on DGFX (Himal)
- Bail on get_pages failure if vram_only and retry count exceeded (Himal)
- s/vram_only/devmem_only
- Update xe_svm_range_is_valid to accept devmem_only argument
v4:
- Fix logic bug get_pages failure
v5:
- Fix commit message (Himal)
- Mention removing always_migrate_to_vram in commit message (Lucas)
- Fix xe_svm_range_is_valid to check for devmem pages
- Bail on devmem_only && !migrate_devmem (Thomas)
v6:
- Add READ_ONCE barriers for opportunistic checks (Thomas)
- Pair READ_ONCE with WRITE_ONCE (Thomas)
v7:
- Adjust comments (Thomas)
--- drivers/gpu/drm/drm_gpusvm.c | 23 +++++-- drivers/gpu/drm/xe/xe_module.c | 3 - drivers/gpu/drm/xe/xe_module.h | 1 - drivers/gpu/drm/xe/xe_pt.c | 14 ++++- drivers/gpu/drm/xe/xe_svm.c | 110 +++++++++++++++++++++++++-------- drivers/gpu/drm/xe/xe_svm.h | 5 -- include/drm/drm_gpusvm.h | 40 +++++++----- 7 files changed, 139 insertions(+), 57 deletions(-)
drivers/gpu/drm/drm_gpusvm.c +18 -5
@@ -1118,6 +1118,10 @@ static void __drm_gpusvm_range_unmap_pages(struct drm_gpusvm *gpusvm,lockdep_assert_held (&gpusvm->notifier_lock);if (range->flags.has_dma_mapping) { +struct drm_gpusvm_range_flags flags = { + .__flags = range->flags.__flags, + }; +for (i =0 , j =0 ; i < npages; j++) {struct drm_pagemap_device_addr *addr = &range->dma_addr[j];
@@ -1131,8 +1135,12 @@ static void __drm_gpusvm_range_unmap_pages(struct drm_gpusvm *gpusvm, dev, *addr); i +=1 << addr->order; } - range->flags.has_devmem_pages = false; - range->flags.has_dma_mapping = false; + +/* WRITE_ONCE pairs with READ_ONCE for opportunistic checks */ + flags.has_devmem_pages = false; + flags.has_dma_mapping = false; +WRITE_ONCE (range->flags.__flags, flags.__flags); + range->dpagemap =NULL ; } }
@@ -1334,6 +1342,7 @@ int drm_gpusvm_range_get_pages(struct drm_gpusvm *gpusvm,int err =0 ;struct dev_pagemap *pagemap;struct drm_pagemap *dpagemap; +struct drm_gpusvm_range_flags flags; retry: hmm_range.notifier_seq =mmu_interval_read_begin (notifier);
@@ -1378,7 +1387,8 @@ int drm_gpusvm_range_get_pages(struct drm_gpusvm *gpusvm,*/ drm_gpusvm_notifier_lock (gpusvm); -if (range->flags.unmapped) { + flags.__flags = range->flags.__flags; +if (flags.unmapped) {drm_gpusvm_notifier_unlock (gpusvm); err = -EFAULT ; goto err_free;
@@ -1474,14 +1484,17 @@ int drm_gpusvm_range_get_pages(struct drm_gpusvm *gpusvm, } i +=1 << order; num_dma_mapped = i; - range->flags.has_dma_mapping = true; + flags.has_dma_mapping = true; }if (zdd) { - range->flags.has_devmem_pages = true; + flags.has_devmem_pages = true; range->dpagemap = dpagemap; } +/* WRITE_ONCE pairs with READ_ONCE for opportunistic checks */ +WRITE_ONCE (range->flags.__flags, flags.__flags); +drm_gpusvm_notifier_unlock (gpusvm);kvfree (pfns); set_seqno:
drivers/gpu/drm/xe/xe_module.c +0 -3
@@ -33,9 +33,6 @@ struct xe_modparam xe_modparam = {module_param_named (svm_notifier_size, xe_modparam.svm_notifier_size, uint,0600 );MODULE_PARM_DESC (svm_notifier_size,"Set the svm notifier size(in MiB), must be power of 2" ); -module_param_named (always_migrate_to_vram, xe_modparam.always_migrate_to_vram, bool,0444 ); -MODULE_PARM_DESC (always_migrate_to_vram,"Always migrate to VRAM on GPU fault" ); -module_param_named_unsafe (force_execlist, xe_modparam.force_execlist, bool,0444 );MODULE_PARM_DESC (force_execlist,"Force Execlist submission" );
drivers/gpu/drm/xe/xe_module.h +0 -1
@@ -12,7 +12,6 @@struct xe_modparam {bool force_execlist;bool probe_display; -bool always_migrate_to_vram;u32 force_vram_bar_size;int guc_log_level;char *guc_firmware_path;
drivers/gpu/drm/xe/xe_pt.c +11 -3
@@ -2270,11 +2270,19 @@ static void op_commit(struct xe_vm *vm, }case DRM_GPUVA_OP_DRIVER : { +/* WRITE_ONCE pairs with READ_ONCE in xe_svm.c */ +if (op->subop ==XE_VMA_SUBOP_MAP_RANGE ) { - op->map_range.range->tile_present |=BIT (tile->id); - op->map_range.range->tile_invalidated &= ~BIT (tile->id); +WRITE_ONCE (op->map_range.range->tile_present, + op->map_range.range->tile_present | +BIT (tile->id)); +WRITE_ONCE (op->map_range.range->tile_invalidated, + op->map_range.range->tile_invalidated & + ~BIT (tile->id)); }else if (op->subop ==XE_VMA_SUBOP_UNMAP_RANGE ) { - op->unmap_range.range->tile_present &= ~BIT (tile->id); +WRITE_ONCE (op->unmap_range.range->tile_present, + op->unmap_range.range->tile_present & + ~BIT (tile->id)); }break ; }
drivers/gpu/drm/xe/xe_svm.c +84 -26
167 lines not on this page · raw message
drivers/gpu/drm/xe/xe_svm.h +0 -5
@@ -39,11 +39,6 @@ struct xe_svm_range {* range. Protected by GPU SVM notifier lock. */ u8 tile_invalidated; -/** -* @skip_migrate: Skip migration to VRAM, protected by GPU fault handler -* locking. -*/ - u8 skip_migrate :1; };/**
include/drm/drm_gpusvm.h +26 -14
@@ -185,6 +185,31 @@ struct drm_gpusvm_notifier { } flags; }; +/** +* struct drm_gpusvm_range_flags - Structure representing a GPU SVM range flags +* +* @migrate_devmem: Flag indicating whether the range can be migrated to device memory +* @unmapped: Flag indicating if the range has been unmapped +* @partial_unmap: Flag indicating if the range has been partially unmapped +* @has_devmem_pages: Flag indicating if the range has devmem pages +* @has_dma_mapping: Flag indicating if the range has a DMA mapping +* @__flags: Flags for range in u16 form (used for READ_ONCE) +*/ +struct drm_gpusvm_range_flags { +union { +struct { +/* All flags below must be set upon creation */ +u16 migrate_devmem :1 ; +/* All flags below must be set / cleared under notifier lock */ +u16 unmapped :1 ; +u16 partial_unmap :1 ; +u16 has_devmem_pages :1 ; +u16 has_dma_mapping :1 ; + }; +u16 __flags; + }; +}; +/** * struct drm_gpusvm_range - Structure representing a GPU SVM range *
@@ -198,11 +223,6 @@ struct drm_gpusvm_notifier {* @dpagemap: The struct drm_pagemap of the device pages we're dma-mapping. * Note this is assuming only one drm_pagemap per range is allowed. * @flags: Flags for range -* @flags.migrate_devmem: Flag indicating whether the range can be migrated to device memory -* @flags.unmapped: Flag indicating if the range has been unmapped -* @flags.partial_unmap: Flag indicating if the range has been partially unmapped -* @flags.has_devmem_pages: Flag indicating if the range has devmem pages -* @flags.has_dma_mapping: Flag indicating if the range has a DMA mapping * * This structure represents a GPU SVM range used for tracking memory ranges * mapped in a DRM device.
@@ -216,15 +236,7 @@ struct drm_gpusvm_range {unsigned long notifier_seq;struct drm_pagemap_device_addr *dma_addr;struct drm_pagemap *dpagemap; -struct { -/* All flags below must be set upon creation */ -u16 migrate_devmem :1 ; -/* All flags below must be set / cleared under notifier lock */ -u16 unmapped :1 ; -u16 partial_unmap :1 ; -u16 has_devmem_pages :1 ; -u16 has_dma_mapping :1 ; - } flags; +struct drm_gpusvm_range_flags flags; };/**
Signature · 1 line
2.34.1
Matthew Brost
maintainerPATCH v7 3/5 drm/gpusvm: Add timeslicing support to GPU SVM
In reply to Matthew BrostAdd timeslicing support to GPU SVM which will guarantee the GPU a minimum execution time on piece of physical memory before migration back to CPU. Intended to implement strict migration policies which require memory to be in a certain placement for correct execution.
Required for shared CPU and GPU atomics on certain devices.
--- drivers/gpu/drm/drm_gpusvm.c | 9 +++++++++ include/drm/drm_gpusvm.h | 5 +++++ 2 files changed, 14 insertions(+)
drivers/gpu/drm/drm_gpusvm.c +9 -0
@@ -1783,6 +1783,8 @@ int drm_gpusvm_migrate_to_devmem(struct drm_gpusvm *gpusvm, goto err_finalize;/* Upon success bind devmem allocation to range and zdd */ + devmem_allocation->timeslice_expiration =get_jiffies_64 () + +msecs_to_jiffies (ctx->timeslice_ms); zdd->devmem_allocation = devmem_allocation;/* Owns ref */ err_finalize:
@@ -2003,6 +2005,13 @@ static int __drm_gpusvm_migrate_to_ram(struct vm_area_struct *vas,void *buf;int i, err =0 ; +if (page) { + zdd = page->zone_device_data; +if (time_before64 (get_jiffies_64 (), + zdd->devmem_allocation->timeslice_expiration)) +return 0 ; + } + start =ALIGN_DOWN (fault_addr, size); end =ALIGN (fault_addr +1 , size);
include/drm/drm_gpusvm.h +5 -0
@@ -89,6 +89,7 @@ struct drm_gpusvm_devmem_ops {* @ops: Pointer to the operations structure for GPU SVM device memory * @dpagemap: The struct drm_pagemap of the pages this allocation belongs to. * @size: Size of device memory allocation +* @timeslice_expiration: Timeslice expiration in jiffies */ struct drm_gpusvm_devmem {struct device *dev;
@@ -97,6 +98,7 @@ struct drm_gpusvm_devmem {const struct drm_gpusvm_devmem_ops *ops;struct drm_pagemap *dpagemap;size_t size; +u64 timeslice_expiration; };/**
@@ -295,6 +297,8 @@ struct drm_gpusvm {* @check_pages_threshold: Check CPU pages for present if chunk is less than or * equal to threshold. If not present, reduce chunk * size. +* @timeslice_ms: The timeslice MS which in minimum time a piece of memory +* remains with either exclusive GPU or CPU access. * @in_notifier: entering from a MMU notifier * @read_only: operating on read-only memory * @devmem_possible: possible to use device memory
@@ -304,6 +308,7 @@ struct drm_gpusvm {*/ struct drm_gpusvm_ctx {unsigned long check_pages_threshold; +unsigned long timeslice_ms;unsigned int in_notifier :1 ;unsigned int read_only :1 ;unsigned int devmem_possible :1 ;
Signature · 1 line
2.34.1
Matthew Brost
maintainerPATCH v7 4/5 drm/xe: Timeslice GPU on atomic SVM fault
In reply to Matthew BrostEnsure GPU can make forward progress on an atomic SVM GPU fault by giving the GPU a timeslice of 5ms
v2:
- Reduce timeslice to 5ms
- Double timeslice on retry
- Split out GPU SVM changes into independent patch
v5:
- Double timeslice in a few more places
--- drivers/gpu/drm/xe/xe_svm.c | 5 +++++ 1 file changed, 5 insertions(+)
drivers/gpu/drm/xe/xe_svm.c +5 -0
@@ -797,6 +797,8 @@ int xe_svm_handle_pagefault(struct xe_vm *vm, struct xe_vma *vma, IS_ENABLED(CONFIG_DRM_XE_DEVMEM_MIRROR) ? SZ_64K : 0, .devmem_only = atomic &&IS_DGFX (vm->xe) && IS_ENABLED(CONFIG_DRM_XE_DEVMEM_MIRROR), + .timeslice_ms = atomic &&IS_DGFX (vm->xe) && + IS_ENABLED(CONFIG_DRM_XE_DEVMEM_MIRROR) ? 5 : 0, };struct xe_svm_range *range;struct drm_gpusvm_range *r;
@@ -836,6 +838,7 @@ int xe_svm_handle_pagefault(struct xe_vm *vm, struct xe_vma *vma,if (--migrate_try_count >=0 &&xe_svm_range_needs_migrate_to_vram (range, vma)) { err =xe_svm_alloc_vram (vm, tile, range, &ctx); + ctx.timeslice_ms <<=1 ;/* Double timeslice if we have to retry */ if (err) {if (migrate_try_count || !ctx.devmem_only) { drm_dbg(&vm->xe->drm,
@@ -855,6 +858,7 @@ int xe_svm_handle_pagefault(struct xe_vm *vm, struct xe_vma *vma, err =drm_gpusvm_range_get_pages (&vm->svm.gpusvm, r, &ctx);/* Corner where CPU mappings have changed */ if (err == -EOPNOTSUPP || err == -EFAULT || err == -EPERM ) { + ctx.timeslice_ms <<=1 ;/* Double timeslice if we have to retry */ if (migrate_try_count >0 || !ctx.devmem_only) {if (err == -EOPNOTSUPP ) {range_debug (range,"PAGE FAULT - EVICT PAGES" );
@@ -894,6 +898,7 @@ int xe_svm_handle_pagefault(struct xe_vm *vm, struct xe_vma *vma,drm_exec_fini (&exec); err =PTR_ERR (fence);if (err == -EAGAIN ) { + ctx.timeslice_ms <<=1 ;/* Double timeslice if we have to retry */ range_debug (range,"PAGE FAULT - RETRY BIND" ); goto retry; }
Signature · 1 line
2.34.1
Matthew Brost
maintainerPATCH v7 5/5 drm/xe: Add atomic_svm_timeslice_ms debugfs entry
In reply to Matthew BrostAdd some informal control for atomic SVM fault GPU timeslice to be able to play around with values and tweak performance.
v2:
- Reduce timeslice default value to 5ms
--- drivers/gpu/drm/xe/xe_debugfs.c | 38 ++++++++++++++++++++++++++++ drivers/gpu/drm/xe/xe_device.c | 1 + drivers/gpu/drm/xe/xe_device_types.h | 3 +++ drivers/gpu/drm/xe/xe_svm.c | 3 ++- 4 files changed, 44 insertions(+), 1 deletion(-)
drivers/gpu/drm/xe/xe_debugfs.c +38 -0
@@ -191,6 +191,41 @@ static const struct file_operations wedged_mode_fops = { .write = wedged_mode_set, }; +static ssize_t atomic_svm_timeslice_ms_show (struct file *f,char __user *ubuf, +size_t size,loff_t *pos) +{ +struct xe_device *xe =file_inode (f)->i_private; +char buf[32 ]; +int len =0 ; + + len =scnprintf (buf,sizeof (buf),"%d\n" , xe->atomic_svm_timeslice_ms); + +return simple_read_from_buffer (ubuf, size, pos, buf, len); +} + +static ssize_t atomic_svm_timeslice_ms_set (struct file *f, +const char __user *ubuf, +size_t size,loff_t *pos) +{ +struct xe_device *xe =file_inode (f)->i_private; +u32 atomic_svm_timeslice_ms; +ssize_t ret; + + ret =kstrtouint_from_user (ubuf, size,0 , &atomic_svm_timeslice_ms); +if (ret) +return ret; + + xe->atomic_svm_timeslice_ms = atomic_svm_timeslice_ms; + +return size; +} + +static const struct file_operations atomic_svm_timeslice_ms_fops = { + .owner =THIS_MODULE , + .read = atomic_svm_timeslice_ms_show, + .write = atomic_svm_timeslice_ms_set, +}; +void xe_debugfs_register (struct xe_device *xe) {struct ttm_device *bdev = &xe->ttm;
@@ -211,6 +246,9 @@ void xe_debugfs_register(struct xe_device *xe)debugfs_create_file ("wedged_mode" ,0600 , root, xe, &wedged_mode_fops); +debugfs_create_file ("atomic_svm_timeslice_ms" ,0600 , root, xe, + &atomic_svm_timeslice_ms_fops); +for (mem_type =XE_PL_VRAM0 ; mem_type <=XE_PL_VRAM1 ; ++mem_type) { man =ttm_manager_type (bdev, mem_type);
drivers/gpu/drm/xe/xe_device.c +1 -0
@@ -444,6 +444,7 @@ struct xe_device *xe_device_create(struct pci_dev *pdev, xe->info.devid = pdev->device; xe->info.revid = pdev->revision; xe->info.force_execlist = xe_modparam.force_execlist; + xe->atomic_svm_timeslice_ms =5 ; err =xe_irq_init (xe);if (err)
drivers/gpu/drm/xe/xe_device_types.h +3 -0
@@ -570,6 +570,9 @@ struct xe_device {/** @pmu: performance monitoring unit */ struct xe_pmu pmu; +/** @atomic_svm_timeslice_ms: Atomic SVM fault timeslice MS */ +u32 atomic_svm_timeslice_ms; +#ifdef TEST_VM_OPS_ERROR /** * @vm_inject_error_position: inject errors at different places in VM
drivers/gpu/drm/xe/xe_svm.c +2 -1
@@ -798,7 +798,8 @@ int xe_svm_handle_pagefault(struct xe_vm *vm, struct xe_vma *vma, .devmem_only = atomic && IS_DGFX(vm->xe) && IS_ENABLED(CONFIG_DRM_XE_DEVMEM_MIRROR), .timeslice_ms = atomic &&IS_DGFX (vm->xe) && - IS_ENABLED(CONFIG_DRM_XE_DEVMEM_MIRROR) ? 5 : 0, +IS_ENABLED (CONFIG_DRM_XE_DEVMEM_MIRROR ) ? + vm->xe->atomic_svm_timeslice_ms : 0, };struct xe_svm_range *range;struct drm_gpusvm_range *r;
Signature · 1 line
2.34.1