Add system_allocator test
v3 cover Add system_allocator test
Matthew Brost (2):
uapi/xe: Sync latest uAPI KMD headers
tests/xe: Add system_allocator test
include/drm-uapi/xe_drm.h | 49 +- lib/xe/xe_ioctl.c | 12 + lib/xe/xe_ioctl.h | 1 + tests/intel/xe_exec_system_allocator.c | 1849 ++++++++++++++++++++++++ tests/meson.build | 1 + 5 files changed, 1909 insertions(+), 3 deletions(-) create mode 100644 tests/intel/xe_exec_system_allocator.c
Signature · 1 line
2.34.1
PATCH v3 1/2 uapi/xe: Sync latest uAPI KMD headers
Pull in latest uAPI KMD headers to enable testing of new features.
--- include/drm-uapi/xe_drm.h | 49 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 46 insertions(+), 3 deletions(-)
include/drm-uapi/xe_drm.h +46 -3
@@ -393,6 +393,10 @@ struct drm_xe_query_mem_regions {* * - %DRM_XE_QUERY_CONFIG_FLAG_HAS_VRAM - Flag is set if the device * has usable VRAM +* - %DRM_XE_QUERY_CONFIG_FLAG_HAS_LOW_LATENCY - Flag is set if the device +* has low latency hint support +* - %DRM_XE_QUERY_CONFIG_FLAG_HAS_CPU_ADDR_MIRROR - Flag is set if the +* device has CPU address mirroring support * - %DRM_XE_QUERY_CONFIG_MIN_ALIGNMENT - Minimal memory alignment * required by this device, typically SZ_4K or SZ_64K * - %DRM_XE_QUERY_CONFIG_VA_BITS - Maximum bits of a virtual address
@@ -409,6 +413,8 @@ struct drm_xe_query_config {#define DRM_XE_QUERY_CONFIG_REV_AND_DEVICE_ID 0#define DRM_XE_QUERY_CONFIG_FLAGS 1#define DRM_XE_QUERY_CONFIG_FLAG_HAS_VRAM (1 << 0) +#define DRM_XE_QUERY_CONFIG_FLAG_HAS_LOW_LATENCY (1 << 1) +#define DRM_XE_QUERY_CONFIG_FLAG_HAS_CPU_ADDR_MIRROR (1 << 2)#define DRM_XE_QUERY_CONFIG_MIN_ALIGNMENT 2#define DRM_XE_QUERY_CONFIG_VA_BITS 3#define DRM_XE_QUERY_CONFIG_MAX_EXEC_QUEUE_PRIORITY 4
@@ -911,7 +917,11 @@ struct drm_xe_gem_mmap_offset {* struct drm_xe_vm_create - Input of &DRM_IOCTL_XE_VM_CREATE * * The @flags can be: -* - %DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE +* - %DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE - Map the whole virtual address +* space of the VM to scratch page. A vm_bind would overwrite the scratch +* page mapping. This flag is mutually exclusive with the +* %DRM_XE_VM_CREATE_FLAG_FAULT_MODE flag, with an exception of on x2 and +* xe3 platform. * - %DRM_XE_VM_CREATE_FLAG_LR_MODE - An LR, or Long Running VM accepts * exec submissions to its exec_queues that don't have an upper time * limit on the job execution time. But exec submissions to these
@@ -987,6 +997,12 @@ struct drm_xe_vm_destroy {* - %DRM_XE_VM_BIND_FLAG_CHECK_PXP - If the object is encrypted via PXP, * reject the binding if the encryption key is no longer valid. This * flag has no effect on BOs that are not marked as using PXP. +* - %DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR - When the CPU address mirror flag is +* set, no mappings are created rather the range is reserved for CPU address +* mirroring which will be populated on GPU page faults or prefetches. Only +* valid on VMs with DRM_XE_VM_CREATE_FLAG_FAULT_MODE set. The CPU address +* mirror flag are only valid for DRM_XE_VM_BIND_OP_MAP operations, the BO +* handle MBZ, and the BO offset MBZ. */ struct drm_xe_vm_bind_op {/** @extensions: Pointer to the first extension struct, if any */
@@ -1039,7 +1055,9 @@ struct drm_xe_vm_bind_op {* on the @pat_index. For such mappings there is no actual memory being * mapped (the address in the PTE is invalid), so the various PAT memory * attributes likely do not apply. Simply leaving as zero is one -* option (still a valid pat_index). +* option (still a valid pat_index). Same applies to +* DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR bindings as for such mapping +* there is no actual memory being mapped. */ __u16 pat_index;
@@ -1055,6 +1073,14 @@ struct drm_xe_vm_bind_op {/** @userptr: user pointer to bind on */ __u64 userptr; + +/** +* @cpu_addr_mirror_offset: Offset from GPU @addr to create +* CPU address mirror mappings. MBZ with current level of +* support (e.g. 1 to 1 mapping between GPU and CPU mappings +* only supported). +*/ +__s64 cpu_addr_mirror_offset; };/**
@@ -1078,6 +1104,7 @@ struct drm_xe_vm_bind_op {#define DRM_XE_VM_BIND_FLAG_NULL (1 << 2)#define DRM_XE_VM_BIND_FLAG_DUMPABLE (1 << 3)#define DRM_XE_VM_BIND_FLAG_CHECK_PXP (1 << 4) +#define DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR (1 << 5)/** @flags: Bind flags */ __u32 flags;
@@ -1205,6 +1232,21 @@ struct drm_xe_vm_bind {* }; * ioctl(fd, DRM_IOCTL_XE_EXEC_QUEUE_CREATE, &exec_queue_create); * +* Allow users to provide a hint to kernel for cases demanding low latency +* profile. Please note it will have impact on power consumption. User can +* indicate low latency hint with flag while creating exec queue as +* mentioned below, +* +* struct drm_xe_exec_queue_create exec_queue_create = { +* .flags = DRM_XE_EXEC_QUEUE_LOW_LATENCY_HINT, +* .extensions = 0, +* .vm_id = vm, +* .num_bb_per_exec = 1, +* .num_eng_per_bb = 1, +* .instances = to_user_pointer(&instance), +* }; +* ioctl(fd, DRM_IOCTL_XE_EXEC_QUEUE_CREATE, &exec_queue_create); +* */ struct drm_xe_exec_queue_create {#define DRM_XE_EXEC_QUEUE_EXTENSION_SET_PROPERTY 0
@@ -1223,7 +1265,8 @@ struct drm_xe_exec_queue_create {/** @vm_id: VM to use for this exec queue */ __u32 vm_id; -/** @flags: MBZ */ +#define DRM_XE_EXEC_QUEUE_LOW_LATENCY_HINT (1 << 0) +/** @flags: flags to use for this exec queue */ __u32 flags;/** @exec_queue_id: Returned exec queue ID */
Signature · 1 line
2.34.1
RE: [PATCH v3 1/2] uapi/xe: Sync latest uAPI KMD headers
-----Original Message-----
Quoted 9 lines
From: igt-dev <[email protected]> On Behalf Of Matthew Brost
Sent: Thursday, April 24, 2025 1:44 PM
To: [email protected]
Cc: Dugast, Francois <[email protected]>
Subject: [PATCH v3 1/2] uapi/xe: Sync latest uAPI KMD headersPull in latest uAPI KMD headers to enable testing of new features.
Signed-off-by Matthew Brost <[email protected]>
LGTM, though I am a bit surprised that cpu_addr_mirror_offset is a signed integer and not a uint.
-Jonathan Cavitt
Quoted 124 lines
--- include/drm-uapi/xe_drm.h | 49 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 46 insertions(+), 3 deletions(-)include/drm-uapi/xe_drm.h +46
-3@@ -393,6 +393,10 @@ struct drm_xe_query_mem_regions {* * - %DRM_XE_QUERY_CONFIG_FLAG_HAS_VRAM - Flag is set if the device * has usable VRAM +* - %DRM_XE_QUERY_CONFIG_FLAG_HAS_LOW_LATENCY - Flag is set if the device +* has low latency hint support +* - %DRM_XE_QUERY_CONFIG_FLAG_HAS_CPU_ADDR_MIRROR - Flag is set if the +* device has CPU address mirroring support * - %DRM_XE_QUERY_CONFIG_MIN_ALIGNMENT - Minimal memory alignment * required by this device, typically SZ_4K or SZ_64K * - %DRM_XE_QUERY_CONFIG_VA_BITS - Maximum bits of a virtual address @@ -409,6 +413,8 @@ struct drm_xe_query_config {#define DRM_XE_QUERY_CONFIG_REV_AND_DEVICE_ID 0#define DRM_XE_QUERY_CONFIG_FLAGS 1#define DRM_XE_QUERY_CONFIG_FLAG_HAS_VRAM (1 << 0) +#define DRM_XE_QUERY_CONFIG_FLAG_HAS_LOW_LATENCY (1 << 1) +#define DRM_XE_QUERY_CONFIG_FLAG_HAS_CPU_ADDR_MIRROR (1 << 2)#define DRM_XE_QUERY_CONFIG_MIN_ALIGNMENT 2#define DRM_XE_QUERY_CONFIG_VA_BITS 3#define DRM_XE_QUERY_CONFIG_MAX_EXEC_QUEUE_PRIORITY 4@@ -911,7 +917,11 @@ struct drm_xe_gem_mmap_offset {* struct drm_xe_vm_create - Input of &DRM_IOCTL_XE_VM_CREATE * * The @flags can be: -* - %DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE +* - %DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE - Map the whole virtual address +* space of the VM to scratch page. A vm_bind would overwrite the scratch +* page mapping. This flag is mutually exclusive with the +* %DRM_XE_VM_CREATE_FLAG_FAULT_MODE flag, with an exception of on x2 and +* xe3 platform. * - %DRM_XE_VM_CREATE_FLAG_LR_MODE - An LR, or Long Running VM accepts * exec submissions to its exec_queues that don't have an upper time * limit on the job execution time. But exec submissions to these @@ -987,6 +997,12 @@ struct drm_xe_vm_destroy {* - %DRM_XE_VM_BIND_FLAG_CHECK_PXP - If the object is encrypted via PXP, * reject the binding if the encryption key is no longer valid. This * flag has no effect on BOs that are not marked as using PXP. +* - %DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR - When the CPU address mirror flag is +* set, no mappings are created rather the range is reserved for CPU address +* mirroring which will be populated on GPU page faults or prefetches. Only +* valid on VMs with DRM_XE_VM_CREATE_FLAG_FAULT_MODE set. The CPU address +* mirror flag are only valid for DRM_XE_VM_BIND_OP_MAP operations, the BO +* handle MBZ, and the BO offset MBZ. */ struct drm_xe_vm_bind_op {/** @extensions: Pointer to the first extension struct, if any */ @@ -1039,7 +1055,9 @@ struct drm_xe_vm_bind_op {* on the @pat_index. For such mappings there is no actual memory being * mapped (the address in the PTE is invalid), so the various PAT memory * attributes likely do not apply. Simply leaving as zero is one -* option (still a valid pat_index). +* option (still a valid pat_index). Same applies to +* DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR bindings as for such mapping +* there is no actual memory being mapped. */ __u16 pat_index;@@ -1055,6 +1073,14 @@ struct drm_xe_vm_bind_op {/** @userptr: user pointer to bind on */ __u64 userptr; + +/** +* @cpu_addr_mirror_offset: Offset from GPU @addr to create +* CPU address mirror mappings. MBZ with current level of +* support (e.g. 1 to 1 mapping between GPU and CPU mappings +* only supported). +*/ +__s64 cpu_addr_mirror_offset; };/** @@ -1078,6 +1104,7 @@ struct drm_xe_vm_bind_op {#define DRM_XE_VM_BIND_FLAG_NULL (1 << 2)#define DRM_XE_VM_BIND_FLAG_DUMPABLE (1 << 3)#define DRM_XE_VM_BIND_FLAG_CHECK_PXP (1 << 4) +#define DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR (1 << 5)/** @flags: Bind flags */ __u32 flags;@@ -1205,6 +1232,21 @@ struct drm_xe_vm_bind {* }; * ioctl(fd, DRM_IOCTL_XE_EXEC_QUEUE_CREATE, &exec_queue_create); * +* Allow users to provide a hint to kernel for cases demanding low latency +* profile. Please note it will have impact on power consumption. User can +* indicate low latency hint with flag while creating exec queue as +* mentioned below, +* +* struct drm_xe_exec_queue_create exec_queue_create = { +* .flags = DRM_XE_EXEC_QUEUE_LOW_LATENCY_HINT, +* .extensions = 0, +* .vm_id = vm, +* .num_bb_per_exec = 1, +* .num_eng_per_bb = 1, +* .instances = to_user_pointer(&instance), +* }; +* ioctl(fd, DRM_IOCTL_XE_EXEC_QUEUE_CREATE, &exec_queue_create); +* */ struct drm_xe_exec_queue_create {#define DRM_XE_EXEC_QUEUE_EXTENSION_SET_PROPERTY 0@@ -1223,7 +1265,8 @@ struct drm_xe_exec_queue_create {/** @vm_id: VM to use for this exec queue */ __u32 vm_id; -/** @flags: MBZ */ +#define DRM_XE_EXEC_QUEUE_LOW_LATENCY_HINT (1 << 0) +/** @flags: flags to use for this exec queue */ __u32 flags;/** @exec_queue_id: Returned exec queue ID */ Signature · 1 line
2.34.1
Re: [PATCH v3 1/2] uapi/xe: Sync latest uAPI KMD headers
In reply to Matthew BrostQuoted 3 lines · Matthew Brost
Pull in latest uAPI KMD headers to enable testing of new features.
Signed-off-by Matthew Brost <[email protected]>
Quoted 123 lines
--- include/drm-uapi/xe_drm.h | 49 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 46 insertions(+), 3 deletions(-)include/drm-uapi/xe_drm.h +46
-3@@ -393,6 +393,10 @@ struct drm_xe_query_mem_regions {* * - %DRM_XE_QUERY_CONFIG_FLAG_HAS_VRAM - Flag is set if the device * has usable VRAM +* - %DRM_XE_QUERY_CONFIG_FLAG_HAS_LOW_LATENCY - Flag is set if the device +* has low latency hint support +* - %DRM_XE_QUERY_CONFIG_FLAG_HAS_CPU_ADDR_MIRROR - Flag is set if the +* device has CPU address mirroring support * - %DRM_XE_QUERY_CONFIG_MIN_ALIGNMENT - Minimal memory alignment * required by this device, typically SZ_4K or SZ_64K * - %DRM_XE_QUERY_CONFIG_VA_BITS - Maximum bits of a virtual address @@ -409,6 +413,8 @@ struct drm_xe_query_config {#define DRM_XE_QUERY_CONFIG_REV_AND_DEVICE_ID 0#define DRM_XE_QUERY_CONFIG_FLAGS 1#define DRM_XE_QUERY_CONFIG_FLAG_HAS_VRAM (1 << 0) +#define DRM_XE_QUERY_CONFIG_FLAG_HAS_LOW_LATENCY (1 << 1) +#define DRM_XE_QUERY_CONFIG_FLAG_HAS_CPU_ADDR_MIRROR (1 << 2)#define DRM_XE_QUERY_CONFIG_MIN_ALIGNMENT 2#define DRM_XE_QUERY_CONFIG_VA_BITS 3#define DRM_XE_QUERY_CONFIG_MAX_EXEC_QUEUE_PRIORITY 4@@ -911,7 +917,11 @@ struct drm_xe_gem_mmap_offset {* struct drm_xe_vm_create - Input of &DRM_IOCTL_XE_VM_CREATE * * The @flags can be: -* - %DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE +* - %DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE - Map the whole virtual address +* space of the VM to scratch page. A vm_bind would overwrite the scratch +* page mapping. This flag is mutually exclusive with the +* %DRM_XE_VM_CREATE_FLAG_FAULT_MODE flag, with an exception of on x2 and +* xe3 platform. * - %DRM_XE_VM_CREATE_FLAG_LR_MODE - An LR, or Long Running VM accepts * exec submissions to its exec_queues that don't have an upper time * limit on the job execution time. But exec submissions to these @@ -987,6 +997,12 @@ struct drm_xe_vm_destroy {* - %DRM_XE_VM_BIND_FLAG_CHECK_PXP - If the object is encrypted via PXP, * reject the binding if the encryption key is no longer valid. This * flag has no effect on BOs that are not marked as using PXP. +* - %DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR - When the CPU address mirror flag is +* set, no mappings are created rather the range is reserved for CPU address +* mirroring which will be populated on GPU page faults or prefetches. Only +* valid on VMs with DRM_XE_VM_CREATE_FLAG_FAULT_MODE set. The CPU address +* mirror flag are only valid for DRM_XE_VM_BIND_OP_MAP operations, the BO +* handle MBZ, and the BO offset MBZ. */ struct drm_xe_vm_bind_op {/** @extensions: Pointer to the first extension struct, if any */ @@ -1039,7 +1055,9 @@ struct drm_xe_vm_bind_op {* on the @pat_index. For such mappings there is no actual memory being * mapped (the address in the PTE is invalid), so the various PAT memory * attributes likely do not apply. Simply leaving as zero is one -* option (still a valid pat_index). +* option (still a valid pat_index). Same applies to +* DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR bindings as for such mapping +* there is no actual memory being mapped. */ __u16 pat_index;@@ -1055,6 +1073,14 @@ struct drm_xe_vm_bind_op {/** @userptr: user pointer to bind on */ __u64 userptr; + +/** +* @cpu_addr_mirror_offset: Offset from GPU @addr to create +* CPU address mirror mappings. MBZ with current level of +* support (e.g. 1 to 1 mapping between GPU and CPU mappings +* only supported). +*/ +__s64 cpu_addr_mirror_offset; };/** @@ -1078,6 +1104,7 @@ struct drm_xe_vm_bind_op {#define DRM_XE_VM_BIND_FLAG_NULL (1 << 2)#define DRM_XE_VM_BIND_FLAG_DUMPABLE (1 << 3)#define DRM_XE_VM_BIND_FLAG_CHECK_PXP (1 << 4) +#define DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR (1 << 5)/** @flags: Bind flags */ __u32 flags;@@ -1205,6 +1232,21 @@ struct drm_xe_vm_bind {* }; * ioctl(fd, DRM_IOCTL_XE_EXEC_QUEUE_CREATE, &exec_queue_create); * +* Allow users to provide a hint to kernel for cases demanding low latency +* profile. Please note it will have impact on power consumption. User can +* indicate low latency hint with flag while creating exec queue as +* mentioned below, +* +* struct drm_xe_exec_queue_create exec_queue_create = { +* .flags = DRM_XE_EXEC_QUEUE_LOW_LATENCY_HINT, +* .extensions = 0, +* .vm_id = vm, +* .num_bb_per_exec = 1, +* .num_eng_per_bb = 1, +* .instances = to_user_pointer(&instance), +* }; +* ioctl(fd, DRM_IOCTL_XE_EXEC_QUEUE_CREATE, &exec_queue_create); +* */ struct drm_xe_exec_queue_create {#define DRM_XE_EXEC_QUEUE_EXTENSION_SET_PROPERTY 0@@ -1223,7 +1265,8 @@ struct drm_xe_exec_queue_create {/** @vm_id: VM to use for this exec queue */ __u32 vm_id; -/** @flags: MBZ */ +#define DRM_XE_EXEC_QUEUE_LOW_LATENCY_HINT (1 << 0) +/** @flags: flags to use for this exec queue */ __u32 flags;/** @exec_queue_id: Returned exec queue ID */ Signature · 1 line
2.34.1
PATCH v3 2/2 tests/xe: Add system_allocator test
In reply to Matthew BrostTest various uses of system allocator in single thread, multiple threads, and multiple processes.
Features tested:
- Malloc with various size
- Mmap with various sizes and flags including file backed mappings
- Mixing BO allocations with system allocator
- Various page sizes
- Dynamically freeing / unmapping memory
- Sharing VM across threads
- Faults racing on different hardware engines / GTs / Tiles
- GPU faults and CPU faults racing
- CPU faults on multiple threads racing
- CPU faults on multiple process racing
- GPU faults of memory not faulted in by CPU
- Partial unmap of allocations
- Attempting to unmap system allocations when GPU has mappings
- Eviction of both system allocations and BOs
- Forking child processes and reading data from VRAM
- mremap data in VRAM
- Protection changes
- Multiple faults per execbuf
Running on LNL, BMG, PVC 1 tile, and PVC 2 tile.
v2:
- Rebase
- Fix memory allocation to not interfear with malloc (Thomas)
v3:
- Fix memory leak (Francois)
- Break out uAPI into own patch (Francois)
- Use mkstemp for sync file (Francois)
- Use mkstemp for file backed data (Francois)
- Drop i argument from READ_VALUE (Francois)
- Fix test description (Francois)
- Add comment to check_all_pages_process (Francois)
- Prefer igt_info over printf (Francois)
- Fix types in messages (Francois)
- Prefer odd macro (Francois)
--- lib/xe/xe_ioctl.c | 12 + lib/xe/xe_ioctl.h | 1 + tests/intel/xe_exec_system_allocator.c | 1849 ++++++++++++++++++++++++ tests/meson.build | 1 + 4 files changed, 1863 insertions(+) create mode 100644 tests/intel/xe_exec_system_allocator.c
lib/xe/xe_ioctl.c +12 -0
@@ -440,6 +440,18 @@ void *xe_bo_map(int fd, uint32_t bo, size_t size)return __xe_bo_map (fd, bo, size,PROT_WRITE ); } +void *xe_bo_map_fixed (int fd,uint32_t bo,size_t size,uint64_t addr) +{ +uint64_t mmo; +void *map; + + mmo =xe_bo_mmap_offset (fd, bo); + map =mmap ((void *)addr, size,PROT_WRITE ,MAP_SHARED |MAP_FIXED , fd, mmo); +igt_assert (map !=MAP_FAILED ); + +return map; +} +void *xe_bo_mmap_ext (int fd,uint32_t bo,size_t size,int prot) {return __xe_bo_map (fd, bo, size, prot);
lib/xe/xe_ioctl.h +1 -0
@@ -86,6 +86,7 @@ uint32_t xe_exec_queue_create_class(int fd, uint32_t vm, uint16_t class);void xe_exec_queue_destroy (int fd,uint32_t exec_queue);uint64_t xe_bo_mmap_offset (int fd,uint32_t bo);void *xe_bo_map (int fd,uint32_t bo,size_t size); +void *xe_bo_map_fixed (int fd,uint32_t bo,size_t size,long unsigned int addr);void *xe_bo_mmap_ext (int fd,uint32_t bo,size_t size,int prot);int __xe_exec (int fd,struct drm_xe_exec *exec);void xe_exec (int fd,struct drm_xe_exec *exec);
tests/intel/xe_exec_system_allocator.c new file +1849 -0
1,849 lines not on this page · raw message
tests/meson.build +1 -0
@@ -295,6 +295,7 @@ intel_xe_progs = [ 'xe_exec_reset', 'xe_exec_sip', 'xe_exec_store', + 'xe_exec_system_allocator', 'xe_exec_threads', 'xe_exercise_blt', 'xe_fault_injection',
Signature · 1 line
2.34.1
RE: [PATCH v3 2/2] tests/xe: Add system_allocator test
-----Original Message-----
From: igt-dev <[email protected]> On Behalf Of Matthew Brost
Sent: Thursday, April 24, 2025 1:44 PM
To: [email protected]
Cc: Dugast, Francois <[email protected]>
Subject: [PATCH v3 2/2] tests/xe: Add system_allocator test
Quoted 41 lines
Test various uses of system allocator in single thread, multiple threads, and multiple processes.
Features tested:
- Malloc with various size
- Mmap with various sizes and flags including file backed mappings
- Mixing BO allocations with system allocator
- Various page sizes
- Dynamically freeing / unmapping memory
- Sharing VM across threads
- Faults racing on different hardware engines / GTs / Tiles
- GPU faults and CPU faults racing
- CPU faults on multiple threads racing
- CPU faults on multiple process racing
- GPU faults of memory not faulted in by CPU
- Partial unmap of allocations
- Attempting to unmap system allocations when GPU has mappings
- Eviction of both system allocations and BOs
- Forking child processes and reading data from VRAM
- mremap data in VRAM
- Protection changes
- Multiple faults per execbufRunning on LNL, BMG, PVC 1 tile, and PVC 2 tile.
v2:
- Rebase
- Fix memory allocation to not interfear with malloc (Thomas)
v3:
- Fix memory leak (Francois)
- Break out uAPI into own patch (Francois)
- Use mkstemp for sync file (Francois)
- Use mkstemp for file backed data (Francois)
- Drop i argument fromREAD_VALUE(Francois)
- Fix test description (Francois)
- Add comment tocheck_all_pages_process(Francois)
- Preferigt_infoover printf (Francois)
- Fix types in messages (Francois)
- Prefer odd macro (Francois)
Quoted 1 line
Signed-off-by Matthew Brost <[email protected]>
LGTM, though maybe it would be better to split the various tests introduced in xe_exec_system_allocator.c across multiple patches, instead of creating all of the tests at once?
I'm okay with creating the test series either way, so this isn't a blocking request.
The remaining nits below are non-blocking as well, so this is:
Quoted 49 lines
--- lib/xe/xe_ioctl.c | 12 + lib/xe/xe_ioctl.h | 1 + tests/intel/xe_exec_system_allocator.c | 1849 ++++++++++++++++++++++++ tests/meson.build | 1 + 4 files changed, 1863 insertions(+) create mode 100644 tests/intel/xe_exec_system_allocator.clib/xe/xe_ioctl.c +12
-0@@ -440,6 +440,18 @@ void *xe_bo_map(int fd, uint32_t bo, size_t size)return __xe_bo_map (fd, bo, size,PROT_WRITE ); } +void *xe_bo_map_fixed (int fd,uint32_t bo,size_t size,uint64_t addr) +{ +uint64_t mmo; +void *map; + + mmo =xe_bo_mmap_offset (fd, bo); + map =mmap ((void *)addr, size,PROT_WRITE ,MAP_SHARED |MAP_FIXED , fd, mmo); +igt_assert (map !=MAP_FAILED ); + +return map; +} +void *xe_bo_mmap_ext (int fd,uint32_t bo,size_t size,int prot) {return __xe_bo_map (fd, bo, size, prot);lib/xe/xe_ioctl.h +1
-0@@ -86,6 +86,7 @@ uint32_t xe_exec_queue_create_class(int fd, uint32_t vm, uint16_t class);void xe_exec_queue_destroy (int fd,uint32_t exec_queue);uint64_t xe_bo_mmap_offset (int fd,uint32_t bo);void *xe_bo_map (int fd,uint32_t bo,size_t size); +void *xe_bo_map_fixed (int fd,uint32_t bo,size_t size,long unsigned int addr);void *xe_bo_mmap_ext (int fd,uint32_t bo,size_t size,int prot);int __xe_exec (int fd,struct drm_xe_exec *exec);void xe_exec (int fd,struct drm_xe_exec *exec);tests/intel/xe_exec_system_allocator.c new file +0
-0
tests/intel/xe_exec_system_allocator.c line 3
Quoted 3 lines
tests/intel/xe_exec_system_allocator.c new file +3
-0@@ -0,0 +1,1849 @@ +// SPDX-License-Identifier: MIT +/* +* Copyright (c) 2024 Intel Corporation
NIT:
Shouldn't this be a 2025 copyright?
Quoted 75 lines
tests/intel/xe_exec_system_allocator.c new file +75
-0+*/ + +/** +* TEST: Basic tests for execbuf functionality using system allocator +* Category: Core +* Mega feature: USM +* Sub-category: System allocator +* Functionality: fault mode, system allocator +* GPU: LNL, BMG, PVC +*/ + +#include <fcntl.h> +#include <linux/mman.h> +#include <time.h> + +#include "igt.h" +#include "lib/igt_syncobj.h" +#include "lib/intel_reg.h" +#include "xe_drm.h" + +#include "xe/xe_ioctl.h" +#include "xe/xe_query.h" +#include <string.h> + +#define USER_FENCE_VALUE 0xdeadbeefdeadbeefull +#define QUARTER_SEC (NSEC_PER_SEC / 4) +#define FIVE_SEC (5LL * NSEC_PER_SEC) + +struct batch_data { +uint32_t batch[16 ]; +uint64_t pad; +uint32_t data; +uint32_t expected_data; +}; + +#define WRITE_VALUE (data__, i__) ({ \ + if (!(data__)->expected_data) \ + (data__)->expected_data = rand() << 12 | (i__); \ + (data__)->expected_data; \ +}) +#define READ_VALUE (data__) ((data__)->expected_data) + +static void __write_dword (uint32_t *batch,uint64_t sdi_addr,uint32_t wdata, +int *idx) +{ + batch[(*idx)++] =MI_STORE_DWORD_IMM_GEN4 ; + batch[(*idx)++] = sdi_addr; + batch[(*idx)++] = sdi_addr >>32 ; + batch[(*idx)++] = wdata; +} + +static void write_dword (uint32_t *batch,uint64_t sdi_addr,uint32_t wdata, +int *idx) +{ +__write_dword (batch, sdi_addr, wdata, idx); + batch[(*idx)++] =MI_BATCH_BUFFER_END ; +} + +static void check_all_pages (void *ptr,uint64_t alloc_size,uint64_t stride, +pthread_barrier_t *barrier) +{ +int i, n_writes = alloc_size / stride; + +for (i =0 ; i < n_writes; ++i) { +struct batch_data *data = ptr + i * stride; + +igt_assert_eq (data->data,READ_VALUE (data)); + +if (barrier) +pthread_barrier_wait (barrier); + } +} + +static char sync_file[] ="/tmp/xe_exec_system_allocator_syncXXXXXX" ; +static int sync_fd;
tests/intel/xe_exec_system_allocator.c line 88
Quoted 10 lines
tests/intel/xe_exec_system_allocator.c new file +10
-0+ +static void open_sync_file (void ) +{ + sync_fd =mkstemp (sync_file); +} + +static void close_sync_file (void ) +{ +close (sync_fd); +}
NIT:
The sync_fd doesn't seem like it's being used for anything. However, just because I don't know what this file does doesn't mean it does nothing, so I won't block on this.
Quoted 671 lines
tests/intel/xe_exec_system_allocator.c new file +671
-0+ +struct process_data { +pthread_mutex_t mutex; +pthread_cond_t cond; +pthread_barrier_t barrier; +bool go; +}; + +static void wait_pdata (struct process_data *pdata) +{ +pthread_mutex_lock (&pdata->mutex); +while (!pdata->go) +pthread_cond_wait (&pdata->cond, &pdata->mutex); +pthread_mutex_unlock (&pdata->mutex); +} + +static void init_pdata (struct process_data *pdata,int n_engine) +{ +pthread_mutexattr_t mutex_attr; +pthread_condattr_t cond_attr; +pthread_barrierattr_t barrier_attr; + +pthread_mutexattr_init (&mutex_attr); +pthread_mutexattr_setpshared (&mutex_attr,PTHREAD_PROCESS_SHARED ); +pthread_mutex_init (&pdata->mutex, &mutex_attr); + +pthread_condattr_init (&cond_attr); +pthread_condattr_setpshared (&cond_attr,PTHREAD_PROCESS_SHARED ); +pthread_cond_init (&pdata->cond, &cond_attr); + +pthread_barrierattr_init (&barrier_attr); +pthread_barrierattr_setpshared (&barrier_attr,PTHREAD_PROCESS_SHARED ); +pthread_barrier_init (&pdata->barrier, &barrier_attr, n_engine); + + pdata->go = false; +} + +static void signal_pdata (struct process_data *pdata) +{ +pthread_mutex_lock (&pdata->mutex); + pdata->go = true; +pthread_cond_broadcast (&pdata->cond); +pthread_mutex_unlock (&pdata->mutex); +} + +/* many_alloc flags */ +#define MIX_BO_ALLOC (0x1 << 0) +#define BENCHMARK (0x1 << 1) +#define CPU_FAULT_THREADS (0x1 << 2) +#define CPU_FAULT_PROCESS (0x1 << 3) +#define CPU_FAULT_SAME_PAGE (0x1 << 4) + +static void process_check (void *ptr,uint64_t alloc_size,uint64_t stride, +unsigned int flags) +{ +struct process_data *pdata; +int map_fd; + + map_fd =open (sync_file,O_RDWR ,0x666 ); + pdata =mmap (NULL ,sizeof (*pdata),PROT_READ | +PROT_WRITE ,MAP_SHARED , map_fd,0 ); +wait_pdata (pdata); + +if (flags &CPU_FAULT_SAME_PAGE ) +check_all_pages (ptr, alloc_size, stride, &pdata->barrier); +else +check_all_pages (ptr, alloc_size, stride,NULL ); + +close (map_fd); +munmap (pdata,sizeof (*pdata)); +} + +/* +* Partition checking of results in chunks which causes multiple processes to +* fault same VRAM allocation in parallel. +*/ +static void +check_all_pages_process (void *ptr,uint64_t alloc_size,uint64_t stride, +int n_process,unsigned int flags) +{ +struct process_data *pdata; +int map_fd, i; + + map_fd =open (sync_file,O_RDWR |O_CREAT ,0x666 ); +posix_fallocate (map_fd,0 ,sizeof (*pdata)); + pdata =mmap (NULL ,sizeof (*pdata),PROT_READ | +PROT_WRITE ,MAP_SHARED , map_fd,0 ); + +init_pdata (pdata, n_process); + +for (i =0 ; i < n_process; ++i) { + igt_fork(child, 1) +if (flags &CPU_FAULT_SAME_PAGE ) +process_check (ptr, alloc_size, stride, flags); +else +process_check (ptr + stride * i, alloc_size, + stride * n_process, flags); + } + +signal_pdata (pdata); +igt_waitchildren (); + +close (map_fd); +munmap (pdata,sizeof (*pdata)); +} + +struct thread_check_data { +pthread_t thread; +pthread_mutex_t *mutex; +pthread_cond_t *cond; +pthread_barrier_t *barrier; +void *ptr; +uint64_t alloc_size; +uint64_t stride; +bool *go; +}; + +static void *thread_check (void *data) +{ +struct thread_check_data *t = data; + +pthread_mutex_lock (t->mutex); +while (!*t->go) +pthread_cond_wait (t->cond, t->mutex); +pthread_mutex_unlock (t->mutex); + +check_all_pages (t->ptr, t->alloc_size, t->stride, t->barrier); + +return NULL ; +} + +/* +* Partition checking of results in chunks which causes multiple threads to +* fault same VRAM allocation in parallel. +*/ +static void +check_all_pages_threads (void *ptr,uint64_t alloc_size,uint64_t stride, +int n_threads,unsigned int flags) +{ +struct thread_check_data *threads_check_data; +pthread_mutex_t mutex; +pthread_cond_t cond; +pthread_barrier_t barrier; +int i; +bool go = false; + + threads_check_data =calloc (n_threads,sizeof (*threads_check_data)); +igt_assert (threads_check_data); + +pthread_mutex_init (&mutex,0 ); +pthread_cond_init (&cond,0 ); +pthread_barrier_init (&barrier,0 , n_threads); + +for (i =0 ; i < n_threads; ++i) { + threads_check_data[i].mutex = &mutex; + threads_check_data[i].cond = &cond; +if (flags &CPU_FAULT_SAME_PAGE ) { + threads_check_data[i].barrier = &barrier; + threads_check_data[i].ptr = ptr; + threads_check_data[i].alloc_size = alloc_size; + threads_check_data[i].stride = stride; + }else { + threads_check_data[i].barrier =NULL ; + threads_check_data[i].ptr = ptr + stride * i; + threads_check_data[i].alloc_size = alloc_size; + threads_check_data[i].stride = n_threads * stride; + } + threads_check_data[i].go = &go; + +pthread_create (&threads_check_data[i].thread,0 , thread_check, + &threads_check_data[i]); + } + +pthread_mutex_lock (&mutex); + go = true; +pthread_cond_broadcast (&cond); +pthread_mutex_unlock (&mutex); + +for (i =0 ; i < n_threads; ++i) +pthread_join (threads_check_data[i].thread,NULL ); +free (threads_check_data); +} + +static void touch_all_pages (int fd,uint32_t exec_queue,void *ptr, +uint64_t alloc_size,uint64_t stride, +struct timespec *tv,uint64_t *submit) +{ +struct drm_xe_sync sync[1 ] = { + { .type =DRM_XE_SYNC_TYPE_USER_FENCE , + .flags =DRM_XE_SYNC_FLAG_SIGNAL , + .timeline_value =USER_FENCE_VALUE }, + }; +struct drm_xe_exec exec = { + .num_batch_buffer =1 , + .num_syncs =0 , + .exec_queue_id = exec_queue, + .syncs =to_user_pointer (&sync), + }; +uint64_t addr =to_user_pointer (ptr); +int i, ret, n_writes = alloc_size / stride; +u64 *exec_ufence =NULL ; +int64_t timeout =FIVE_SEC ; + + exec_ufence =mmap (NULL ,SZ_4K ,PROT_READ | +PROT_WRITE ,MAP_SHARED | +MAP_ANONYMOUS ,-1 ,0 ); +igt_assert (exec_ufence !=MAP_FAILED ); +memset (exec_ufence,0 ,SZ_4K ); + sync[0 ].addr =to_user_pointer (exec_ufence); + +for (i =0 ; i < n_writes; ++i, addr += stride) { +struct batch_data *data = ptr + i * stride; +uint64_t sdi_offset = (char *)&data->data - (char *)data; +uint64_t sdi_addr = addr + sdi_offset; +int b =0 ; + +write_dword (data->batch, sdi_addr,WRITE_VALUE (data, i), &b); +igt_assert (b <=ARRAY_SIZE (data->batch)); + } + +igt_nsec_elapsed (tv); + *submit =igt_nsec_elapsed (tv); + + addr =to_user_pointer (ptr); +for (i =0 ; i < n_writes; ++i, addr += stride) { +struct batch_data *data = ptr + i * stride; +uint64_t batch_offset = (char *)&data->batch - (char *)data; +uint64_t batch_addr = addr + batch_offset; + + exec.address = batch_addr; +if (i +1 == n_writes) + exec.num_syncs =1 ; +xe_exec (fd, &exec); + } + + ret =__xe_wait_ufence (fd, exec_ufence,USER_FENCE_VALUE , exec_queue, + &timeout); +if (ret) { +igt_info ("FAIL EXEC_UFENCE_ADDR: 0x%016llx\n" , sync[0 ].addr); +igt_info ("FAIL EXEC_UFENCE: EXPECTED=0x%016llx, ACTUAL=0x%016lx\n" , +USER_FENCE_VALUE , exec_ufence[0 ]); + + addr =to_user_pointer (ptr); +for (i =0 ; i < n_writes; ++i, addr += stride) { +struct batch_data *data = ptr + i * stride; +uint64_t batch_offset = (char *)&data->batch - (char *)data; +uint64_t batch_addr = addr + batch_offset; +uint64_t sdi_offset = (char *)&data->data - (char *)data; +uint64_t sdi_addr = addr + sdi_offset; + +igt_info ("FAIL BATCH_ADDR: 0x%016lx\n" , batch_addr); +igt_info ("FAIL SDI_ADDR: 0x%016lx\n" , sdi_addr); +igt_info ("FAIL SDI_ADDR (in batch): 0x%016lx\n" , + (((u64 )data->batch[2 ]) <<32 ) | data->batch[1 ]); +igt_info ("FAIL DATA: EXPECTED=0x%08x, ACTUAL=0x%08x\n" , + data->expected_data, data->data); + } +igt_assert_eq (ret,0 ); + } +munmap (exec_ufence,SZ_4K ); +} + +static int va_bits; + +#define bind_system_allocator (__sync, __num_sync) \ + __xe_vm_bind_assert(fd, vm, 0, \ + 0, 0, 0, 0x1ull << va_bits, \ + DRM_XE_VM_BIND_OP_MAP, \ + DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR, \ + (__sync), (__num_sync), 0, 0) + +#define unbind_system_allocator () \ + __xe_vm_bind(fd, vm, 0, 0, 0, 0, 0x1ull << va_bits, \ + DRM_XE_VM_BIND_OP_UNMAP, 0, \ + NULL, 0, 0, 0, 0) + +#define odd (__i) (__i & 1) + +struct aligned_alloc_type { +void *__ptr; +void *ptr; +size_t __size; +size_t size; +}; + +static struct aligned_alloc_type __aligned_alloc (size_t alignment,size_t size) +{ +struct aligned_alloc_type aligned_alloc_type; + + aligned_alloc_type.__ptr =mmap (NULL , alignment + size,PROT_NONE ,MAP_PRIVATE | +MAP_ANONYMOUS ,-1 ,0 ); +igt_assert (aligned_alloc_type.__ptr !=MAP_FAILED ); + + aligned_alloc_type.ptr = (void *)ALIGN ((uint64_t )aligned_alloc_type.__ptr, alignment); + aligned_alloc_type.size = size; + aligned_alloc_type.__size = size + alignment; + +return aligned_alloc_type; +} + +static void __aligned_free (struct aligned_alloc_type *aligned_alloc_type) +{ +munmap (aligned_alloc_type->__ptr, aligned_alloc_type->__size); +} + +static void __aligned_partial_free (struct aligned_alloc_type *aligned_alloc_type) +{ +size_t begin_size = (size_t )(aligned_alloc_type->ptr - aligned_alloc_type->__ptr); + +if (begin_size) +munmap (aligned_alloc_type->__ptr, begin_size); +if (aligned_alloc_type->__size - aligned_alloc_type->size - begin_size) +munmap (aligned_alloc_type->ptr + aligned_alloc_type->size, + aligned_alloc_type->__size - aligned_alloc_type->size - begin_size); +} + +/** +* SUBTEST: unaligned-alloc +* Description: allocate unaligned sizes of memory +* Test category: functionality test +* +* SUBTEST: fault-benchmark +* Description: Benchmark how long GPU / CPU take +* Test category: performance test +* +* SUBTEST: fault-threads-benchmark +* Description: Benchmark how long GPU / CPU take, reading results with multiple threads +* Test category: performance and functionality test +* +* SUBTEST: fault-threads-same-page-benchmark +* Description: Benchmark how long GPU / CPU take, reading results with multiple threads, hammer same page +* Test category: performance and functionality test +* +* SUBTEST: fault-process-benchmark +* Description: Benchmark how long GPU / CPU take, reading results with multiple process +* Test category: performance and functionality test +* +* SUBTEST: fault-process-same-page-benchmark +* Description: Benchmark how long GPU / CPU take, reading results with multiple process, hammer same page +* Test category: performance and functionality test +* +* SUBTEST: evict-malloc +* Description: trigger eviction of VRAM allocated via malloc +* Test category: functionality test +* +* SUBTEST: evict-malloc-mix-bo +* Description: trigger eviction of VRAM allocated via malloc and BO create +* Test category: functionality test +* +* SUBTEST: processes-evict-malloc +* Description: multi-process trigger eviction of VRAM allocated via malloc +* Test category: stress test +* +* SUBTEST: processes-evict-malloc-mix-bo +* Description: multi-process trigger eviction of VRAM allocated via malloc and BO create +* Test category: stress test +*/ + +static void +many_allocs (int fd,struct drm_xe_engine_class_instance *eci, +uint64_t total_alloc,uint64_t alloc_size,uint64_t stride, +pthread_barrier_t *barrier,unsigned int flags) +{ +uint32_t vm, exec_queue; +int num_allocs = flags &BENCHMARK ?1 : + (9 * (total_alloc / alloc_size)) /8 ; +struct aligned_alloc_type *allocs; +uint32_t *bos =NULL ; +struct timespec tv = {}; +uint64_t submit, read, elapsed; +int i; + + vm =xe_vm_create (fd,DRM_XE_VM_CREATE_FLAG_LR_MODE | +DRM_XE_VM_CREATE_FLAG_FAULT_MODE ,0 ); + exec_queue =xe_exec_queue_create (fd, vm, eci,0 ); + +bind_system_allocator (NULL ,0 ); + + allocs =malloc (sizeof (*allocs) * num_allocs); +igt_assert (allocs); +memset (allocs,0 ,sizeof (*allocs) * num_allocs); + +if (flags &MIX_BO_ALLOC ) { + bos =malloc (sizeof (*bos) * num_allocs); +igt_assert (bos); +memset (bos,0 ,sizeof (*bos) * num_allocs); + } + +for (i =0 ; i < num_allocs; ++i) { +struct aligned_alloc_type alloc; + +if (flags &MIX_BO_ALLOC &&odd (i)) { +uint32_t bo_flags = +DRM_XE_GEM_CREATE_FLAG_NEEDS_VISIBLE_VRAM ; + + alloc =__aligned_alloc (SZ_2M , alloc_size); +igt_assert (alloc.ptr); + + bos[i] =xe_bo_create (fd, vm, alloc_size, +vram_if_possible (fd, eci->gt_id), + bo_flags); + alloc.ptr =xe_bo_map_fixed (fd, bos[i], alloc_size, +to_user_pointer (alloc.ptr)); +xe_vm_bind_async (fd, vm,0 , bos[i],0 , +to_user_pointer (alloc.ptr), + alloc_size,0 ,0 ); + }else { + alloc.ptr =aligned_alloc (SZ_2M , alloc_size); +igt_assert (alloc.ptr); + } + allocs[i] = alloc; + +touch_all_pages (fd, exec_queue, allocs[i].ptr, alloc_size, stride, + &tv, &submit); + } + +if (barrier) +pthread_barrier_wait (barrier); + +for (i =0 ; i < num_allocs; ++i) { +if (flags &BENCHMARK ) + read =igt_nsec_elapsed (&tv); +#define NUM_CHECK_THREADS 8 +if (flags &CPU_FAULT_PROCESS ) +check_all_pages_process (allocs[i].ptr, alloc_size, stride, +NUM_CHECK_THREADS , flags); +else if (flags &CPU_FAULT_THREADS ) +check_all_pages_threads (allocs[i].ptr, alloc_size, stride, +NUM_CHECK_THREADS , flags); +else +check_all_pages (allocs[i].ptr, alloc_size, stride,NULL ); +if (flags &BENCHMARK ) { + elapsed =igt_nsec_elapsed (&tv); +igt_info ("Execution took %.3fms (submit %.1fus, read %.1fus, total %.1fus, read_total %.1fus)\n" , +1e-6 * elapsed,1e-3 * submit,1e-3 * read, +1e-3 * (elapsed - submit), +1e-3 * (elapsed - read)); + } +if (bos && bos[i]) { +__aligned_free (allocs + i); +gem_close (fd, bos[i]); + }else { +free (allocs[i].ptr); + } + } +if (bos) +free (bos); +free (allocs); +xe_exec_queue_destroy (fd, exec_queue); +xe_vm_destroy (fd, vm); +} + +static void process_evict (struct drm_xe_engine_class_instance *hwe, +uint64_t total_alloc,uint64_t alloc_size, +uint64_t stride,unsigned int flags) +{ +struct process_data *pdata; +int map_fd; +int fd; + + map_fd =open (sync_file,O_RDWR ,0x666 ); + pdata =mmap (NULL ,sizeof (*pdata),PROT_READ | +PROT_WRITE ,MAP_SHARED , map_fd,0 ); +wait_pdata (pdata); + + fd =drm_open_driver (DRIVER_XE ); +many_allocs (fd, hwe, total_alloc, alloc_size, stride, &pdata->barrier, + flags); +drm_close_driver (fd); + +close (map_fd); +munmap (pdata,sizeof (*pdata)); +} + +static void +processes_evict (int fd,uint64_t alloc_size,uint64_t stride, +unsigned int flags) +{ +struct drm_xe_engine_class_instance *hwe; +struct process_data *pdata; +int n_engine_gt[2 ] = {0 ,0 }, n_engine =0 ; +int map_fd; + + map_fd =open (sync_file,O_RDWR |O_CREAT ,0x666 ); +posix_fallocate (map_fd,0 ,sizeof (*pdata)); + pdata =mmap (NULL ,sizeof (*pdata),PROT_READ | +PROT_WRITE ,MAP_SHARED , map_fd,0 ); + +xe_for_each_engine (fd, hwe) { +igt_assert (hwe->gt_id <2 ); + n_engine_gt[hwe->gt_id]++; + n_engine++; + } + +init_pdata (pdata, n_engine); + +xe_for_each_engine (fd, hwe) { + igt_fork(child, 1) +process_evict (hwe, +xe_visible_vram_size (fd, hwe->gt_id) / + n_engine_gt[hwe->gt_id], alloc_size, + stride, flags); + } + +signal_pdata (pdata); +igt_waitchildren (); + +close (map_fd); +munmap (pdata,sizeof (*pdata)); +} + +#define CPU_FAULT (0x1 << 0) +#define REMAP (0x1 << 1) +#define MIDDLE (0x1 << 2) + +/** +* SUBTEST: partial-munmap-cpu-fault +* Description: munmap partially with cpu access in between +* Test category: functionality test +* +* SUBTEST: partial-munmap-no-cpu-fault +* Description: munmap partially with no cpu access in between +* Test category: functionality test +* +* SUBTEST: partial-remap-cpu-fault +* Description: remap partially with cpu access in between +* Test category: functionality test +* +* SUBTEST: partial-remap-no-cpu-fault +* Description: remap partially with no cpu access in between +* Test category: functionality test +* +* SUBTEST: partial-middle-munmap-cpu-fault +* Description: munmap middle with cpu access in between +* Test category: functionality test +* +* SUBTEST: partial-middle-munmap-no-cpu-fault +* Description: munmap middle with no cpu access in between +* Test category: functionality test +* +* SUBTEST: partial-middle-remap-cpu-fault +* Description: remap middle with cpu access in between +* Test category: functionality test +* +* SUBTEST: partial-middle-remap-no-cpu-fault +* Description: remap middle with no cpu access in between +* Test category: functionality test +*/ + +static void +partial (int fd,struct drm_xe_engine_class_instance *eci,unsigned int flags) +{ +struct drm_xe_sync sync[1 ] = { + { .type =DRM_XE_SYNC_TYPE_USER_FENCE , .flags =DRM_XE_SYNC_FLAG_SIGNAL , + .timeline_value =USER_FENCE_VALUE }, + }; +struct drm_xe_exec exec = { + .num_batch_buffer =1 , + .num_syncs =1 , + .syncs =to_user_pointer (sync), + }; +struct { +uint32_t batch[16 ]; +uint64_t pad; +uint64_t vm_sync; +uint64_t exec_sync; +uint32_t data; +uint32_t expected_data; + } *data; +size_t bo_size =SZ_2M , unmap_offset =0 ; +uint32_t vm, exec_queue; +u64 *exec_ufence =NULL ; +int i; +void *old, *new =NULL ; +struct aligned_alloc_type alloc; + +if (flags &MIDDLE ) + unmap_offset = bo_size /4 ; + + vm =xe_vm_create (fd,DRM_XE_VM_CREATE_FLAG_LR_MODE | +DRM_XE_VM_CREATE_FLAG_FAULT_MODE ,0 ); + + alloc =__aligned_alloc (bo_size, bo_size); +igt_assert (alloc.ptr); + + data =mmap (alloc.ptr, bo_size,PROT_READ |PROT_WRITE , +MAP_PRIVATE |MAP_ANONYMOUS |MAP_FIXED ,-1 ,0 ); +igt_assert (data !=MAP_FAILED ); +memset (data,0 , bo_size); + old = data; + + exec_queue =xe_exec_queue_create (fd, vm, eci,0 ); + + sync[0 ].addr =to_user_pointer (&data[0 ].vm_sync); +bind_system_allocator (sync,1 ); +xe_wait_ufence (fd, &data[0 ].vm_sync,USER_FENCE_VALUE ,0 ,FIVE_SEC ); + data[0 ].vm_sync =0 ; + + exec_ufence =mmap (NULL ,SZ_4K ,PROT_READ | +PROT_WRITE ,MAP_SHARED | +MAP_ANONYMOUS ,-1 ,0 ); +igt_assert (exec_ufence !=MAP_FAILED ); +memset (exec_ufence,0 ,SZ_4K ); + +for (i =0 ; i <2 ; i++) { +uint64_t addr =to_user_pointer (data); +uint64_t sdi_offset = (char *)&data[i].data - (char *)data; +uint64_t sdi_addr = addr + sdi_offset; +int b =0 ; + +write_dword (data[i].batch, sdi_addr,WRITE_VALUE (&data[i], i), &b); +igt_assert (b <=ARRAY_SIZE (data[i].batch)); + +if (!i) + data = old + unmap_offset + bo_size /2 ; + } + + data = old; + exec.exec_queue_id = exec_queue; + +for (i =0 ; i <2 ; i++) { +uint64_t addr =to_user_pointer (data); +uint64_t batch_offset = (char *)&data[i].batch - (char *)data; +uint64_t batch_addr = addr + batch_offset; + + sync[0 ].addr = new ?to_user_pointer (new) : +to_user_pointer (exec_ufence); + exec.address = batch_addr; +xe_exec (fd, &exec); + +xe_wait_ufence (fd, new ?: exec_ufence,USER_FENCE_VALUE , + exec_queue,FIVE_SEC ); +if (i || (flags &CPU_FAULT )) +igt_assert_eq (data[i].data,READ_VALUE (&data[i])); + exec_ufence[0 ] =0 ; + +if (!i) { + data = old + unmap_offset + bo_size /2 ; +munmap (old + unmap_offset, bo_size /2 ); +if (flags &REMAP ) { + new =mmap (old + unmap_offset, bo_size /2 , +PROT_READ |PROT_WRITE , +MAP_PRIVATE |MAP_ANONYMOUS |MAP_FIXED | +MAP_LOCKED ,-1 ,0 ); +igt_assert (new !=MAP_FAILED ); + } + } + } + +xe_exec_queue_destroy (fd, exec_queue); +munmap (exec_ufence,SZ_4K ); +__aligned_free (&alloc); +if (new) +munmap (new, bo_size /2 ); +xe_vm_destroy (fd, vm); +} + +#define MAX_N_EXEC_QUEUES 16 + +#define MMAP (0x1 << 0) +#define NEW (0x1 << 1) +#define BO_UNMAP (0x1 << 2) +#define FREE (0x1 << 3) +#define BUSY (0x1 << 4) +#define BO_MAP (0x1 << 5) +#define RACE (0x1 << 6) +#define SKIP_MEMSET (0x1 << 7) +#define FAULT (0x1 << 8) +#define FILE_BACKED (0x1 << 9) +#define LOCK (0x1 << 10) +#define MMAP_SHARED (0x1 << 11)
tests/intel/xe_exec_system_allocator.c line 769
Quoted 10 lines
tests/intel/xe_exec_system_allocator.c new file +10
-0+#define HUGE_PAGE (0x1 << 12) +#define SHARED_ALLOC (0x1 << 13) +#define FORK_READ (0x1 << 14) +#define FORK_READ_AFTER (0x1 << 15) +#define MREMAP (0x1 << 16) +#define DONTUNMAP (0x1 << 17) +#define READ_ONLY_REMAP (0x1 << 18) +#define SYNC_EXEC (0x1 << 19) +#define EVERY_OTHER_CHECK (0x1 << 20) +#define MULTI_FAULT (0x1 << 21)
NIT:
The above doesn't look aligned, but I'm willing to bet that has to do with my email client and that it's aligned in code.
However, there is a notable standout with EVERY_OTHER_CHECK. IMO, I think all of the values should be aligned to that if they aren't already.
-Jonathan Cavitt
Quoted 1,096 lines
tests/intel/xe_exec_system_allocator.c new file +1080
-0+ +#define N_MULTI_FAULT 4 + +/** +* SUBTEST: once-%s +* Description: Run %arg[1] system allocator test only once +* Test category: functionality test +* +* SUBTEST: once-large-%s +* Description: Run %arg[1] system allocator test only once with large allocation +* Test category: functionality test +* +* SUBTEST: twice-%s +* Description: Run %arg[1] system allocator test twice +* Test category: functionality test +* +* SUBTEST: twice-large-%s +* Description: Run %arg[1] system allocator test twice with large allocation +* Test category: functionality test +* +* SUBTEST: many-%s +* Description: Run %arg[1] system allocator test many times +* Test category: stress test +* +* SUBTEST: many-stride-%s +* Description: Run %arg[1] system allocator test many times with a stride on each exec +* Test category: stress test +* +* SUBTEST: many-execqueues-%s +* Description: Run %arg[1] system allocator test on many exec_queues +* Test category: stress test +* +* SUBTEST: many-large-%s +* Description: Run %arg[1] system allocator test many times with large allocations +* Test category: stress test +* +* SUBTEST: many-large-execqueues-%s +* Description: Run %arg[1] system allocator test on many exec_queues with large allocations +* +* SUBTEST: threads-many-%s +* Description: Run %arg[1] system allocator threaded test many times +* Test category: stress test +* +* SUBTEST: threads-many-stride-%s +* Description: Run %arg[1] system allocator threaded test many times with a stride on each exec +* Test category: stress test +* +* SUBTEST: threads-many-execqueues-%s +* Description: Run %arg[1] system allocator threaded test on many exec_queues +* Test category: stress test +* +* SUBTEST: threads-many-large-%s +* Description: Run %arg[1] system allocator threaded test many times with large allocations +* Test category: stress test +* +* SUBTEST: threads-many-large-execqueues-%s +* Description: Run %arg[1] system allocator threaded test on many exec_queues with large allocations +* +* SUBTEST: threads-shared-vm-many-%s +* Description: Run %arg[1] system allocator threaded, shared vm test many times +* Test category: stress test +* +* SUBTEST: threads-shared-vm-many-stride-%s +* Description: Run %arg[1] system allocator threaded, shared vm test many times with a stride on each exec +* Test category: stress test +* +* SUBTEST: threads-shared-vm-many-execqueues-%s +* Description: Run %arg[1] system allocator threaded, shared vm test on many exec_queues +* Test category: stress test +* +* SUBTEST: threads-shared-vm-many-large-%s +* Description: Run %arg[1] system allocator threaded, shared vm test many times with large allocations +* Test category: stress test +* +* SUBTEST: threads-shared-vm-many-large-execqueues-%s +* Description: Run %arg[1] system allocator threaded, shared vm test on many exec_queues with large allocations +* Test category: stress test +* +* SUBTEST: process-many-%s +* Description: Run %arg[1] system allocator multi-process test many times +* Test category: stress test +* +* SUBTEST: process-many-stride-%s +* Description: Run %arg[1] system allocator multi-process test many times with a stride on each exec +* Test category: stress test +* +* SUBTEST: process-many-execqueues-%s +* Description: Run %arg[1] system allocator multi-process test on many exec_queues +* Test category: stress test +* +* SUBTEST: process-many-large-%s +* Description: Run %arg[1] system allocator multi-process test many times with large allocations +* Test category: stress test +* +* SUBTEST: process-many-large-execqueues-%s +* Description: Run %arg[1] system allocator multi-process test on many exec_queues with large allocations +* +* SUBTEST: fault +* Description: use a bad system allocator address resulting in a fault +* Test category: bad input +* +* arg[1]: +* +* @malloc: malloc single buffer for all execs, issue a command which will trigger multiple faults +* @malloc-multi-fault: malloc single buffer for all execs +* @malloc-fork-read: malloc single buffer for all execs, fork a process to read test output +* @malloc-fork-read-after: malloc single buffer for all execs, fork a process to read test output, check again after fork returns in parent +* @malloc-mlock: malloc and mlock single buffer for all execs +* @malloc-race: malloc single buffer for all execs with race between cpu and gpu access +* @malloc-bo-unmap: malloc single buffer for all execs, bind and unbind a BO to same address before execs +* @malloc-busy: malloc single buffer for all execs, try to unbind while buffer valid +* @mmap: mmap single buffer for all execs +* @mmap-remap: mmap and mremap a buffer for all execs +* @mmap-remap-dontunmap: mmap and mremap a buffer with dontunmap flag for all execs +* @mmap-remap-ro: mmap and mremap a read-only buffer for all execs +* @mmap-remap-ro-dontunmap: mmap and mremap a read-only buffer with dontunmap flag for all execs +* @mmap-remap-eocheck: mmap and mremap a buffer for all execs, check data every other loop iteration +* @mmap-remap-dontunmap-eocheck: mmap and mremap a buffer with dontunmap flag for all execs, check data every other loop iteration +* @mmap-remap-ro-eocheck: mmap and mremap a read-only buffer for all execs, check data every other loop iteration +* @mmap-remap-ro-dontunmap-eocheck: mmap and mremap a read-only buffer with dontunmap flag for all execs, check data every other loop iteration +* @mmap-huge: mmap huge page single buffer for all execs +* @mmap-shared: mmap shared single buffer for all execs +* @mmap-shared-remap: mmap shared and mremap a buffer for all execs +* @mmap-shared-remap-dontunmap: mmap shared and mremap a buffer with dontunmap flag for all execs +* @mmap-shared-remap-eocheck: mmap shared and mremap a buffer for all execs, check data every other loop iteration +* @mmap-shared-remap-dontunmap-eocheck: mmap shared and mremap a buffer with dontunmap flag for all execs, check data every other loop iteration +* @mmap-mlock: mmap and mlock single buffer for all execs +* @mmap-file: mmap single buffer, with file backing, for all execs +* @mmap-file-mlock: mmap and mlock single buffer, with file backing, for all execs +* @mmap-race: mmap single buffer for all execs with race between cpu and gpu access +* @free: malloc and free buffer for each exec +* @free-race: malloc and free buffer for each exec with race between cpu and gpu access +* @new: malloc a new buffer for each exec +* @new-race: malloc a new buffer for each exec with race between cpu and gpu access +* @new-bo-map: malloc a new buffer or map BO for each exec +* @new-busy: malloc a new buffer for each exec, try to unbind while buffers valid +* @mmap-free: mmap and free buffer for each exec +* @mmap-free-huge: mmap huge page and free buffer for each exec +* @mmap-free-race: mmap and free buffer for each exec with race between cpu and gpu access +* @mmap-new: mmap a new buffer for each exec +* @mmap-new-huge: mmap huge page a new buffer for each exec +* @mmap-new-race: mmap a new buffer for each exec with race between cpu and gpu access +* @malloc-nomemset: malloc single buffer for all execs, skip memset of buffers +* @malloc-mlock-nomemset: malloc and mlock single buffer for all execs, skip memset of buffers +* @malloc-race-nomemset: malloc single buffer for all execs with race between cpu and gpu access, skip memset of buffers +* @malloc-bo-unmap-nomemset: malloc single buffer for all execs, bind and unbind a BO to same address before execs, skip memset of buffers +* @malloc-busy-nomemset: malloc single buffer for all execs, try to unbind while buffer valid, skip memset of buffers +* @mmap-nomemset: mmap single buffer for all execs, skip memset of buffers +* @mmap-huge-nomemset: mmap huge page single buffer for all execs, skip memset of buffers +* @mmap-shared-nomemset: mmap shared single buffer for all execs, skip memset of buffers +* @mmap-mlock-nomemset: mmap and mlock single buffer for all execs, skip memset of buffers +* @mmap-file-nomemset: mmap single buffer, with file backing, for all execs, skip memset of buffers +* @mmap-file-mlock-nomemset: mmap and mlock single buffer, with file backing, for all execs, skip memset of buffers +* @mmap-race-nomemset: mmap single buffer for all execs with race between cpu and gpu access, skip memset of buffers +* @free-nomemset: malloc and free buffer for each exec, skip memset of buffers +* @free-race-nomemset: malloc and free buffer for each exec with race between cpu and gpu access, skip memset of buffers +* @new-nomemset: malloc a new buffer for each exec, skip memset of buffers +* @new-race-nomemset: malloc a new buffer for each exec with race between cpu and gpu access, skip memset of buffers +* @new-bo-map-nomemset: malloc a new buffer or map BO for each exec, skip memset of buffers +* @new-busy-nomemset: malloc a new buffer for each exec, try to unbind while buffers valid, skip memset of buffers +* @mmap-free-nomemset: mmap and free buffer for each exec, skip memset of buffers +* @mmap-free-huge-nomemset: mmap huge page and free buffer for each exec, skip memset of buffers +* @mmap-free-race-nomemset: mmap and free buffer for each exec with race between cpu and gpu access, skip memset of buffers +* @mmap-new-nomemset: mmap a new buffer for each exec, skip memset of buffers +* @mmap-new-huge-nomemset: mmap huge page new buffer for each exec, skip memset of buffers +* @mmap-new-race-nomemset: mmap a new buffer for each exec with race between cpu and gpu access, skip memset of buffers +* +* SUBTEST: threads-shared-vm-shared-alloc-many-stride-malloc +* Description: Create multiple threads with a shared VM triggering faults on different hardware engines to same addresses +* Test category: stress test +* +* SUBTEST: threads-shared-vm-shared-alloc-many-stride-malloc-race +* Description: Create multiple threads with a shared VM triggering faults on different hardware engines to same addresses, racing between CPU and GPU access +* Test category: stress test +* +* SUBTEST: threads-shared-alloc-many-stride-malloc +* Description: Create multiple threads with a faults on different hardware engines to same addresses +* Test category: stress test +* +* SUBTEST: threads-shared-alloc-many-stride-malloc-sync +* Description: Create multiple threads with a faults on different hardware engines to same addresses, syncing on each exec +* Test category: stress test +* +* SUBTEST: threads-shared-alloc-many-stride-malloc-race +* Description: Create multiple threads with a faults on different hardware engines to same addresses, racing between CPU and GPU access +* Test category: stress test +*/ + +struct test_exec_data { +uint32_t batch[32 ]; +uint64_t pad; +uint64_t vm_sync; +uint64_t exec_sync; +uint32_t data; +uint32_t expected_data; +}; + +static void +test_exec (int fd,struct drm_xe_engine_class_instance *eci, +int n_exec_queues,int n_execs,size_t bo_size, +size_t stride,uint32_t vm,void *alloc,pthread_barrier_t *barrier, +unsigned int flags) +{ +uint64_t addr; +struct drm_xe_sync sync[1 ] = { + { .type =DRM_XE_SYNC_TYPE_USER_FENCE , .flags =DRM_XE_SYNC_FLAG_SIGNAL , + .timeline_value =USER_FENCE_VALUE }, + }; +struct drm_xe_exec exec = { + .num_batch_buffer =1 , + .num_syncs =1 , + .syncs =to_user_pointer (sync), + }; +uint32_t exec_queues[MAX_N_EXEC_QUEUES ]; +struct test_exec_data *data, *next_data =NULL ; +uint32_t bo_flags; +uint32_t bo =0 ; +void **pending_free; +u64 *exec_ufence =NULL ; +int i, j, b, file_fd =-1 , prev_idx; +bool free_vm = false; +size_t aligned_size = bo_size ?:xe_get_default_alignment (fd); +size_t orig_size = bo_size; +struct aligned_alloc_type aligned_alloc_type; + +if (flags &MULTI_FAULT ) { +if (!bo_size) +return ; + + bo_size *=N_MULTI_FAULT ; + } + +if (flags &SHARED_ALLOC ) +return ; + +if (flags &EVERY_OTHER_CHECK &&odd (n_execs)) +return ; + +if (flags &EVERY_OTHER_CHECK ) +igt_assert (flags &MREMAP ); + +igt_assert (n_exec_queues <=MAX_N_EXEC_QUEUES ); + +if (flags &NEW && !(flags &FREE )) { + pending_free =malloc (sizeof (*pending_free) * n_execs); +igt_assert (pending_free); +memset (pending_free,0 ,sizeof (*pending_free) * n_execs); + } + +if (!vm) { + vm =xe_vm_create (fd,DRM_XE_VM_CREATE_FLAG_LR_MODE | +DRM_XE_VM_CREATE_FLAG_FAULT_MODE ,0 ); + free_vm = true; + } +if (!bo_size) { +if (!stride) { + bo_size =sizeof (*data) * n_execs; + bo_size =xe_bb_size (fd, bo_size); + }else { + bo_size = stride * n_execs *sizeof (*data); + bo_size =xe_bb_size (fd, bo_size); + } + } +if (flags &HUGE_PAGE ) { + aligned_size =ALIGN (aligned_size,SZ_2M ); + bo_size =ALIGN (bo_size,SZ_2M ); + } + +if (alloc) { + data = alloc; + }else { +if (flags &MMAP ) { +int mmap_flags =MAP_FIXED ; + + aligned_alloc_type =__aligned_alloc (aligned_size, bo_size); + data = aligned_alloc_type.ptr; +igt_assert (data); +__aligned_partial_free (&aligned_alloc_type); + +if (flags &MMAP_SHARED ) + mmap_flags |=MAP_SHARED ; +else + mmap_flags |=MAP_PRIVATE ; + +if (flags &HUGE_PAGE ) + mmap_flags |=MAP_HUGETLB |MAP_HUGE_2MB ; + +if (flags &FILE_BACKED ) { +char name[] ="/tmp/xe_exec_system_allocator_datXXXXXX" ; + +igt_assert (!(flags &NEW )); + + file_fd =mkstemp (name); +posix_fallocate (file_fd,0 , bo_size); + }else { + mmap_flags |=MAP_ANONYMOUS ; + } + + data =mmap (data, bo_size,PROT_READ | +PROT_WRITE , mmap_flags, file_fd,0 ); +igt_assert (data !=MAP_FAILED ); + }else { + data =aligned_alloc (aligned_size, bo_size); +igt_assert (data); + } +if (!(flags &SKIP_MEMSET )) +memset (data,0 , bo_size); +if (flags &LOCK ) { +igt_assert (!(flags &NEW )); +mlock (data, bo_size); + } + } + +for (i =0 ; i < n_exec_queues; i++) + exec_queues[i] =xe_exec_queue_create (fd, vm, eci,0 ); + + sync[0 ].addr =to_user_pointer (&data[0 ].vm_sync); +if (free_vm) { +bind_system_allocator (sync,1 ); +xe_wait_ufence (fd, &data[0 ].vm_sync,USER_FENCE_VALUE ,0 ,FIVE_SEC ); + } + data[0 ].vm_sync =0 ; + + addr =to_user_pointer (data); + +if (flags &BO_UNMAP ) { + bo_flags =DRM_XE_GEM_CREATE_FLAG_NEEDS_VISIBLE_VRAM ; + bo =xe_bo_create (fd, vm, bo_size, +vram_if_possible (fd, eci->gt_id), bo_flags); +xe_vm_bind_async (fd, vm,0 , bo,0 , addr, bo_size,0 ,0 ); + +__xe_vm_bind_assert (fd, vm,0 , +0 ,0 , addr, bo_size, +DRM_XE_VM_BIND_OP_MAP , +DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR , sync, +1 ,0 ,0 ); +xe_wait_ufence (fd, &data[0 ].vm_sync,USER_FENCE_VALUE ,0 , +FIVE_SEC ); + data[0 ].vm_sync =0 ; +gem_close (fd, bo); + bo =0 ; + } + +if (!(flags &RACE )) { + exec_ufence =mmap (NULL ,SZ_4K ,PROT_READ | +PROT_WRITE ,MAP_SHARED | +MAP_ANONYMOUS ,-1 ,0 ); +igt_assert (exec_ufence !=MAP_FAILED ); +memset (exec_ufence,0 ,SZ_4K ); + } + +for (i =0 ; i < n_execs; i++) { +int idx = !stride ? i : i * stride, next_idx = !stride + ? (i +1 ) : (i +1 ) * stride; +uint64_t batch_offset = (char *)&data[idx].batch - (char *)data; +uint64_t batch_addr = addr + batch_offset; +uint64_t sdi_offset = (char *)&data[idx].data - (char *)data; +uint64_t sdi_addr = addr + sdi_offset; +int e = i % n_exec_queues, err; +bool fault_inject = (FAULT & flags) && i == n_execs /2 ; +bool fault_injected = (FAULT & flags) && i > n_execs; + +if (barrier) +pthread_barrier_wait (barrier); + +if (flags &MULTI_FAULT ) { + b =0 ; +for (j =0 ; j <N_MULTI_FAULT -1 ; ++j) +__write_dword (data[idx].batch, + sdi_addr + j * orig_size, +WRITE_VALUE (&data[idx], idx), &b); +write_dword (data[idx].batch, sdi_addr + j * orig_size, +WRITE_VALUE (&data[idx], idx), &b); +igt_assert (b <=ARRAY_SIZE (data[idx].batch)); + }else if (!(flags &EVERY_OTHER_CHECK )) { + b =0 ; +write_dword (data[idx].batch, sdi_addr, +WRITE_VALUE (&data[idx], idx), &b); +igt_assert (b <=ARRAY_SIZE (data[idx].batch)); + }else if (flags &EVERY_OTHER_CHECK && !odd (i)) { + b =0 ; +write_dword (data[idx].batch, sdi_addr, +WRITE_VALUE (&data[idx], idx), &b); +igt_assert (b <=ARRAY_SIZE (data[idx].batch)); + + aligned_alloc_type =__aligned_alloc (aligned_size, bo_size); + next_data = aligned_alloc_type.ptr; +igt_assert (next_data); +__aligned_partial_free (&aligned_alloc_type); + + b =0 ; +write_dword (data[next_idx].batch, +to_user_pointer (next_data) + + (char *)&data[next_idx].data - (char *)data, +WRITE_VALUE (&data[next_idx], next_idx), &b); +igt_assert (b <=ARRAY_SIZE (data[next_idx].batch)); + } + +if (!exec_ufence) + data[idx].exec_sync =0 ; + + sync[0 ].addr = exec_ufence ?to_user_pointer (exec_ufence) : + addr + (char *)&data[idx].exec_sync - (char *)data; + + exec.exec_queue_id = exec_queues[e]; +if (fault_inject) + exec.address = batch_addr *2 ; +else + exec.address = batch_addr; + +if (fault_injected) { + err =__xe_exec (fd, &exec); +igt_assert (err == -ENOENT ); + }else { +xe_exec (fd, &exec); + } + +if (barrier) +pthread_barrier_wait (barrier); + +if (fault_inject || fault_injected) { +int64_t timeout =QUARTER_SEC ; + + err =__xe_wait_ufence (fd, exec_ufence ? exec_ufence : + &data[idx].exec_sync, +USER_FENCE_VALUE , + exec_queues[e], &timeout); +igt_assert (err == -ETIME || err == -EIO ); + }else { +xe_wait_ufence (fd, exec_ufence ? exec_ufence : + &data[idx].exec_sync,USER_FENCE_VALUE , + exec_queues[e],FIVE_SEC ); +if (flags &LOCK && !i) +munlock (data, bo_size); + +if (flags &MREMAP ) { +void *old = data; +int remap_flags =MREMAP_MAYMOVE |MREMAP_FIXED ; + +if (flags &DONTUNMAP ) + remap_flags |=MREMAP_DONTUNMAP ; + +if (flags &READ_ONLY_REMAP ) +igt_assert (!mprotect (old, bo_size, +PROT_READ )); + +if (!next_data) { + aligned_alloc_type =__aligned_alloc (aligned_size, + bo_size); + data = aligned_alloc_type.ptr; +__aligned_partial_free (&aligned_alloc_type); + }else { + data = next_data; + } + next_data =NULL ; +igt_assert (data); + + data =mremap (old, bo_size, bo_size, + remap_flags, data); +igt_assert (data !=MAP_FAILED ); + +if (flags &READ_ONLY_REMAP ) +igt_assert (!mprotect (data, bo_size, +PROT_READ | +PROT_WRITE )); + + addr =to_user_pointer (data); +if (flags &DONTUNMAP ) +munmap (old, bo_size); + } + +if (!(flags &EVERY_OTHER_CHECK ) ||odd (i)) { +if (flags &FORK_READ ) { + igt_fork(child, 1) +igt_assert_eq (data[idx].data, +READ_VALUE (&data[idx])); +if (!(flags &FORK_READ_AFTER )) +igt_assert_eq (data[idx].data, +READ_VALUE (&data[idx])); +igt_waitchildren (); +if (flags &FORK_READ_AFTER ) +igt_assert_eq (data[idx].data, +READ_VALUE (&data[idx])); + }else { +igt_assert_eq (data[idx].data, +READ_VALUE (&data[idx])); + +if (flags &MULTI_FAULT ) { +for (j =1 ; j <N_MULTI_FAULT ; ++j) { +struct test_exec_data *__data = + ((void *)data) + j * orig_size; + +igt_assert_eq (__data[idx].data, +READ_VALUE (&data[idx])); + } + } + } +if (flags &EVERY_OTHER_CHECK ) +igt_assert_eq (data[prev_idx].data, +READ_VALUE (&data[prev_idx])); + } + } + +if (exec_ufence) + exec_ufence[0 ] =0 ; + +if (bo) { +__xe_vm_bind_assert (fd, vm,0 , +0 ,0 , addr, bo_size, +DRM_XE_VM_BIND_OP_MAP , +DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR , +NULL ,0 ,0 ,0 ); +munmap (data, bo_size); +gem_close (fd, bo); + } + +if (flags &NEW ) { +if (flags &MMAP ) { +if (flags &FREE ) +munmap (data, bo_size); +else + pending_free[i] = data; + data =mmap (NULL , bo_size,PROT_READ | +PROT_WRITE ,MAP_SHARED | +MAP_ANONYMOUS ,-1 ,0 ); +igt_assert (data !=MAP_FAILED ); + }else if (flags &BO_MAP &&odd (i)) { +if (!bo) { +if (flags &FREE ) +free (data); +else + pending_free[i] = data; + } + + aligned_alloc_type =__aligned_alloc (aligned_size, bo_size); + data = aligned_alloc_type.ptr; +igt_assert (data); +__aligned_partial_free (&aligned_alloc_type); + + bo_flags =DRM_XE_GEM_CREATE_FLAG_NEEDS_VISIBLE_VRAM ; + bo =xe_bo_create (fd, vm, bo_size, +vram_if_possible (fd, eci->gt_id), + bo_flags); + data =xe_bo_map_fixed (fd, bo, bo_size, +to_user_pointer (data)); + +xe_vm_bind_async (fd, vm,0 , bo,0 , +to_user_pointer (data), + bo_size,0 ,0 ); + }else { +if (!bo) { +if (flags &FREE ) +free (data); +else + pending_free[i] = data; + } + bo =0 ; + data =aligned_alloc (aligned_size, bo_size); +igt_assert (data); + } + addr =to_user_pointer (data); +if (!(flags &SKIP_MEMSET )) +memset (data,0 , bo_size); + } + + prev_idx = idx; + } + +if (bo) { +__xe_vm_bind_assert (fd, vm,0 , +0 ,0 , addr, bo_size, +DRM_XE_VM_BIND_OP_MAP , +DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR , +NULL ,0 ,0 ,0 ); +munmap (data, bo_size); + data =NULL ; +gem_close (fd, bo); + } + +if (flags &BUSY ) +igt_assert_eq (unbind_system_allocator (), -EBUSY ); + +for (i =0 ; i < n_exec_queues; i++) +xe_exec_queue_destroy (fd, exec_queues[i]); + +if (exec_ufence) +munmap (exec_ufence,SZ_4K ); + +if (flags &LOCK ) +munlock (data, bo_size); + +if (file_fd !=-1 ) +close (file_fd); + +if (flags &NEW && !(flags &FREE )) { +for (i =0 ; i < n_execs; i++) { +if (!pending_free[i]) +continue ; + +if (flags &MMAP ) +munmap (pending_free[i], bo_size); +else +free (pending_free[i]); + } +free (pending_free); + } +if (data) { +if (flags &MMAP ) +munmap (data, bo_size); +else if (!alloc) +free (data); + } +if (free_vm) +xe_vm_destroy (fd, vm); +} + +struct thread_data { +pthread_t thread; +pthread_mutex_t *mutex; +pthread_cond_t *cond; +pthread_barrier_t *barrier; +int fd; +struct drm_xe_engine_class_instance *eci; +int n_exec_queues; +int n_execs; +size_t bo_size; +size_t stride; +uint32_t vm; +unsigned int flags; +void *alloc; +bool *go; +}; + +static void *thread (void *data) +{ +struct thread_data *t = data; + +pthread_mutex_lock (t->mutex); +while (!*t->go) +pthread_cond_wait (t->cond, t->mutex); +pthread_mutex_unlock (t->mutex); + +test_exec (t->fd, t->eci, t->n_exec_queues, t->n_execs, + t->bo_size, t->stride, t->vm, t->alloc, t->barrier, + t->flags); + +return NULL ; +} + +static void +threads (int fd,int n_exec_queues,int n_execs,size_t bo_size, +size_t stride,unsigned int flags,bool shared_vm) +{ +struct drm_xe_engine_class_instance *hwe; +struct thread_data *threads_data; +int n_engines =0 , i =0 ; +pthread_mutex_t mutex; +pthread_cond_t cond; +pthread_barrier_t barrier; +uint32_t vm =0 ; +bool go = false; +void *alloc =NULL ; + +if ((FILE_BACKED |FORK_READ ) & flags) +return ; + +xe_for_each_engine (fd, hwe) + ++n_engines; + +if (shared_vm) { + vm =xe_vm_create (fd,DRM_XE_VM_CREATE_FLAG_LR_MODE | +DRM_XE_VM_CREATE_FLAG_FAULT_MODE ,0 ); +bind_system_allocator (NULL ,0 ); + } + +if (flags &SHARED_ALLOC ) { +uint64_t alloc_size; + +igt_assert (stride); + + alloc_size =sizeof (struct test_exec_data ) * stride * + n_execs * n_engines; + alloc_size =xe_bb_size (fd, alloc_size); + alloc =aligned_alloc (SZ_2M , alloc_size); +igt_assert (alloc); + +memset (alloc,0 , alloc_size); + flags &= ~SHARED_ALLOC ; + } + + threads_data =calloc (n_engines,sizeof (*threads_data)); +igt_assert (threads_data); + +pthread_mutex_init (&mutex,0 ); +pthread_cond_init (&cond,0 ); +pthread_barrier_init (&barrier,0 , n_engines); + +xe_for_each_engine (fd, hwe) { + threads_data[i].mutex = &mutex; + threads_data[i].cond = &cond; + threads_data[i].barrier = (flags &SYNC_EXEC ) ? &barrier :NULL ; + threads_data[i].fd = fd; + threads_data[i].eci = hwe; + threads_data[i].n_exec_queues = n_exec_queues; + threads_data[i].n_execs = n_execs; + threads_data[i].bo_size = bo_size; + threads_data[i].stride = stride; + threads_data[i].vm = vm; + threads_data[i].flags = flags; + threads_data[i].alloc = alloc ? alloc + i * +sizeof (struct test_exec_data ) :NULL ; + threads_data[i].go = &go; +pthread_create (&threads_data[i].thread,0 , thread, + &threads_data[i]); + ++i; + } + +pthread_mutex_lock (&mutex); + go = true; +pthread_cond_broadcast (&cond); +pthread_mutex_unlock (&mutex); + +for (i =0 ; i < n_engines; ++i) +pthread_join (threads_data[i].thread,NULL ); + +if (shared_vm) { +int ret; + +if (flags &MMAP ) { +int tries =300 ; + +while (tries && (ret =unbind_system_allocator ()) == -EBUSY ) { +sleep (.01 ); + --tries; + } +igt_assert_eq (ret,0 ); + } +xe_vm_destroy (fd, vm); +if (alloc) +free (alloc); + } +free (threads_data); +} + +static void process (struct drm_xe_engine_class_instance *hwe,int n_exec_queues, +int n_execs,size_t bo_size,size_t stride, +unsigned int flags) +{ +struct process_data *pdata; +int map_fd; +int fd; + + map_fd =open (sync_file,O_RDWR ,0x666 ); + pdata =mmap (NULL ,sizeof (*pdata),PROT_READ | +PROT_WRITE ,MAP_SHARED , map_fd,0 ); +wait_pdata (pdata); + + fd =drm_open_driver (DRIVER_XE ); +test_exec (fd, hwe, n_exec_queues, n_execs, + bo_size, stride,0 ,NULL ,NULL , flags); +drm_close_driver (fd); + +close (map_fd); +munmap (pdata,sizeof (*pdata)); +} + +static void +processes (int fd,int n_exec_queues,int n_execs,size_t bo_size, +size_t stride,unsigned int flags) +{ +struct drm_xe_engine_class_instance *hwe; +struct process_data *pdata; +int map_fd; + +if (flags &FORK_READ ) +return ; + + map_fd =open (sync_file,O_RDWR |O_CREAT ,0x666 ); +posix_fallocate (map_fd,0 ,sizeof (*pdata)); + pdata =mmap (NULL ,sizeof (*pdata),PROT_READ | +PROT_WRITE ,MAP_SHARED , map_fd,0 ); + +init_pdata (pdata,0 ); + +xe_for_each_engine (fd, hwe) { + igt_fork(child, 1) +process (hwe, n_exec_queues, n_execs, bo_size, + stride, flags); + } + +signal_pdata (pdata); +igt_waitchildren (); + +close (map_fd); +munmap (pdata,sizeof (*pdata)); +} + +struct section { +const char *name; +unsigned int flags; +}; + +igt_main +{ +struct drm_xe_engine_class_instance *hwe; +const struct section sections[] = { + {"malloc" ,0 }, + {"malloc-multi-fault" ,MULTI_FAULT }, + {"malloc-fork-read" ,FORK_READ }, + {"malloc-fork-read-after" ,FORK_READ |FORK_READ_AFTER }, + {"malloc-mlock" ,LOCK }, + {"malloc-race" ,RACE }, + {"malloc-busy" ,BUSY }, + {"malloc-bo-unmap" ,BO_UNMAP }, + {"mmap" ,MMAP }, + {"mmap-remap" ,MMAP |MREMAP }, + {"mmap-remap-dontunmap" ,MMAP |MREMAP |DONTUNMAP }, + {"mmap-remap-ro" ,MMAP |MREMAP |READ_ONLY_REMAP }, + {"mmap-remap-ro-dontunmap" ,MMAP |MREMAP |DONTUNMAP | +READ_ONLY_REMAP }, + {"mmap-remap-eocheck" ,MMAP |MREMAP |EVERY_OTHER_CHECK }, + {"mmap-remap-dontunmap-eocheck" ,MMAP |MREMAP |DONTUNMAP | +EVERY_OTHER_CHECK }, + {"mmap-remap-ro-eocheck" ,MMAP |MREMAP |READ_ONLY_REMAP | +EVERY_OTHER_CHECK }, + {"mmap-remap-ro-dontunmap-eocheck" ,MMAP |MREMAP |DONTUNMAP | +READ_ONLY_REMAP |EVERY_OTHER_CHECK }, + {"mmap-huge" ,MMAP |HUGE_PAGE }, + {"mmap-shared" ,MMAP |LOCK |MMAP_SHARED }, + {"mmap-shared-remap" ,MMAP |LOCK |MMAP_SHARED |MREMAP }, + {"mmap-shared-remap-dontunmap" ,MMAP |LOCK |MMAP_SHARED | +MREMAP |DONTUNMAP }, + {"mmap-shared-remap-eocheck" ,MMAP |LOCK |MMAP_SHARED | +MREMAP |EVERY_OTHER_CHECK }, + {"mmap-shared-remap-dontunmap-eocheck" ,MMAP |LOCK | +MMAP_SHARED |MREMAP |DONTUNMAP |EVERY_OTHER_CHECK }, + {"mmap-mlock" ,MMAP |LOCK }, + {"mmap-file" ,MMAP |FILE_BACKED }, + {"mmap-file-mlock" ,MMAP |LOCK |FILE_BACKED }, + {"mmap-race" ,MMAP |RACE }, + {"free" ,NEW |FREE }, + {"free-race" ,NEW |FREE |RACE }, + {"new" ,NEW }, + {"new-race" ,NEW |RACE }, + {"new-bo-map" ,NEW |BO_MAP }, + {"new-busy" ,NEW |BUSY }, + {"mmap-free" ,MMAP |NEW |FREE }, + {"mmap-free-huge" ,MMAP |NEW |FREE |HUGE_PAGE }, + {"mmap-free-race" ,MMAP |NEW |FREE |RACE }, + {"mmap-new" ,MMAP |NEW }, + {"mmap-new-huge" ,MMAP |NEW |HUGE_PAGE }, + {"mmap-new-race" ,MMAP |NEW |RACE }, + {"malloc-nomemset" ,SKIP_MEMSET }, + {"malloc-mlock-nomemset" ,SKIP_MEMSET |LOCK }, + {"malloc-race-nomemset" ,SKIP_MEMSET |RACE }, + {"malloc-busy-nomemset" ,SKIP_MEMSET |BUSY }, + {"malloc-bo-unmap-nomemset" ,SKIP_MEMSET |BO_UNMAP }, + {"mmap-nomemset" ,SKIP_MEMSET |MMAP }, + {"mmap-huge-nomemset" ,SKIP_MEMSET |MMAP |HUGE_PAGE }, + {"mmap-shared-nomemset" ,SKIP_MEMSET |MMAP |MMAP_SHARED }, + {"mmap-mlock-nomemset" ,SKIP_MEMSET |MMAP |LOCK }, + {"mmap-file-nomemset" ,SKIP_MEMSET |MMAP |FILE_BACKED }, + {"mmap-file-mlock-nomemset" ,SKIP_MEMSET |MMAP |LOCK |FILE_BACKED }, + {"mmap-race-nomemset" ,SKIP_MEMSET |MMAP |RACE }, + {"free-nomemset" ,SKIP_MEMSET |NEW |FREE }, + {"free-race-nomemset" ,SKIP_MEMSET |NEW |FREE |RACE }, + {"new-nomemset" ,SKIP_MEMSET |NEW }, + {"new-race-nomemset" ,SKIP_MEMSET |NEW |RACE }, + {"new-bo-map-nomemset" ,SKIP_MEMSET |NEW |BO_MAP }, + {"new-busy-nomemset" ,SKIP_MEMSET |NEW |BUSY }, + {"mmap-free-nomemset" ,SKIP_MEMSET |MMAP |NEW |FREE }, + {"mmap-free-huge-nomemset" ,SKIP_MEMSET |MMAP |NEW |FREE |HUGE_PAGE }, + {"mmap-free-race-nomemset" ,SKIP_MEMSET |MMAP |NEW |FREE |RACE }, + {"mmap-new-nomemset" ,SKIP_MEMSET |MMAP |NEW }, + {"mmap-new-huge-nomemset" ,SKIP_MEMSET |MMAP |NEW |HUGE_PAGE }, + {"mmap-new-race-nomemset" ,SKIP_MEMSET |MMAP |NEW |RACE }, + {NULL }, + }; +const struct section psections[] = { + {"munmap-cpu-fault" ,CPU_FAULT }, + {"munmap-no-cpu-fault" ,0 }, + {"remap-cpu-fault" ,CPU_FAULT |REMAP }, + {"remap-no-cpu-fault" ,REMAP }, + {"middle-munmap-cpu-fault" ,MIDDLE |CPU_FAULT }, + {"middle-munmap-no-cpu-fault" ,MIDDLE }, + {"middle-remap-cpu-fault" ,MIDDLE |CPU_FAULT |REMAP }, + {"middle-remap-no-cpu-fault" ,MIDDLE |REMAP }, + {NULL }, + }; +const struct section esections[] = { + {"malloc" ,0 }, + {"malloc-mix-bo" ,MIX_BO_ALLOC }, + {NULL }, + }; +int fd; + + igt_fixture { +struct xe_device *xe; + + fd =drm_open_driver (DRIVER_XE ); +igt_require (!xe_supports_faults (fd)); + + xe =xe_device_get (fd); + va_bits = xe->va_bits; +open_sync_file (); + } + +for (const struct section *s = sections; s->name; s++) { + igt_subtest_f("once-%s", s->name) +xe_for_each_engine (fd, hwe) +test_exec (fd, hwe,1 ,1 ,0 ,0 ,0 ,NULL , +NULL , s->flags); + + igt_subtest_f("once-large-%s", s->name) +xe_for_each_engine (fd, hwe) +test_exec (fd, hwe,1 ,1 ,SZ_2M ,0 ,0 ,NULL , +NULL , s->flags); + + igt_subtest_f("twice-%s", s->name) +xe_for_each_engine (fd, hwe) +test_exec (fd, hwe,1 ,2 ,0 ,0 ,0 ,NULL , +NULL , s->flags); + + igt_subtest_f("twice-large-%s", s->name) +xe_for_each_engine (fd, hwe) +test_exec (fd, hwe,1 ,2 ,SZ_2M ,0 ,0 ,NULL , +NULL , s->flags); + + igt_subtest_f("many-%s", s->name) +xe_for_each_engine (fd, hwe) +test_exec (fd, hwe,1 ,128 ,0 ,0 ,0 ,NULL , +NULL , s->flags); + + igt_subtest_f("many-stride-%s", s->name) +xe_for_each_engine (fd, hwe) +test_exec (fd, hwe,1 ,128 ,0 ,256 ,0 ,NULL , +NULL , s->flags); + + igt_subtest_f("many-execqueues-%s", s->name) +xe_for_each_engine (fd, hwe) +test_exec (fd, hwe,16 ,128 ,0 ,0 ,0 ,NULL , +NULL , s->flags); + + igt_subtest_f("many-large-%s", s->name) +xe_for_each_engine (fd, hwe) +test_exec (fd, hwe,1 ,128 ,SZ_2M ,0 ,0 ,NULL , +NULL , s->flags); + + igt_subtest_f("many-large-execqueues-%s", s->name) +xe_for_each_engine (fd, hwe) +test_exec (fd, hwe,16 ,128 ,SZ_2M ,0 ,0 ,NULL , +NULL , s->flags); + + igt_subtest_f("threads-many-%s", s->name) +threads (fd,1 ,128 ,0 ,0 , s->flags, false); + + igt_subtest_f("threads-many-stride-%s", s->name) +threads (fd,1 ,128 ,0 ,256 , s->flags, false); + + igt_subtest_f("threads-many-execqueues-%s", s->name) +threads (fd,16 ,128 ,0 ,0 , s->flags, false); + + igt_subtest_f("threads-many-large-%s", s->name) +threads (fd,1 ,128 ,SZ_2M ,0 , s->flags, false); + + igt_subtest_f("threads-many-large-execqueues-%s", s->name) +threads (fd,16 ,128 ,SZ_2M ,0 , s->flags, false); + + igt_subtest_f("threads-shared-vm-many-%s", s->name) +threads (fd,1 ,128 ,0 ,0 , s->flags, true); + + igt_subtest_f("threads-shared-vm-many-stride-%s", s->name) +threads (fd,1 ,128 ,0 ,256 , s->flags, true); + + igt_subtest_f("threads-shared-vm-many-execqueues-%s", s->name) +threads (fd,16 ,128 ,0 ,0 , s->flags, true); + + igt_subtest_f("threads-shared-vm-many-large-%s", s->name) +threads (fd,1 ,128 ,SZ_2M ,0 , s->flags, true); + + igt_subtest_f("threads-shared-vm-many-large-execqueues-%s", s->name) +threads (fd,16 ,128 ,SZ_2M ,0 , s->flags, true); + + igt_subtest_f("process-many-%s", s->name) +processes (fd,1 ,128 ,0 ,0 , s->flags); + + igt_subtest_f("process-many-stride-%s", s->name) +processes (fd,1 ,128 ,0 ,256 , s->flags); + + igt_subtest_f("process-many-execqueues-%s", s->name) +processes (fd,16 ,128 ,0 ,0 , s->flags); + + igt_subtest_f("process-many-large-%s", s->name) +processes (fd,1 ,128 ,SZ_2M ,0 , s->flags); + + igt_subtest_f("process-many-large-execqueues-%s", s->name) +processes (fd,16 ,128 ,SZ_2M ,0 , s->flags); + } + + igt_subtest("threads-shared-vm-shared-alloc-many-stride-malloc") +threads (fd,1 ,128 ,0 ,256 ,SHARED_ALLOC , true); + + igt_subtest("threads-shared-vm-shared-alloc-many-stride-malloc-race") +threads (fd,1 ,128 ,0 ,256 ,RACE |SHARED_ALLOC , true); + + igt_subtest("threads-shared-alloc-many-stride-malloc") +threads (fd,1 ,128 ,0 ,256 ,SHARED_ALLOC , false); + + igt_subtest("threads-shared-alloc-many-stride-malloc-sync") +threads (fd,1 ,128 ,0 ,256 ,SHARED_ALLOC |SYNC_EXEC , false); + + igt_subtest("threads-shared-alloc-many-stride-malloc-race") +threads (fd,1 ,128 ,0 ,256 ,RACE |SHARED_ALLOC , false); + + igt_subtest_f("fault") +xe_for_each_engine (fd, hwe) +test_exec (fd, hwe,4 ,1 ,SZ_2M ,0 ,0 ,NULL ,NULL , +FAULT ); + +for (const struct section *s = psections; s->name; s++) { + igt_subtest_f("partial-%s", s->name) +xe_for_each_engine (fd, hwe) +partial (fd, hwe, s->flags); + } + + igt_subtest_f("unaligned-alloc") +xe_for_each_engine (fd, hwe) { +many_allocs (fd, hwe, (SZ_1M +SZ_512K ) *8 , +SZ_1M +SZ_512K ,SZ_4K ,NULL ,0 ); +break ; + } + + igt_subtest_f("fault-benchmark") +xe_for_each_engine (fd, hwe) +many_allocs (fd, hwe,SZ_64M ,SZ_64M ,SZ_4K ,NULL , +BENCHMARK ); + + igt_subtest_f("fault-threads-benchmark") +xe_for_each_engine (fd, hwe) +many_allocs (fd, hwe,SZ_64M ,SZ_64M ,SZ_4K ,NULL , +BENCHMARK |CPU_FAULT_THREADS ); + + igt_subtest_f("fault-threads-same-page-benchmark") +xe_for_each_engine (fd, hwe) +many_allocs (fd, hwe,SZ_64M ,SZ_64M ,SZ_4K ,NULL , +BENCHMARK |CPU_FAULT_THREADS | +CPU_FAULT_SAME_PAGE ); + + igt_subtest_f("fault-process-benchmark") +xe_for_each_engine (fd, hwe) +many_allocs (fd, hwe,SZ_64M ,SZ_64M ,SZ_4K ,NULL , +BENCHMARK |CPU_FAULT_PROCESS ); + + igt_subtest_f("fault-process-same-page-benchmark") +xe_for_each_engine (fd, hwe) +many_allocs (fd, hwe,SZ_64M ,SZ_64M ,SZ_4K ,NULL , +BENCHMARK |CPU_FAULT_PROCESS | +CPU_FAULT_SAME_PAGE ); + +for (const struct section *s = esections; s->name; s++) { + igt_subtest_f("evict-%s", s->name) +xe_for_each_engine (fd, hwe) { +many_allocs (fd, hwe, +xe_visible_vram_size (fd, hwe->gt_id), +SZ_8M ,SZ_1M ,NULL , s->flags); +break ; + } + } + +for (const struct section *s = esections; s->name; s++) { + igt_subtest_f("processes-evict-%s", s->name) +processes_evict (fd,SZ_8M ,SZ_1M , s->flags); + } + + igt_fixture { +xe_device_put (fd); +drm_close_driver (fd); +close_sync_file (); + } +}tests/meson.build +1
-0@@ -295,6 +295,7 @@ intel_xe_progs = [ 'xe_exec_reset', 'xe_exec_sip', 'xe_exec_store', + 'xe_exec_system_allocator', 'xe_exec_threads', 'xe_exercise_blt', 'xe_fault_injection',Signature · 1 line
2.34.1
Re: [PATCH v3 2/2] tests/xe: Add system_allocator test
Quoted 47 lines · Jonathan
-----Original Message-----
From: igt-dev <[email protected]> On Behalf Of Matthew Brost
Sent: Thursday, April 24, 2025 1:44 PM
To: [email protected]
Cc: Dugast, Francois <[email protected]>
Subject: [PATCH v3 2/2] tests/xe: Addsystem_allocatortestQuoted 41 lines
Test various uses of system allocator in single thread, multiple threads, and multiple processes.
Features tested:
- Malloc with various size
- Mmap with various sizes and flags including file backed mappings
- Mixing BO allocations with system allocator
- Various page sizes
- Dynamically freeing / unmapping memory
- Sharing VM across threads
- Faults racing on different hardware engines / GTs / Tiles
- GPU faults and CPU faults racing
- CPU faults on multiple threads racing
- CPU faults on multiple process racing
- GPU faults of memory not faulted in by CPU
- Partial unmap of allocations
- Attempting to unmap system allocations when GPU has mappings
- Eviction of both system allocations and BOs
- Forking child processes and reading data from VRAM
- mremap data in VRAM
- Protection changes
- Multiple faults per execbufRunning on LNL, BMG, PVC 1 tile, and PVC 2 tile.
v2:
- Rebase
- Fix memory allocation to not interfear with malloc (Thomas)
v3:
- Fix memory leak (Francois)
- Break out uAPI into own patch (Francois)
- Use mkstemp for sync file (Francois)
- Use mkstemp for file backed data (Francois)
- Drop i argument fromREAD_VALUE(Francois)
- Fix test description (Francois)
- Add comment tocheck_all_pages_process(Francois)
- Preferigt_infoover printf (Francois)
- Fix types in messages (Francois)
- Prefer odd macro (Francois)
Quoted 6 lines
Quoted 1 line
Signed-off-by Matthew Brost <[email protected]>LGTM, though maybe it would be better to split the various tests introduced in
xe_exec_system_allocator.cacross multiple patches, instead of creating all of the tests at once?
Francois and I discussed this but there is really a great way to split this and IMO large patches as IGTs matters a whole lot less than large KMD patches.
Quoted 53 lines
I'm okay with creating the test series either way, so this isn't a blocking request.
The remaining nits below are non-blocking as well, so this is:
Reviewed-by Jonathan Cavitt <[email protected]>Quoted 47 lines
--- lib/xe/xe_ioctl.c | 12 + lib/xe/xe_ioctl.h | 1 + tests/intel/xe_exec_system_allocator.c | 1849 ++++++++++++++++++++++++ tests/meson.build | 1 + 4 files changed, 1863 insertions(+) create mode 100644 tests/intel/xe_exec_system_allocator.clib/xe/xe_ioctl.c +12
-0@@ -440,6 +440,18 @@ void *xe_bo_map(int fd, uint32_t bo, size_t size)return __xe_bo_map (fd, bo, size,PROT_WRITE ); } +void *xe_bo_map_fixed (int fd,uint32_t bo,size_t size,uint64_t addr) +{ +uint64_t mmo; +void *map; + + mmo =xe_bo_mmap_offset (fd, bo); + map =mmap ((void *)addr, size,PROT_WRITE ,MAP_SHARED |MAP_FIXED , fd, mmo); +igt_assert (map !=MAP_FAILED ); + +return map; +} +void *xe_bo_mmap_ext (int fd,uint32_t bo,size_t size,int prot) {return __xe_bo_map (fd, bo, size, prot);lib/xe/xe_ioctl.h +1
-0@@ -86,6 +86,7 @@ uint32_t xe_exec_queue_create_class(int fd, uint32_t vm, uint16_t class);void xe_exec_queue_destroy (int fd,uint32_t exec_queue);uint64_t xe_bo_mmap_offset (int fd,uint32_t bo);void *xe_bo_map (int fd,uint32_t bo,size_t size); +void *xe_bo_map_fixed (int fd,uint32_t bo,size_t size,long unsigned int addr);void *xe_bo_mmap_ext (int fd,uint32_t bo,size_t size,int prot);int __xe_exec (int fd,struct drm_xe_exec *exec);void xe_exec (int fd,struct drm_xe_exec *exec);tests/intel/xe_exec_system_allocator.c new file +0
-1--- /dev/null
Quoted 9 lines
tests/intel/xe_exec_system_allocator.c line 3
Quoted 5 lines
+++
b/tests/intel/xe_exec_system_allocator.ctests/intel/xe_exec_system_allocator.c new file +3
-0@@ -0,0 +1,1849 @@ +// SPDX-License-Identifier: MIT +/* +* Copyright (c) 2024 Intel Corporation NIT:
Shouldn't this be a 2025 copyright?
Originally authored in 2024.
Quoted 82 lines
Quoted 82 lines
tests/intel/xe_exec_system_allocator.c new file +82
-0+*/ + +/** +* TEST: Basic tests for execbuf functionality using system allocator +* Category: Core +* Mega feature: USM +* Sub-category: System allocator +* Functionality: fault mode, system allocator +* GPU: LNL, BMG, PVC +*/ + +#include <fcntl.h> +#include <linux/mman.h> +#include <time.h> + +#include "igt.h" +#include "lib/igt_syncobj.h" +#include "lib/intel_reg.h" +#include "xe_drm.h" + +#include "xe/xe_ioctl.h" +#include "xe/xe_query.h" +#include <string.h> + +#define USER_FENCE_VALUE 0xdeadbeefdeadbeefull +#define QUARTER_SEC (NSEC_PER_SEC / 4) +#define FIVE_SEC (5LL * NSEC_PER_SEC) + +struct batch_data { +uint32_t batch[16 ]; +uint64_t pad; +uint32_t data; +uint32_t expected_data; +}; + +#define WRITE_VALUE (data__, i__) ({ \ + if (!(data__)->expected_data) \ + (data__)->expected_data = rand() << 12 | (i__); \ + (data__)->expected_data; \ +}) +#define READ_VALUE (data__) ((data__)->expected_data) + +static void __write_dword (uint32_t *batch,uint64_t sdi_addr,uint32_t wdata, +int *idx) +{ + batch[(*idx)++] =MI_STORE_DWORD_IMM_GEN4 ; + batch[(*idx)++] = sdi_addr; + batch[(*idx)++] = sdi_addr >>32 ; + batch[(*idx)++] = wdata; +} + +static void write_dword (uint32_t *batch,uint64_t sdi_addr,uint32_t wdata, +int *idx) +{ +__write_dword (batch, sdi_addr, wdata, idx); + batch[(*idx)++] =MI_BATCH_BUFFER_END ; +} + +static void check_all_pages (void *ptr,uint64_t alloc_size,uint64_t stride, +pthread_barrier_t *barrier) +{ +int i, n_writes = alloc_size / stride; + +for (i =0 ; i < n_writes; ++i) { +struct batch_data *data = ptr + i * stride; + +igt_assert_eq (data->data,READ_VALUE (data)); + +if (barrier) +pthread_barrier_wait (barrier); + } +} + +static char sync_file[] ="/tmp/xe_exec_system_allocator_syncXXXXXX" ; +static int sync_fd; + +static void open_sync_file (void ) +{ + sync_fd =mkstemp (sync_file); +} + +static void close_sync_file (void )
Quoted 9 lines
tests/intel/xe_exec_system_allocator.c line 88
Quoted 3 lines
tests/intel/xe_exec_system_allocator.c new file +3
-0+{ +close (sync_fd); +}NIT:
Thesync_fddoesn't seem like it's being used for anything. However, just because I don't know what this file does doesn't mean it does nothing, so I won't block on this.
The name is used by child processes to open their open fd. The sync_fd is just used by the master process to close the file upon exit.
Quoted 686 lines
Quoted 671 lines
tests/intel/xe_exec_system_allocator.c new file +671
-0+ +struct process_data { +pthread_mutex_t mutex; +pthread_cond_t cond; +pthread_barrier_t barrier; +bool go; +}; + +static void wait_pdata (struct process_data *pdata) +{ +pthread_mutex_lock (&pdata->mutex); +while (!pdata->go) +pthread_cond_wait (&pdata->cond, &pdata->mutex); +pthread_mutex_unlock (&pdata->mutex); +} + +static void init_pdata (struct process_data *pdata,int n_engine) +{ +pthread_mutexattr_t mutex_attr; +pthread_condattr_t cond_attr; +pthread_barrierattr_t barrier_attr; + +pthread_mutexattr_init (&mutex_attr); +pthread_mutexattr_setpshared (&mutex_attr,PTHREAD_PROCESS_SHARED ); +pthread_mutex_init (&pdata->mutex, &mutex_attr); + +pthread_condattr_init (&cond_attr); +pthread_condattr_setpshared (&cond_attr,PTHREAD_PROCESS_SHARED ); +pthread_cond_init (&pdata->cond, &cond_attr); + +pthread_barrierattr_init (&barrier_attr); +pthread_barrierattr_setpshared (&barrier_attr,PTHREAD_PROCESS_SHARED ); +pthread_barrier_init (&pdata->barrier, &barrier_attr, n_engine); + + pdata->go = false; +} + +static void signal_pdata (struct process_data *pdata) +{ +pthread_mutex_lock (&pdata->mutex); + pdata->go = true; +pthread_cond_broadcast (&pdata->cond); +pthread_mutex_unlock (&pdata->mutex); +} + +/* many_alloc flags */ +#define MIX_BO_ALLOC (0x1 << 0) +#define BENCHMARK (0x1 << 1) +#define CPU_FAULT_THREADS (0x1 << 2) +#define CPU_FAULT_PROCESS (0x1 << 3) +#define CPU_FAULT_SAME_PAGE (0x1 << 4) + +static void process_check (void *ptr,uint64_t alloc_size,uint64_t stride, +unsigned int flags) +{ +struct process_data *pdata; +int map_fd; + + map_fd =open (sync_file,O_RDWR ,0x666 ); + pdata =mmap (NULL ,sizeof (*pdata),PROT_READ | +PROT_WRITE ,MAP_SHARED , map_fd,0 ); +wait_pdata (pdata); + +if (flags &CPU_FAULT_SAME_PAGE ) +check_all_pages (ptr, alloc_size, stride, &pdata->barrier); +else +check_all_pages (ptr, alloc_size, stride,NULL ); + +close (map_fd); +munmap (pdata,sizeof (*pdata)); +} + +/* +* Partition checking of results in chunks which causes multiple processes to +* fault same VRAM allocation in parallel. +*/ +static void +check_all_pages_process (void *ptr,uint64_t alloc_size,uint64_t stride, +int n_process,unsigned int flags) +{ +struct process_data *pdata; +int map_fd, i; + + map_fd =open (sync_file,O_RDWR |O_CREAT ,0x666 ); +posix_fallocate (map_fd,0 ,sizeof (*pdata)); + pdata =mmap (NULL ,sizeof (*pdata),PROT_READ | +PROT_WRITE ,MAP_SHARED , map_fd,0 ); + +init_pdata (pdata, n_process); + +for (i =0 ; i < n_process; ++i) { + igt_fork(child, 1) +if (flags &CPU_FAULT_SAME_PAGE ) +process_check (ptr, alloc_size, stride, flags); +else +process_check (ptr + stride * i, alloc_size, + stride * n_process, flags); + } + +signal_pdata (pdata); +igt_waitchildren (); + +close (map_fd); +munmap (pdata,sizeof (*pdata)); +} + +struct thread_check_data { +pthread_t thread; +pthread_mutex_t *mutex; +pthread_cond_t *cond; +pthread_barrier_t *barrier; +void *ptr; +uint64_t alloc_size; +uint64_t stride; +bool *go; +}; + +static void *thread_check (void *data) +{ +struct thread_check_data *t = data; + +pthread_mutex_lock (t->mutex); +while (!*t->go) +pthread_cond_wait (t->cond, t->mutex); +pthread_mutex_unlock (t->mutex); + +check_all_pages (t->ptr, t->alloc_size, t->stride, t->barrier); + +return NULL ; +} + +/* +* Partition checking of results in chunks which causes multiple threads to +* fault same VRAM allocation in parallel. +*/ +static void +check_all_pages_threads (void *ptr,uint64_t alloc_size,uint64_t stride, +int n_threads,unsigned int flags) +{ +struct thread_check_data *threads_check_data; +pthread_mutex_t mutex; +pthread_cond_t cond; +pthread_barrier_t barrier; +int i; +bool go = false; + + threads_check_data =calloc (n_threads,sizeof (*threads_check_data)); +igt_assert (threads_check_data); + +pthread_mutex_init (&mutex,0 ); +pthread_cond_init (&cond,0 ); +pthread_barrier_init (&barrier,0 , n_threads); + +for (i =0 ; i < n_threads; ++i) { + threads_check_data[i].mutex = &mutex; + threads_check_data[i].cond = &cond; +if (flags &CPU_FAULT_SAME_PAGE ) { + threads_check_data[i].barrier = &barrier; + threads_check_data[i].ptr = ptr; + threads_check_data[i].alloc_size = alloc_size; + threads_check_data[i].stride = stride; + }else { + threads_check_data[i].barrier =NULL ; + threads_check_data[i].ptr = ptr + stride * i; + threads_check_data[i].alloc_size = alloc_size; + threads_check_data[i].stride = n_threads * stride; + } + threads_check_data[i].go = &go; + +pthread_create (&threads_check_data[i].thread,0 , thread_check, + &threads_check_data[i]); + } + +pthread_mutex_lock (&mutex); + go = true; +pthread_cond_broadcast (&cond); +pthread_mutex_unlock (&mutex); + +for (i =0 ; i < n_threads; ++i) +pthread_join (threads_check_data[i].thread,NULL ); +free (threads_check_data); +} + +static void touch_all_pages (int fd,uint32_t exec_queue,void *ptr, +uint64_t alloc_size,uint64_t stride, +struct timespec *tv,uint64_t *submit) +{ +struct drm_xe_sync sync[1 ] = { + { .type =DRM_XE_SYNC_TYPE_USER_FENCE , + .flags =DRM_XE_SYNC_FLAG_SIGNAL , + .timeline_value =USER_FENCE_VALUE }, + }; +struct drm_xe_exec exec = { + .num_batch_buffer =1 , + .num_syncs =0 , + .exec_queue_id = exec_queue, + .syncs =to_user_pointer (&sync), + }; +uint64_t addr =to_user_pointer (ptr); +int i, ret, n_writes = alloc_size / stride; +u64 *exec_ufence =NULL ; +int64_t timeout =FIVE_SEC ; + + exec_ufence =mmap (NULL ,SZ_4K ,PROT_READ | +PROT_WRITE ,MAP_SHARED | +MAP_ANONYMOUS ,-1 ,0 ); +igt_assert (exec_ufence !=MAP_FAILED ); +memset (exec_ufence,0 ,SZ_4K ); + sync[0 ].addr =to_user_pointer (exec_ufence); + +for (i =0 ; i < n_writes; ++i, addr += stride) { +struct batch_data *data = ptr + i * stride; +uint64_t sdi_offset = (char *)&data->data - (char *)data; +uint64_t sdi_addr = addr + sdi_offset; +int b =0 ; + +write_dword (data->batch, sdi_addr,WRITE_VALUE (data, i), &b); +igt_assert (b <=ARRAY_SIZE (data->batch)); + } + +igt_nsec_elapsed (tv); + *submit =igt_nsec_elapsed (tv); + + addr =to_user_pointer (ptr); +for (i =0 ; i < n_writes; ++i, addr += stride) { +struct batch_data *data = ptr + i * stride; +uint64_t batch_offset = (char *)&data->batch - (char *)data; +uint64_t batch_addr = addr + batch_offset; + + exec.address = batch_addr; +if (i +1 == n_writes) + exec.num_syncs =1 ; +xe_exec (fd, &exec); + } + + ret =__xe_wait_ufence (fd, exec_ufence,USER_FENCE_VALUE , exec_queue, + &timeout); +if (ret) { +igt_info ("FAIL EXEC_UFENCE_ADDR: 0x%016llx\n" , sync[0 ].addr); +igt_info ("FAIL EXEC_UFENCE: EXPECTED=0x%016llx, ACTUAL=0x%016lx\n" , +USER_FENCE_VALUE , exec_ufence[0 ]); + + addr =to_user_pointer (ptr); +for (i =0 ; i < n_writes; ++i, addr += stride) { +struct batch_data *data = ptr + i * stride; +uint64_t batch_offset = (char *)&data->batch - (char *)data; +uint64_t batch_addr = addr + batch_offset; +uint64_t sdi_offset = (char *)&data->data - (char *)data; +uint64_t sdi_addr = addr + sdi_offset; + +igt_info ("FAIL BATCH_ADDR: 0x%016lx\n" , batch_addr); +igt_info ("FAIL SDI_ADDR: 0x%016lx\n" , sdi_addr); +igt_info ("FAIL SDI_ADDR (in batch): 0x%016lx\n" , + (((u64 )data->batch[2 ]) <<32 ) | data->batch[1 ]); +igt_info ("FAIL DATA: EXPECTED=0x%08x, ACTUAL=0x%08x\n" , + data->expected_data, data->data); + } +igt_assert_eq (ret,0 ); + } +munmap (exec_ufence,SZ_4K ); +} + +static int va_bits; + +#define bind_system_allocator (__sync, __num_sync) \ + __xe_vm_bind_assert(fd, vm, 0, \ + 0, 0, 0, 0x1ull << va_bits, \ + DRM_XE_VM_BIND_OP_MAP, \ + DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR, \ + (__sync), (__num_sync), 0, 0) + +#define unbind_system_allocator () \ + __xe_vm_bind(fd, vm, 0, 0, 0, 0, 0x1ull << va_bits, \ + DRM_XE_VM_BIND_OP_UNMAP, 0, \ + NULL, 0, 0, 0, 0) + +#define odd (__i) (__i & 1) + +struct aligned_alloc_type { +void *__ptr; +void *ptr; +size_t __size; +size_t size; +}; + +static struct aligned_alloc_type __aligned_alloc (size_t alignment,size_t size) +{ +struct aligned_alloc_type aligned_alloc_type; + + aligned_alloc_type.__ptr =mmap (NULL , alignment + size,PROT_NONE ,MAP_PRIVATE | +MAP_ANONYMOUS ,-1 ,0 ); +igt_assert (aligned_alloc_type.__ptr !=MAP_FAILED ); + + aligned_alloc_type.ptr = (void *)ALIGN ((uint64_t )aligned_alloc_type.__ptr, alignment); + aligned_alloc_type.size = size; + aligned_alloc_type.__size = size + alignment; + +return aligned_alloc_type; +} + +static void __aligned_free (struct aligned_alloc_type *aligned_alloc_type) +{ +munmap (aligned_alloc_type->__ptr, aligned_alloc_type->__size); +} + +static void __aligned_partial_free (struct aligned_alloc_type *aligned_alloc_type) +{ +size_t begin_size = (size_t )(aligned_alloc_type->ptr - aligned_alloc_type->__ptr); + +if (begin_size) +munmap (aligned_alloc_type->__ptr, begin_size); +if (aligned_alloc_type->__size - aligned_alloc_type->size - begin_size) +munmap (aligned_alloc_type->ptr + aligned_alloc_type->size, + aligned_alloc_type->__size - aligned_alloc_type->size - begin_size); +} + +/** +* SUBTEST: unaligned-alloc +* Description: allocate unaligned sizes of memory +* Test category: functionality test +* +* SUBTEST: fault-benchmark +* Description: Benchmark how long GPU / CPU take +* Test category: performance test +* +* SUBTEST: fault-threads-benchmark +* Description: Benchmark how long GPU / CPU take, reading results with multiple threads +* Test category: performance and functionality test +* +* SUBTEST: fault-threads-same-page-benchmark +* Description: Benchmark how long GPU / CPU take, reading results with multiple threads, hammer same page +* Test category: performance and functionality test +* +* SUBTEST: fault-process-benchmark +* Description: Benchmark how long GPU / CPU take, reading results with multiple process +* Test category: performance and functionality test +* +* SUBTEST: fault-process-same-page-benchmark +* Description: Benchmark how long GPU / CPU take, reading results with multiple process, hammer same page +* Test category: performance and functionality test +* +* SUBTEST: evict-malloc +* Description: trigger eviction of VRAM allocated via malloc +* Test category: functionality test +* +* SUBTEST: evict-malloc-mix-bo +* Description: trigger eviction of VRAM allocated via malloc and BO create +* Test category: functionality test +* +* SUBTEST: processes-evict-malloc +* Description: multi-process trigger eviction of VRAM allocated via malloc +* Test category: stress test +* +* SUBTEST: processes-evict-malloc-mix-bo +* Description: multi-process trigger eviction of VRAM allocated via malloc and BO create +* Test category: stress test +*/ + +static void +many_allocs (int fd,struct drm_xe_engine_class_instance *eci, +uint64_t total_alloc,uint64_t alloc_size,uint64_t stride, +pthread_barrier_t *barrier,unsigned int flags) +{ +uint32_t vm, exec_queue; +int num_allocs = flags &BENCHMARK ?1 : + (9 * (total_alloc / alloc_size)) /8 ; +struct aligned_alloc_type *allocs; +uint32_t *bos =NULL ; +struct timespec tv = {}; +uint64_t submit, read, elapsed; +int i; + + vm =xe_vm_create (fd,DRM_XE_VM_CREATE_FLAG_LR_MODE | +DRM_XE_VM_CREATE_FLAG_FAULT_MODE ,0 ); + exec_queue =xe_exec_queue_create (fd, vm, eci,0 ); + +bind_system_allocator (NULL ,0 ); + + allocs =malloc (sizeof (*allocs) * num_allocs); +igt_assert (allocs); +memset (allocs,0 ,sizeof (*allocs) * num_allocs); + +if (flags &MIX_BO_ALLOC ) { + bos =malloc (sizeof (*bos) * num_allocs); +igt_assert (bos); +memset (bos,0 ,sizeof (*bos) * num_allocs); + } + +for (i =0 ; i < num_allocs; ++i) { +struct aligned_alloc_type alloc; + +if (flags &MIX_BO_ALLOC &&odd (i)) { +uint32_t bo_flags = +DRM_XE_GEM_CREATE_FLAG_NEEDS_VISIBLE_VRAM ; + + alloc =__aligned_alloc (SZ_2M , alloc_size); +igt_assert (alloc.ptr); + + bos[i] =xe_bo_create (fd, vm, alloc_size, +vram_if_possible (fd, eci->gt_id), + bo_flags); + alloc.ptr =xe_bo_map_fixed (fd, bos[i], alloc_size, +to_user_pointer (alloc.ptr)); +xe_vm_bind_async (fd, vm,0 , bos[i],0 , +to_user_pointer (alloc.ptr), + alloc_size,0 ,0 ); + }else { + alloc.ptr =aligned_alloc (SZ_2M , alloc_size); +igt_assert (alloc.ptr); + } + allocs[i] = alloc; + +touch_all_pages (fd, exec_queue, allocs[i].ptr, alloc_size, stride, + &tv, &submit); + } + +if (barrier) +pthread_barrier_wait (barrier); + +for (i =0 ; i < num_allocs; ++i) { +if (flags &BENCHMARK ) + read =igt_nsec_elapsed (&tv); +#define NUM_CHECK_THREADS 8 +if (flags &CPU_FAULT_PROCESS ) +check_all_pages_process (allocs[i].ptr, alloc_size, stride, +NUM_CHECK_THREADS , flags); +else if (flags &CPU_FAULT_THREADS ) +check_all_pages_threads (allocs[i].ptr, alloc_size, stride, +NUM_CHECK_THREADS , flags); +else +check_all_pages (allocs[i].ptr, alloc_size, stride,NULL ); +if (flags &BENCHMARK ) { + elapsed =igt_nsec_elapsed (&tv); +igt_info ("Execution took %.3fms (submit %.1fus, read %.1fus, total %.1fus, read_total %.1fus)\n" , +1e-6 * elapsed,1e-3 * submit,1e-3 * read, +1e-3 * (elapsed - submit), +1e-3 * (elapsed - read)); + } +if (bos && bos[i]) { +__aligned_free (allocs + i); +gem_close (fd, bos[i]); + }else { +free (allocs[i].ptr); + } + } +if (bos) +free (bos); +free (allocs); +xe_exec_queue_destroy (fd, exec_queue); +xe_vm_destroy (fd, vm); +} + +static void process_evict (struct drm_xe_engine_class_instance *hwe, +uint64_t total_alloc,uint64_t alloc_size, +uint64_t stride,unsigned int flags) +{ +struct process_data *pdata; +int map_fd; +int fd; + + map_fd =open (sync_file,O_RDWR ,0x666 ); + pdata =mmap (NULL ,sizeof (*pdata),PROT_READ | +PROT_WRITE ,MAP_SHARED , map_fd,0 ); +wait_pdata (pdata); + + fd =drm_open_driver (DRIVER_XE ); +many_allocs (fd, hwe, total_alloc, alloc_size, stride, &pdata->barrier, + flags); +drm_close_driver (fd); + +close (map_fd); +munmap (pdata,sizeof (*pdata)); +} + +static void +processes_evict (int fd,uint64_t alloc_size,uint64_t stride, +unsigned int flags) +{ +struct drm_xe_engine_class_instance *hwe; +struct process_data *pdata; +int n_engine_gt[2 ] = {0 ,0 }, n_engine =0 ; +int map_fd; + + map_fd =open (sync_file,O_RDWR |O_CREAT ,0x666 ); +posix_fallocate (map_fd,0 ,sizeof (*pdata)); + pdata =mmap (NULL ,sizeof (*pdata),PROT_READ | +PROT_WRITE ,MAP_SHARED , map_fd,0 ); + +xe_for_each_engine (fd, hwe) { +igt_assert (hwe->gt_id <2 ); + n_engine_gt[hwe->gt_id]++; + n_engine++; + } + +init_pdata (pdata, n_engine); + +xe_for_each_engine (fd, hwe) { + igt_fork(child, 1) +process_evict (hwe, +xe_visible_vram_size (fd, hwe->gt_id) / + n_engine_gt[hwe->gt_id], alloc_size, + stride, flags); + } + +signal_pdata (pdata); +igt_waitchildren (); + +close (map_fd); +munmap (pdata,sizeof (*pdata)); +} + +#define CPU_FAULT (0x1 << 0) +#define REMAP (0x1 << 1) +#define MIDDLE (0x1 << 2) + +/** +* SUBTEST: partial-munmap-cpu-fault +* Description: munmap partially with cpu access in between +* Test category: functionality test +* +* SUBTEST: partial-munmap-no-cpu-fault +* Description: munmap partially with no cpu access in between +* Test category: functionality test +* +* SUBTEST: partial-remap-cpu-fault +* Description: remap partially with cpu access in between +* Test category: functionality test +* +* SUBTEST: partial-remap-no-cpu-fault +* Description: remap partially with no cpu access in between +* Test category: functionality test +* +* SUBTEST: partial-middle-munmap-cpu-fault +* Description: munmap middle with cpu access in between +* Test category: functionality test +* +* SUBTEST: partial-middle-munmap-no-cpu-fault +* Description: munmap middle with no cpu access in between +* Test category: functionality test +* +* SUBTEST: partial-middle-remap-cpu-fault +* Description: remap middle with cpu access in between +* Test category: functionality test +* +* SUBTEST: partial-middle-remap-no-cpu-fault +* Description: remap middle with no cpu access in between +* Test category: functionality test +*/ + +static void +partial (int fd,struct drm_xe_engine_class_instance *eci,unsigned int flags) +{ +struct drm_xe_sync sync[1 ] = { + { .type =DRM_XE_SYNC_TYPE_USER_FENCE , .flags =DRM_XE_SYNC_FLAG_SIGNAL , + .timeline_value =USER_FENCE_VALUE }, + }; +struct drm_xe_exec exec = { + .num_batch_buffer =1 , + .num_syncs =1 , + .syncs =to_user_pointer (sync), + }; +struct { +uint32_t batch[16 ]; +uint64_t pad; +uint64_t vm_sync; +uint64_t exec_sync; +uint32_t data; +uint32_t expected_data; + } *data; +size_t bo_size =SZ_2M , unmap_offset =0 ; +uint32_t vm, exec_queue; +u64 *exec_ufence =NULL ; +int i; +void *old, *new =NULL ; +struct aligned_alloc_type alloc; + +if (flags &MIDDLE ) + unmap_offset = bo_size /4 ; + + vm =xe_vm_create (fd,DRM_XE_VM_CREATE_FLAG_LR_MODE | +DRM_XE_VM_CREATE_FLAG_FAULT_MODE ,0 ); + + alloc =__aligned_alloc (bo_size, bo_size); +igt_assert (alloc.ptr); + + data =mmap (alloc.ptr, bo_size,PROT_READ |PROT_WRITE , +MAP_PRIVATE |MAP_ANONYMOUS |MAP_FIXED ,-1 ,0 ); +igt_assert (data !=MAP_FAILED ); +memset (data,0 , bo_size); + old = data; + + exec_queue =xe_exec_queue_create (fd, vm, eci,0 ); + + sync[0 ].addr =to_user_pointer (&data[0 ].vm_sync); +bind_system_allocator (sync,1 ); +xe_wait_ufence (fd, &data[0 ].vm_sync,USER_FENCE_VALUE ,0 ,FIVE_SEC ); + data[0 ].vm_sync =0 ; + + exec_ufence =mmap (NULL ,SZ_4K ,PROT_READ | +PROT_WRITE ,MAP_SHARED | +MAP_ANONYMOUS ,-1 ,0 ); +igt_assert (exec_ufence !=MAP_FAILED ); +memset (exec_ufence,0 ,SZ_4K ); + +for (i =0 ; i <2 ; i++) { +uint64_t addr =to_user_pointer (data); +uint64_t sdi_offset = (char *)&data[i].data - (char *)data; +uint64_t sdi_addr = addr + sdi_offset; +int b =0 ; + +write_dword (data[i].batch, sdi_addr,WRITE_VALUE (&data[i], i), &b); +igt_assert (b <=ARRAY_SIZE (data[i].batch)); + +if (!i) + data = old + unmap_offset + bo_size /2 ; + } + + data = old; + exec.exec_queue_id = exec_queue; + +for (i =0 ; i <2 ; i++) { +uint64_t addr =to_user_pointer (data); +uint64_t batch_offset = (char *)&data[i].batch - (char *)data; +uint64_t batch_addr = addr + batch_offset; + + sync[0 ].addr = new ?to_user_pointer (new) : +to_user_pointer (exec_ufence); + exec.address = batch_addr; +xe_exec (fd, &exec); + +xe_wait_ufence (fd, new ?: exec_ufence,USER_FENCE_VALUE , + exec_queue,FIVE_SEC ); +if (i || (flags &CPU_FAULT )) +igt_assert_eq (data[i].data,READ_VALUE (&data[i])); + exec_ufence[0 ] =0 ; + +if (!i) { + data = old + unmap_offset + bo_size /2 ; +munmap (old + unmap_offset, bo_size /2 ); +if (flags &REMAP ) { + new =mmap (old + unmap_offset, bo_size /2 , +PROT_READ |PROT_WRITE , +MAP_PRIVATE |MAP_ANONYMOUS |MAP_FIXED | +MAP_LOCKED ,-1 ,0 ); +igt_assert (new !=MAP_FAILED ); + } + } + } + +xe_exec_queue_destroy (fd, exec_queue); +munmap (exec_ufence,SZ_4K ); +__aligned_free (&alloc); +if (new) +munmap (new, bo_size /2 ); +xe_vm_destroy (fd, vm); +} + +#define MAX_N_EXEC_QUEUES 16 + +#define MMAP (0x1 << 0) +#define NEW (0x1 << 1) +#define BO_UNMAP (0x1 << 2) +#define FREE (0x1 << 3) +#define BUSY (0x1 << 4) +#define BO_MAP (0x1 << 5) +#define RACE (0x1 << 6) +#define SKIP_MEMSET (0x1 << 7) +#define FAULT (0x1 << 8) +#define FILE_BACKED (0x1 << 9) +#define LOCK (0x1 << 10) +#define MMAP_SHARED (0x1 << 11)tests/intel/xe_exec_system_allocator.c line 769
Quoted 10 lines
tests/intel/xe_exec_system_allocator.c new file +10
-0+#define HUGE_PAGE (0x1 << 12) +#define SHARED_ALLOC (0x1 << 13) +#define FORK_READ (0x1 << 14) +#define FORK_READ_AFTER (0x1 << 15) +#define MREMAP (0x1 << 16) +#define DONTUNMAP (0x1 << 17) +#define READ_ONLY_REMAP (0x1 << 18) +#define SYNC_EXEC (0x1 << 19) +#define EVERY_OTHER_CHECK (0x1 << 20) +#define MULTI_FAULT (0x1 << 21)NIT:
The above doesn't look aligned, but I'm willing to bet that has to do with my email client and that it's aligned in code.
Quoted 4 lines
However, there is a notable standout with
EVERY_OTHER_CHECK. IMO, I think all of the values should be aligned to that if they aren't already.
Yea EVERY_OTHER_CHECK is unaligned, I can align all the flags to that alignment.
Matt
Quoted 1,098 lines
-Jonathan Cavitt
Quoted 1,096 lines
tests/intel/xe_exec_system_allocator.c new file +1080
-0+ +#define N_MULTI_FAULT 4 + +/** +* SUBTEST: once-%s +* Description: Run %arg[1] system allocator test only once +* Test category: functionality test +* +* SUBTEST: once-large-%s +* Description: Run %arg[1] system allocator test only once with large allocation +* Test category: functionality test +* +* SUBTEST: twice-%s +* Description: Run %arg[1] system allocator test twice +* Test category: functionality test +* +* SUBTEST: twice-large-%s +* Description: Run %arg[1] system allocator test twice with large allocation +* Test category: functionality test +* +* SUBTEST: many-%s +* Description: Run %arg[1] system allocator test many times +* Test category: stress test +* +* SUBTEST: many-stride-%s +* Description: Run %arg[1] system allocator test many times with a stride on each exec +* Test category: stress test +* +* SUBTEST: many-execqueues-%s +* Description: Run %arg[1] system allocator test on many exec_queues +* Test category: stress test +* +* SUBTEST: many-large-%s +* Description: Run %arg[1] system allocator test many times with large allocations +* Test category: stress test +* +* SUBTEST: many-large-execqueues-%s +* Description: Run %arg[1] system allocator test on many exec_queues with large allocations +* +* SUBTEST: threads-many-%s +* Description: Run %arg[1] system allocator threaded test many times +* Test category: stress test +* +* SUBTEST: threads-many-stride-%s +* Description: Run %arg[1] system allocator threaded test many times with a stride on each exec +* Test category: stress test +* +* SUBTEST: threads-many-execqueues-%s +* Description: Run %arg[1] system allocator threaded test on many exec_queues +* Test category: stress test +* +* SUBTEST: threads-many-large-%s +* Description: Run %arg[1] system allocator threaded test many times with large allocations +* Test category: stress test +* +* SUBTEST: threads-many-large-execqueues-%s +* Description: Run %arg[1] system allocator threaded test on many exec_queues with large allocations +* +* SUBTEST: threads-shared-vm-many-%s +* Description: Run %arg[1] system allocator threaded, shared vm test many times +* Test category: stress test +* +* SUBTEST: threads-shared-vm-many-stride-%s +* Description: Run %arg[1] system allocator threaded, shared vm test many times with a stride on each exec +* Test category: stress test +* +* SUBTEST: threads-shared-vm-many-execqueues-%s +* Description: Run %arg[1] system allocator threaded, shared vm test on many exec_queues +* Test category: stress test +* +* SUBTEST: threads-shared-vm-many-large-%s +* Description: Run %arg[1] system allocator threaded, shared vm test many times with large allocations +* Test category: stress test +* +* SUBTEST: threads-shared-vm-many-large-execqueues-%s +* Description: Run %arg[1] system allocator threaded, shared vm test on many exec_queues with large allocations +* Test category: stress test +* +* SUBTEST: process-many-%s +* Description: Run %arg[1] system allocator multi-process test many times +* Test category: stress test +* +* SUBTEST: process-many-stride-%s +* Description: Run %arg[1] system allocator multi-process test many times with a stride on each exec +* Test category: stress test +* +* SUBTEST: process-many-execqueues-%s +* Description: Run %arg[1] system allocator multi-process test on many exec_queues +* Test category: stress test +* +* SUBTEST: process-many-large-%s +* Description: Run %arg[1] system allocator multi-process test many times with large allocations +* Test category: stress test +* +* SUBTEST: process-many-large-execqueues-%s +* Description: Run %arg[1] system allocator multi-process test on many exec_queues with large allocations +* +* SUBTEST: fault +* Description: use a bad system allocator address resulting in a fault +* Test category: bad input +* +* arg[1]: +* +* @malloc: malloc single buffer for all execs, issue a command which will trigger multiple faults +* @malloc-multi-fault: malloc single buffer for all execs +* @malloc-fork-read: malloc single buffer for all execs, fork a process to read test output +* @malloc-fork-read-after: malloc single buffer for all execs, fork a process to read test output, check again after fork returns in parent +* @malloc-mlock: malloc and mlock single buffer for all execs +* @malloc-race: malloc single buffer for all execs with race between cpu and gpu access +* @malloc-bo-unmap: malloc single buffer for all execs, bind and unbind a BO to same address before execs +* @malloc-busy: malloc single buffer for all execs, try to unbind while buffer valid +* @mmap: mmap single buffer for all execs +* @mmap-remap: mmap and mremap a buffer for all execs +* @mmap-remap-dontunmap: mmap and mremap a buffer with dontunmap flag for all execs +* @mmap-remap-ro: mmap and mremap a read-only buffer for all execs +* @mmap-remap-ro-dontunmap: mmap and mremap a read-only buffer with dontunmap flag for all execs +* @mmap-remap-eocheck: mmap and mremap a buffer for all execs, check data every other loop iteration +* @mmap-remap-dontunmap-eocheck: mmap and mremap a buffer with dontunmap flag for all execs, check data every other loop iteration +* @mmap-remap-ro-eocheck: mmap and mremap a read-only buffer for all execs, check data every other loop iteration +* @mmap-remap-ro-dontunmap-eocheck: mmap and mremap a read-only buffer with dontunmap flag for all execs, check data every other loop iteration +* @mmap-huge: mmap huge page single buffer for all execs +* @mmap-shared: mmap shared single buffer for all execs +* @mmap-shared-remap: mmap shared and mremap a buffer for all execs +* @mmap-shared-remap-dontunmap: mmap shared and mremap a buffer with dontunmap flag for all execs +* @mmap-shared-remap-eocheck: mmap shared and mremap a buffer for all execs, check data every other loop iteration +* @mmap-shared-remap-dontunmap-eocheck: mmap shared and mremap a buffer with dontunmap flag for all execs, check data every other loop iteration +* @mmap-mlock: mmap and mlock single buffer for all execs +* @mmap-file: mmap single buffer, with file backing, for all execs +* @mmap-file-mlock: mmap and mlock single buffer, with file backing, for all execs +* @mmap-race: mmap single buffer for all execs with race between cpu and gpu access +* @free: malloc and free buffer for each exec +* @free-race: malloc and free buffer for each exec with race between cpu and gpu access +* @new: malloc a new buffer for each exec +* @new-race: malloc a new buffer for each exec with race between cpu and gpu access +* @new-bo-map: malloc a new buffer or map BO for each exec +* @new-busy: malloc a new buffer for each exec, try to unbind while buffers valid +* @mmap-free: mmap and free buffer for each exec +* @mmap-free-huge: mmap huge page and free buffer for each exec +* @mmap-free-race: mmap and free buffer for each exec with race between cpu and gpu access +* @mmap-new: mmap a new buffer for each exec +* @mmap-new-huge: mmap huge page a new buffer for each exec +* @mmap-new-race: mmap a new buffer for each exec with race between cpu and gpu access +* @malloc-nomemset: malloc single buffer for all execs, skip memset of buffers +* @malloc-mlock-nomemset: malloc and mlock single buffer for all execs, skip memset of buffers +* @malloc-race-nomemset: malloc single buffer for all execs with race between cpu and gpu access, skip memset of buffers +* @malloc-bo-unmap-nomemset: malloc single buffer for all execs, bind and unbind a BO to same address before execs, skip memset of buffers +* @malloc-busy-nomemset: malloc single buffer for all execs, try to unbind while buffer valid, skip memset of buffers +* @mmap-nomemset: mmap single buffer for all execs, skip memset of buffers +* @mmap-huge-nomemset: mmap huge page single buffer for all execs, skip memset of buffers +* @mmap-shared-nomemset: mmap shared single buffer for all execs, skip memset of buffers +* @mmap-mlock-nomemset: mmap and mlock single buffer for all execs, skip memset of buffers +* @mmap-file-nomemset: mmap single buffer, with file backing, for all execs, skip memset of buffers +* @mmap-file-mlock-nomemset: mmap and mlock single buffer, with file backing, for all execs, skip memset of buffers +* @mmap-race-nomemset: mmap single buffer for all execs with race between cpu and gpu access, skip memset of buffers +* @free-nomemset: malloc and free buffer for each exec, skip memset of buffers +* @free-race-nomemset: malloc and free buffer for each exec with race between cpu and gpu access, skip memset of buffers +* @new-nomemset: malloc a new buffer for each exec, skip memset of buffers +* @new-race-nomemset: malloc a new buffer for each exec with race between cpu and gpu access, skip memset of buffers +* @new-bo-map-nomemset: malloc a new buffer or map BO for each exec, skip memset of buffers +* @new-busy-nomemset: malloc a new buffer for each exec, try to unbind while buffers valid, skip memset of buffers +* @mmap-free-nomemset: mmap and free buffer for each exec, skip memset of buffers +* @mmap-free-huge-nomemset: mmap huge page and free buffer for each exec, skip memset of buffers +* @mmap-free-race-nomemset: mmap and free buffer for each exec with race between cpu and gpu access, skip memset of buffers +* @mmap-new-nomemset: mmap a new buffer for each exec, skip memset of buffers +* @mmap-new-huge-nomemset: mmap huge page new buffer for each exec, skip memset of buffers +* @mmap-new-race-nomemset: mmap a new buffer for each exec with race between cpu and gpu access, skip memset of buffers +* +* SUBTEST: threads-shared-vm-shared-alloc-many-stride-malloc +* Description: Create multiple threads with a shared VM triggering faults on different hardware engines to same addresses +* Test category: stress test +* +* SUBTEST: threads-shared-vm-shared-alloc-many-stride-malloc-race +* Description: Create multiple threads with a shared VM triggering faults on different hardware engines to same addresses, racing between CPU and GPU access +* Test category: stress test +* +* SUBTEST: threads-shared-alloc-many-stride-malloc +* Description: Create multiple threads with a faults on different hardware engines to same addresses +* Test category: stress test +* +* SUBTEST: threads-shared-alloc-many-stride-malloc-sync +* Description: Create multiple threads with a faults on different hardware engines to same addresses, syncing on each exec +* Test category: stress test +* +* SUBTEST: threads-shared-alloc-many-stride-malloc-race +* Description: Create multiple threads with a faults on different hardware engines to same addresses, racing between CPU and GPU access +* Test category: stress test +*/ + +struct test_exec_data { +uint32_t batch[32 ]; +uint64_t pad; +uint64_t vm_sync; +uint64_t exec_sync; +uint32_t data; +uint32_t expected_data; +}; + +static void +test_exec (int fd,struct drm_xe_engine_class_instance *eci, +int n_exec_queues,int n_execs,size_t bo_size, +size_t stride,uint32_t vm,void *alloc,pthread_barrier_t *barrier, +unsigned int flags) +{ +uint64_t addr; +struct drm_xe_sync sync[1 ] = { + { .type =DRM_XE_SYNC_TYPE_USER_FENCE , .flags =DRM_XE_SYNC_FLAG_SIGNAL , + .timeline_value =USER_FENCE_VALUE }, + }; +struct drm_xe_exec exec = { + .num_batch_buffer =1 , + .num_syncs =1 , + .syncs =to_user_pointer (sync), + }; +uint32_t exec_queues[MAX_N_EXEC_QUEUES ]; +struct test_exec_data *data, *next_data =NULL ; +uint32_t bo_flags; +uint32_t bo =0 ; +void **pending_free; +u64 *exec_ufence =NULL ; +int i, j, b, file_fd =-1 , prev_idx; +bool free_vm = false; +size_t aligned_size = bo_size ?:xe_get_default_alignment (fd); +size_t orig_size = bo_size; +struct aligned_alloc_type aligned_alloc_type; + +if (flags &MULTI_FAULT ) { +if (!bo_size) +return ; + + bo_size *=N_MULTI_FAULT ; + } + +if (flags &SHARED_ALLOC ) +return ; + +if (flags &EVERY_OTHER_CHECK &&odd (n_execs)) +return ; + +if (flags &EVERY_OTHER_CHECK ) +igt_assert (flags &MREMAP ); + +igt_assert (n_exec_queues <=MAX_N_EXEC_QUEUES ); + +if (flags &NEW && !(flags &FREE )) { + pending_free =malloc (sizeof (*pending_free) * n_execs); +igt_assert (pending_free); +memset (pending_free,0 ,sizeof (*pending_free) * n_execs); + } + +if (!vm) { + vm =xe_vm_create (fd,DRM_XE_VM_CREATE_FLAG_LR_MODE | +DRM_XE_VM_CREATE_FLAG_FAULT_MODE ,0 ); + free_vm = true; + } +if (!bo_size) { +if (!stride) { + bo_size =sizeof (*data) * n_execs; + bo_size =xe_bb_size (fd, bo_size); + }else { + bo_size = stride * n_execs *sizeof (*data); + bo_size =xe_bb_size (fd, bo_size); + } + } +if (flags &HUGE_PAGE ) { + aligned_size =ALIGN (aligned_size,SZ_2M ); + bo_size =ALIGN (bo_size,SZ_2M ); + } + +if (alloc) { + data = alloc; + }else { +if (flags &MMAP ) { +int mmap_flags =MAP_FIXED ; + + aligned_alloc_type =__aligned_alloc (aligned_size, bo_size); + data = aligned_alloc_type.ptr; +igt_assert (data); +__aligned_partial_free (&aligned_alloc_type); + +if (flags &MMAP_SHARED ) + mmap_flags |=MAP_SHARED ; +else + mmap_flags |=MAP_PRIVATE ; + +if (flags &HUGE_PAGE ) + mmap_flags |=MAP_HUGETLB |MAP_HUGE_2MB ; + +if (flags &FILE_BACKED ) { +char name[] ="/tmp/xe_exec_system_allocator_datXXXXXX" ; + +igt_assert (!(flags &NEW )); + + file_fd =mkstemp (name); +posix_fallocate (file_fd,0 , bo_size); + }else { + mmap_flags |=MAP_ANONYMOUS ; + } + + data =mmap (data, bo_size,PROT_READ | +PROT_WRITE , mmap_flags, file_fd,0 ); +igt_assert (data !=MAP_FAILED ); + }else { + data =aligned_alloc (aligned_size, bo_size); +igt_assert (data); + } +if (!(flags &SKIP_MEMSET )) +memset (data,0 , bo_size); +if (flags &LOCK ) { +igt_assert (!(flags &NEW )); +mlock (data, bo_size); + } + } + +for (i =0 ; i < n_exec_queues; i++) + exec_queues[i] =xe_exec_queue_create (fd, vm, eci,0 ); + + sync[0 ].addr =to_user_pointer (&data[0 ].vm_sync); +if (free_vm) { +bind_system_allocator (sync,1 ); +xe_wait_ufence (fd, &data[0 ].vm_sync,USER_FENCE_VALUE ,0 ,FIVE_SEC ); + } + data[0 ].vm_sync =0 ; + + addr =to_user_pointer (data); + +if (flags &BO_UNMAP ) { + bo_flags =DRM_XE_GEM_CREATE_FLAG_NEEDS_VISIBLE_VRAM ; + bo =xe_bo_create (fd, vm, bo_size, +vram_if_possible (fd, eci->gt_id), bo_flags); +xe_vm_bind_async (fd, vm,0 , bo,0 , addr, bo_size,0 ,0 ); + +__xe_vm_bind_assert (fd, vm,0 , +0 ,0 , addr, bo_size, +DRM_XE_VM_BIND_OP_MAP , +DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR , sync, +1 ,0 ,0 ); +xe_wait_ufence (fd, &data[0 ].vm_sync,USER_FENCE_VALUE ,0 , +FIVE_SEC ); + data[0 ].vm_sync =0 ; +gem_close (fd, bo); + bo =0 ; + } + +if (!(flags &RACE )) { + exec_ufence =mmap (NULL ,SZ_4K ,PROT_READ | +PROT_WRITE ,MAP_SHARED | +MAP_ANONYMOUS ,-1 ,0 ); +igt_assert (exec_ufence !=MAP_FAILED ); +memset (exec_ufence,0 ,SZ_4K ); + } + +for (i =0 ; i < n_execs; i++) { +int idx = !stride ? i : i * stride, next_idx = !stride + ? (i +1 ) : (i +1 ) * stride; +uint64_t batch_offset = (char *)&data[idx].batch - (char *)data; +uint64_t batch_addr = addr + batch_offset; +uint64_t sdi_offset = (char *)&data[idx].data - (char *)data; +uint64_t sdi_addr = addr + sdi_offset; +int e = i % n_exec_queues, err; +bool fault_inject = (FAULT & flags) && i == n_execs /2 ; +bool fault_injected = (FAULT & flags) && i > n_execs; + +if (barrier) +pthread_barrier_wait (barrier); + +if (flags &MULTI_FAULT ) { + b =0 ; +for (j =0 ; j <N_MULTI_FAULT -1 ; ++j) +__write_dword (data[idx].batch, + sdi_addr + j * orig_size, +WRITE_VALUE (&data[idx], idx), &b); +write_dword (data[idx].batch, sdi_addr + j * orig_size, +WRITE_VALUE (&data[idx], idx), &b); +igt_assert (b <=ARRAY_SIZE (data[idx].batch)); + }else if (!(flags &EVERY_OTHER_CHECK )) { + b =0 ; +write_dword (data[idx].batch, sdi_addr, +WRITE_VALUE (&data[idx], idx), &b); +igt_assert (b <=ARRAY_SIZE (data[idx].batch)); + }else if (flags &EVERY_OTHER_CHECK && !odd (i)) { + b =0 ; +write_dword (data[idx].batch, sdi_addr, +WRITE_VALUE (&data[idx], idx), &b); +igt_assert (b <=ARRAY_SIZE (data[idx].batch)); + + aligned_alloc_type =__aligned_alloc (aligned_size, bo_size); + next_data = aligned_alloc_type.ptr; +igt_assert (next_data); +__aligned_partial_free (&aligned_alloc_type); + + b =0 ; +write_dword (data[next_idx].batch, +to_user_pointer (next_data) + + (char *)&data[next_idx].data - (char *)data, +WRITE_VALUE (&data[next_idx], next_idx), &b); +igt_assert (b <=ARRAY_SIZE (data[next_idx].batch)); + } + +if (!exec_ufence) + data[idx].exec_sync =0 ; + + sync[0 ].addr = exec_ufence ?to_user_pointer (exec_ufence) : + addr + (char *)&data[idx].exec_sync - (char *)data; + + exec.exec_queue_id = exec_queues[e]; +if (fault_inject) + exec.address = batch_addr *2 ; +else + exec.address = batch_addr; + +if (fault_injected) { + err =__xe_exec (fd, &exec); +igt_assert (err == -ENOENT ); + }else { +xe_exec (fd, &exec); + } + +if (barrier) +pthread_barrier_wait (barrier); + +if (fault_inject || fault_injected) { +int64_t timeout =QUARTER_SEC ; + + err =__xe_wait_ufence (fd, exec_ufence ? exec_ufence : + &data[idx].exec_sync, +USER_FENCE_VALUE , + exec_queues[e], &timeout); +igt_assert (err == -ETIME || err == -EIO ); + }else { +xe_wait_ufence (fd, exec_ufence ? exec_ufence : + &data[idx].exec_sync,USER_FENCE_VALUE , + exec_queues[e],FIVE_SEC ); +if (flags &LOCK && !i) +munlock (data, bo_size); + +if (flags &MREMAP ) { +void *old = data; +int remap_flags =MREMAP_MAYMOVE |MREMAP_FIXED ; + +if (flags &DONTUNMAP ) + remap_flags |=MREMAP_DONTUNMAP ; + +if (flags &READ_ONLY_REMAP ) +igt_assert (!mprotect (old, bo_size, +PROT_READ )); + +if (!next_data) { + aligned_alloc_type =__aligned_alloc (aligned_size, + bo_size); + data = aligned_alloc_type.ptr; +__aligned_partial_free (&aligned_alloc_type); + }else { + data = next_data; + } + next_data =NULL ; +igt_assert (data); + + data =mremap (old, bo_size, bo_size, + remap_flags, data); +igt_assert (data !=MAP_FAILED ); + +if (flags &READ_ONLY_REMAP ) +igt_assert (!mprotect (data, bo_size, +PROT_READ | +PROT_WRITE )); + + addr =to_user_pointer (data); +if (flags &DONTUNMAP ) +munmap (old, bo_size); + } + +if (!(flags &EVERY_OTHER_CHECK ) ||odd (i)) { +if (flags &FORK_READ ) { + igt_fork(child, 1) +igt_assert_eq (data[idx].data, +READ_VALUE (&data[idx])); +if (!(flags &FORK_READ_AFTER )) +igt_assert_eq (data[idx].data, +READ_VALUE (&data[idx])); +igt_waitchildren (); +if (flags &FORK_READ_AFTER ) +igt_assert_eq (data[idx].data, +READ_VALUE (&data[idx])); + }else { +igt_assert_eq (data[idx].data, +READ_VALUE (&data[idx])); + +if (flags &MULTI_FAULT ) { +for (j =1 ; j <N_MULTI_FAULT ; ++j) { +struct test_exec_data *__data = + ((void *)data) + j * orig_size; + +igt_assert_eq (__data[idx].data, +READ_VALUE (&data[idx])); + } + } + } +if (flags &EVERY_OTHER_CHECK ) +igt_assert_eq (data[prev_idx].data, +READ_VALUE (&data[prev_idx])); + } + } + +if (exec_ufence) + exec_ufence[0 ] =0 ; + +if (bo) { +__xe_vm_bind_assert (fd, vm,0 , +0 ,0 , addr, bo_size, +DRM_XE_VM_BIND_OP_MAP , +DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR , +NULL ,0 ,0 ,0 ); +munmap (data, bo_size); +gem_close (fd, bo); + } + +if (flags &NEW ) { +if (flags &MMAP ) { +if (flags &FREE ) +munmap (data, bo_size); +else + pending_free[i] = data; + data =mmap (NULL , bo_size,PROT_READ | +PROT_WRITE ,MAP_SHARED | +MAP_ANONYMOUS ,-1 ,0 ); +igt_assert (data !=MAP_FAILED ); + }else if (flags &BO_MAP &&odd (i)) { +if (!bo) { +if (flags &FREE ) +free (data); +else + pending_free[i] = data; + } + + aligned_alloc_type =__aligned_alloc (aligned_size, bo_size); + data = aligned_alloc_type.ptr; +igt_assert (data); +__aligned_partial_free (&aligned_alloc_type); + + bo_flags =DRM_XE_GEM_CREATE_FLAG_NEEDS_VISIBLE_VRAM ; + bo =xe_bo_create (fd, vm, bo_size, +vram_if_possible (fd, eci->gt_id), + bo_flags); + data =xe_bo_map_fixed (fd, bo, bo_size, +to_user_pointer (data)); + +xe_vm_bind_async (fd, vm,0 , bo,0 , +to_user_pointer (data), + bo_size,0 ,0 ); + }else { +if (!bo) { +if (flags &FREE ) +free (data); +else + pending_free[i] = data; + } + bo =0 ; + data =aligned_alloc (aligned_size, bo_size); +igt_assert (data); + } + addr =to_user_pointer (data); +if (!(flags &SKIP_MEMSET )) +memset (data,0 , bo_size); + } + + prev_idx = idx; + } + +if (bo) { +__xe_vm_bind_assert (fd, vm,0 , +0 ,0 , addr, bo_size, +DRM_XE_VM_BIND_OP_MAP , +DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR , +NULL ,0 ,0 ,0 ); +munmap (data, bo_size); + data =NULL ; +gem_close (fd, bo); + } + +if (flags &BUSY ) +igt_assert_eq (unbind_system_allocator (), -EBUSY ); + +for (i =0 ; i < n_exec_queues; i++) +xe_exec_queue_destroy (fd, exec_queues[i]); + +if (exec_ufence) +munmap (exec_ufence,SZ_4K ); + +if (flags &LOCK ) +munlock (data, bo_size); + +if (file_fd !=-1 ) +close (file_fd); + +if (flags &NEW && !(flags &FREE )) { +for (i =0 ; i < n_execs; i++) { +if (!pending_free[i]) +continue ; + +if (flags &MMAP ) +munmap (pending_free[i], bo_size); +else +free (pending_free[i]); + } +free (pending_free); + } +if (data) { +if (flags &MMAP ) +munmap (data, bo_size); +else if (!alloc) +free (data); + } +if (free_vm) +xe_vm_destroy (fd, vm); +} + +struct thread_data { +pthread_t thread; +pthread_mutex_t *mutex; +pthread_cond_t *cond; +pthread_barrier_t *barrier; +int fd; +struct drm_xe_engine_class_instance *eci; +int n_exec_queues; +int n_execs; +size_t bo_size; +size_t stride; +uint32_t vm; +unsigned int flags; +void *alloc; +bool *go; +}; + +static void *thread (void *data) +{ +struct thread_data *t = data; + +pthread_mutex_lock (t->mutex); +while (!*t->go) +pthread_cond_wait (t->cond, t->mutex); +pthread_mutex_unlock (t->mutex); + +test_exec (t->fd, t->eci, t->n_exec_queues, t->n_execs, + t->bo_size, t->stride, t->vm, t->alloc, t->barrier, + t->flags); + +return NULL ; +} + +static void +threads (int fd,int n_exec_queues,int n_execs,size_t bo_size, +size_t stride,unsigned int flags,bool shared_vm) +{ +struct drm_xe_engine_class_instance *hwe; +struct thread_data *threads_data; +int n_engines =0 , i =0 ; +pthread_mutex_t mutex; +pthread_cond_t cond; +pthread_barrier_t barrier; +uint32_t vm =0 ; +bool go = false; +void *alloc =NULL ; + +if ((FILE_BACKED |FORK_READ ) & flags) +return ; + +xe_for_each_engine (fd, hwe) + ++n_engines; + +if (shared_vm) { + vm =xe_vm_create (fd,DRM_XE_VM_CREATE_FLAG_LR_MODE | +DRM_XE_VM_CREATE_FLAG_FAULT_MODE ,0 ); +bind_system_allocator (NULL ,0 ); + } + +if (flags &SHARED_ALLOC ) { +uint64_t alloc_size; + +igt_assert (stride); + + alloc_size =sizeof (struct test_exec_data ) * stride * + n_execs * n_engines; + alloc_size =xe_bb_size (fd, alloc_size); + alloc =aligned_alloc (SZ_2M , alloc_size); +igt_assert (alloc); + +memset (alloc,0 , alloc_size); + flags &= ~SHARED_ALLOC ; + } + + threads_data =calloc (n_engines,sizeof (*threads_data)); +igt_assert (threads_data); + +pthread_mutex_init (&mutex,0 ); +pthread_cond_init (&cond,0 ); +pthread_barrier_init (&barrier,0 , n_engines); + +xe_for_each_engine (fd, hwe) { + threads_data[i].mutex = &mutex; + threads_data[i].cond = &cond; + threads_data[i].barrier = (flags &SYNC_EXEC ) ? &barrier :NULL ; + threads_data[i].fd = fd; + threads_data[i].eci = hwe; + threads_data[i].n_exec_queues = n_exec_queues; + threads_data[i].n_execs = n_execs; + threads_data[i].bo_size = bo_size; + threads_data[i].stride = stride; + threads_data[i].vm = vm; + threads_data[i].flags = flags; + threads_data[i].alloc = alloc ? alloc + i * +sizeof (struct test_exec_data ) :NULL ; + threads_data[i].go = &go; +pthread_create (&threads_data[i].thread,0 , thread, + &threads_data[i]); + ++i; + } + +pthread_mutex_lock (&mutex); + go = true; +pthread_cond_broadcast (&cond); +pthread_mutex_unlock (&mutex); + +for (i =0 ; i < n_engines; ++i) +pthread_join (threads_data[i].thread,NULL ); + +if (shared_vm) { +int ret; + +if (flags &MMAP ) { +int tries =300 ; + +while (tries && (ret =unbind_system_allocator ()) == -EBUSY ) { +sleep (.01 ); + --tries; + } +igt_assert_eq (ret,0 ); + } +xe_vm_destroy (fd, vm); +if (alloc) +free (alloc); + } +free (threads_data); +} + +static void process (struct drm_xe_engine_class_instance *hwe,int n_exec_queues, +int n_execs,size_t bo_size,size_t stride, +unsigned int flags) +{ +struct process_data *pdata; +int map_fd; +int fd; + + map_fd =open (sync_file,O_RDWR ,0x666 ); + pdata =mmap (NULL ,sizeof (*pdata),PROT_READ | +PROT_WRITE ,MAP_SHARED , map_fd,0 ); +wait_pdata (pdata); + + fd =drm_open_driver (DRIVER_XE ); +test_exec (fd, hwe, n_exec_queues, n_execs, + bo_size, stride,0 ,NULL ,NULL , flags); +drm_close_driver (fd); + +close (map_fd); +munmap (pdata,sizeof (*pdata)); +} + +static void +processes (int fd,int n_exec_queues,int n_execs,size_t bo_size, +size_t stride,unsigned int flags) +{ +struct drm_xe_engine_class_instance *hwe; +struct process_data *pdata; +int map_fd; + +if (flags &FORK_READ ) +return ; + + map_fd =open (sync_file,O_RDWR |O_CREAT ,0x666 ); +posix_fallocate (map_fd,0 ,sizeof (*pdata)); + pdata =mmap (NULL ,sizeof (*pdata),PROT_READ | +PROT_WRITE ,MAP_SHARED , map_fd,0 ); + +init_pdata (pdata,0 ); + +xe_for_each_engine (fd, hwe) { + igt_fork(child, 1) +process (hwe, n_exec_queues, n_execs, bo_size, + stride, flags); + } + +signal_pdata (pdata); +igt_waitchildren (); + +close (map_fd); +munmap (pdata,sizeof (*pdata)); +} + +struct section { +const char *name; +unsigned int flags; +}; + +igt_main +{ +struct drm_xe_engine_class_instance *hwe; +const struct section sections[] = { + {"malloc" ,0 }, + {"malloc-multi-fault" ,MULTI_FAULT }, + {"malloc-fork-read" ,FORK_READ }, + {"malloc-fork-read-after" ,FORK_READ |FORK_READ_AFTER }, + {"malloc-mlock" ,LOCK }, + {"malloc-race" ,RACE }, + {"malloc-busy" ,BUSY }, + {"malloc-bo-unmap" ,BO_UNMAP }, + {"mmap" ,MMAP }, + {"mmap-remap" ,MMAP |MREMAP }, + {"mmap-remap-dontunmap" ,MMAP |MREMAP |DONTUNMAP }, + {"mmap-remap-ro" ,MMAP |MREMAP |READ_ONLY_REMAP }, + {"mmap-remap-ro-dontunmap" ,MMAP |MREMAP |DONTUNMAP | +READ_ONLY_REMAP }, + {"mmap-remap-eocheck" ,MMAP |MREMAP |EVERY_OTHER_CHECK }, + {"mmap-remap-dontunmap-eocheck" ,MMAP |MREMAP |DONTUNMAP | +EVERY_OTHER_CHECK }, + {"mmap-remap-ro-eocheck" ,MMAP |MREMAP |READ_ONLY_REMAP | +EVERY_OTHER_CHECK }, + {"mmap-remap-ro-dontunmap-eocheck" ,MMAP |MREMAP |DONTUNMAP | +READ_ONLY_REMAP |EVERY_OTHER_CHECK }, + {"mmap-huge" ,MMAP |HUGE_PAGE }, + {"mmap-shared" ,MMAP |LOCK |MMAP_SHARED }, + {"mmap-shared-remap" ,MMAP |LOCK |MMAP_SHARED |MREMAP }, + {"mmap-shared-remap-dontunmap" ,MMAP |LOCK |MMAP_SHARED | +MREMAP |DONTUNMAP }, + {"mmap-shared-remap-eocheck" ,MMAP |LOCK |MMAP_SHARED | +MREMAP |EVERY_OTHER_CHECK }, + {"mmap-shared-remap-dontunmap-eocheck" ,MMAP |LOCK | +MMAP_SHARED |MREMAP |DONTUNMAP |EVERY_OTHER_CHECK }, + {"mmap-mlock" ,MMAP |LOCK }, + {"mmap-file" ,MMAP |FILE_BACKED }, + {"mmap-file-mlock" ,MMAP |LOCK |FILE_BACKED }, + {"mmap-race" ,MMAP |RACE }, + {"free" ,NEW |FREE }, + {"free-race" ,NEW |FREE |RACE }, + {"new" ,NEW }, + {"new-race" ,NEW |RACE }, + {"new-bo-map" ,NEW |BO_MAP }, + {"new-busy" ,NEW |BUSY }, + {"mmap-free" ,MMAP |NEW |FREE }, + {"mmap-free-huge" ,MMAP |NEW |FREE |HUGE_PAGE }, + {"mmap-free-race" ,MMAP |NEW |FREE |RACE }, + {"mmap-new" ,MMAP |NEW }, + {"mmap-new-huge" ,MMAP |NEW |HUGE_PAGE }, + {"mmap-new-race" ,MMAP |NEW |RACE }, + {"malloc-nomemset" ,SKIP_MEMSET }, + {"malloc-mlock-nomemset" ,SKIP_MEMSET |LOCK }, + {"malloc-race-nomemset" ,SKIP_MEMSET |RACE }, + {"malloc-busy-nomemset" ,SKIP_MEMSET |BUSY }, + {"malloc-bo-unmap-nomemset" ,SKIP_MEMSET |BO_UNMAP }, + {"mmap-nomemset" ,SKIP_MEMSET |MMAP }, + {"mmap-huge-nomemset" ,SKIP_MEMSET |MMAP |HUGE_PAGE }, + {"mmap-shared-nomemset" ,SKIP_MEMSET |MMAP |MMAP_SHARED }, + {"mmap-mlock-nomemset" ,SKIP_MEMSET |MMAP |LOCK }, + {"mmap-file-nomemset" ,SKIP_MEMSET |MMAP |FILE_BACKED }, + {"mmap-file-mlock-nomemset" ,SKIP_MEMSET |MMAP |LOCK |FILE_BACKED }, + {"mmap-race-nomemset" ,SKIP_MEMSET |MMAP |RACE }, + {"free-nomemset" ,SKIP_MEMSET |NEW |FREE }, + {"free-race-nomemset" ,SKIP_MEMSET |NEW |FREE |RACE }, + {"new-nomemset" ,SKIP_MEMSET |NEW }, + {"new-race-nomemset" ,SKIP_MEMSET |NEW |RACE }, + {"new-bo-map-nomemset" ,SKIP_MEMSET |NEW |BO_MAP }, + {"new-busy-nomemset" ,SKIP_MEMSET |NEW |BUSY }, + {"mmap-free-nomemset" ,SKIP_MEMSET |MMAP |NEW |FREE }, + {"mmap-free-huge-nomemset" ,SKIP_MEMSET |MMAP |NEW |FREE |HUGE_PAGE }, + {"mmap-free-race-nomemset" ,SKIP_MEMSET |MMAP |NEW |FREE |RACE }, + {"mmap-new-nomemset" ,SKIP_MEMSET |MMAP |NEW }, + {"mmap-new-huge-nomemset" ,SKIP_MEMSET |MMAP |NEW |HUGE_PAGE }, + {"mmap-new-race-nomemset" ,SKIP_MEMSET |MMAP |NEW |RACE }, + {NULL }, + }; +const struct section psections[] = { + {"munmap-cpu-fault" ,CPU_FAULT }, + {"munmap-no-cpu-fault" ,0 }, + {"remap-cpu-fault" ,CPU_FAULT |REMAP }, + {"remap-no-cpu-fault" ,REMAP }, + {"middle-munmap-cpu-fault" ,MIDDLE |CPU_FAULT }, + {"middle-munmap-no-cpu-fault" ,MIDDLE }, + {"middle-remap-cpu-fault" ,MIDDLE |CPU_FAULT |REMAP }, + {"middle-remap-no-cpu-fault" ,MIDDLE |REMAP }, + {NULL }, + }; +const struct section esections[] = { + {"malloc" ,0 }, + {"malloc-mix-bo" ,MIX_BO_ALLOC }, + {NULL }, + }; +int fd; + + igt_fixture { +struct xe_device *xe; + + fd =drm_open_driver (DRIVER_XE ); +igt_require (!xe_supports_faults (fd)); + + xe =xe_device_get (fd); + va_bits = xe->va_bits; +open_sync_file (); + } + +for (const struct section *s = sections; s->name; s++) { + igt_subtest_f("once-%s", s->name) +xe_for_each_engine (fd, hwe) +test_exec (fd, hwe,1 ,1 ,0 ,0 ,0 ,NULL , +NULL , s->flags); + + igt_subtest_f("once-large-%s", s->name) +xe_for_each_engine (fd, hwe) +test_exec (fd, hwe,1 ,1 ,SZ_2M ,0 ,0 ,NULL , +NULL , s->flags); + + igt_subtest_f("twice-%s", s->name) +xe_for_each_engine (fd, hwe) +test_exec (fd, hwe,1 ,2 ,0 ,0 ,0 ,NULL , +NULL , s->flags); + + igt_subtest_f("twice-large-%s", s->name) +xe_for_each_engine (fd, hwe) +test_exec (fd, hwe,1 ,2 ,SZ_2M ,0 ,0 ,NULL , +NULL , s->flags); + + igt_subtest_f("many-%s", s->name) +xe_for_each_engine (fd, hwe) +test_exec (fd, hwe,1 ,128 ,0 ,0 ,0 ,NULL , +NULL , s->flags); + + igt_subtest_f("many-stride-%s", s->name) +xe_for_each_engine (fd, hwe) +test_exec (fd, hwe,1 ,128 ,0 ,256 ,0 ,NULL , +NULL , s->flags); + + igt_subtest_f("many-execqueues-%s", s->name) +xe_for_each_engine (fd, hwe) +test_exec (fd, hwe,16 ,128 ,0 ,0 ,0 ,NULL , +NULL , s->flags); + + igt_subtest_f("many-large-%s", s->name) +xe_for_each_engine (fd, hwe) +test_exec (fd, hwe,1 ,128 ,SZ_2M ,0 ,0 ,NULL , +NULL , s->flags); + + igt_subtest_f("many-large-execqueues-%s", s->name) +xe_for_each_engine (fd, hwe) +test_exec (fd, hwe,16 ,128 ,SZ_2M ,0 ,0 ,NULL , +NULL , s->flags); + + igt_subtest_f("threads-many-%s", s->name) +threads (fd,1 ,128 ,0 ,0 , s->flags, false); + + igt_subtest_f("threads-many-stride-%s", s->name) +threads (fd,1 ,128 ,0 ,256 , s->flags, false); + + igt_subtest_f("threads-many-execqueues-%s", s->name) +threads (fd,16 ,128 ,0 ,0 , s->flags, false); + + igt_subtest_f("threads-many-large-%s", s->name) +threads (fd,1 ,128 ,SZ_2M ,0 , s->flags, false); + + igt_subtest_f("threads-many-large-execqueues-%s", s->name) +threads (fd,16 ,128 ,SZ_2M ,0 , s->flags, false); + + igt_subtest_f("threads-shared-vm-many-%s", s->name) +threads (fd,1 ,128 ,0 ,0 , s->flags, true); + + igt_subtest_f("threads-shared-vm-many-stride-%s", s->name) +threads (fd,1 ,128 ,0 ,256 , s->flags, true); + + igt_subtest_f("threads-shared-vm-many-execqueues-%s", s->name) +threads (fd,16 ,128 ,0 ,0 , s->flags, true); + + igt_subtest_f("threads-shared-vm-many-large-%s", s->name) +threads (fd,1 ,128 ,SZ_2M ,0 , s->flags, true); + + igt_subtest_f("threads-shared-vm-many-large-execqueues-%s", s->name) +threads (fd,16 ,128 ,SZ_2M ,0 , s->flags, true); + + igt_subtest_f("process-many-%s", s->name) +processes (fd,1 ,128 ,0 ,0 , s->flags); + + igt_subtest_f("process-many-stride-%s", s->name) +processes (fd,1 ,128 ,0 ,256 , s->flags); + + igt_subtest_f("process-many-execqueues-%s", s->name) +processes (fd,16 ,128 ,0 ,0 , s->flags); + + igt_subtest_f("process-many-large-%s", s->name) +processes (fd,1 ,128 ,SZ_2M ,0 , s->flags); + + igt_subtest_f("process-many-large-execqueues-%s", s->name) +processes (fd,16 ,128 ,SZ_2M ,0 , s->flags); + } + + igt_subtest("threads-shared-vm-shared-alloc-many-stride-malloc") +threads (fd,1 ,128 ,0 ,256 ,SHARED_ALLOC , true); + + igt_subtest("threads-shared-vm-shared-alloc-many-stride-malloc-race") +threads (fd,1 ,128 ,0 ,256 ,RACE |SHARED_ALLOC , true); + + igt_subtest("threads-shared-alloc-many-stride-malloc") +threads (fd,1 ,128 ,0 ,256 ,SHARED_ALLOC , false); + + igt_subtest("threads-shared-alloc-many-stride-malloc-sync") +threads (fd,1 ,128 ,0 ,256 ,SHARED_ALLOC |SYNC_EXEC , false); + + igt_subtest("threads-shared-alloc-many-stride-malloc-race") +threads (fd,1 ,128 ,0 ,256 ,RACE |SHARED_ALLOC , false); + + igt_subtest_f("fault") +xe_for_each_engine (fd, hwe) +test_exec (fd, hwe,4 ,1 ,SZ_2M ,0 ,0 ,NULL ,NULL , +FAULT ); + +for (const struct section *s = psections; s->name; s++) { + igt_subtest_f("partial-%s", s->name) +xe_for_each_engine (fd, hwe) +partial (fd, hwe, s->flags); + } + + igt_subtest_f("unaligned-alloc") +xe_for_each_engine (fd, hwe) { +many_allocs (fd, hwe, (SZ_1M +SZ_512K ) *8 , +SZ_1M +SZ_512K ,SZ_4K ,NULL ,0 ); +break ; + } + + igt_subtest_f("fault-benchmark") +xe_for_each_engine (fd, hwe) +many_allocs (fd, hwe,SZ_64M ,SZ_64M ,SZ_4K ,NULL , +BENCHMARK ); + + igt_subtest_f("fault-threads-benchmark") +xe_for_each_engine (fd, hwe) +many_allocs (fd, hwe,SZ_64M ,SZ_64M ,SZ_4K ,NULL , +BENCHMARK |CPU_FAULT_THREADS ); + + igt_subtest_f("fault-threads-same-page-benchmark") +xe_for_each_engine (fd, hwe) +many_allocs (fd, hwe,SZ_64M ,SZ_64M ,SZ_4K ,NULL , +BENCHMARK |CPU_FAULT_THREADS | +CPU_FAULT_SAME_PAGE ); + + igt_subtest_f("fault-process-benchmark") +xe_for_each_engine (fd, hwe) +many_allocs (fd, hwe,SZ_64M ,SZ_64M ,SZ_4K ,NULL , +BENCHMARK |CPU_FAULT_PROCESS ); + + igt_subtest_f("fault-process-same-page-benchmark") +xe_for_each_engine (fd, hwe) +many_allocs (fd, hwe,SZ_64M ,SZ_64M ,SZ_4K ,NULL , +BENCHMARK |CPU_FAULT_PROCESS | +CPU_FAULT_SAME_PAGE ); + +for (const struct section *s = esections; s->name; s++) { + igt_subtest_f("evict-%s", s->name) +xe_for_each_engine (fd, hwe) { +many_allocs (fd, hwe, +xe_visible_vram_size (fd, hwe->gt_id), +SZ_8M ,SZ_1M ,NULL , s->flags); +break ; + } + } + +for (const struct section *s = esections; s->name; s++) { + igt_subtest_f("processes-evict-%s", s->name) +processes_evict (fd,SZ_8M ,SZ_1M , s->flags); + } + + igt_fixture { +xe_device_put (fd); +drm_close_driver (fd); +close_sync_file (); + } +}tests/meson.build +1
-0@@ -295,6 +295,7 @@ intel_xe_progs = [ 'xe_exec_reset', 'xe_exec_sip', 'xe_exec_store', + 'xe_exec_system_allocator', 'xe_exec_threads', 'xe_exercise_blt', 'xe_fault_injection',Signature · 1 line
2.34.1
Re: [PATCH v3 2/2] tests/xe: Add system_allocator test
In reply to Matthew BrostQuoted 40 lines · Matthew Brost
Test various uses of system allocator in single thread, multiple threads, and multiple processes.
Features tested:
- Malloc with various size
- Mmap with various sizes and flags including file backed mappings
- Mixing BO allocations with system allocator
- Various page sizes
- Dynamically freeing / unmapping memory
- Sharing VM across threads
- Faults racing on different hardware engines / GTs / Tiles
- GPU faults and CPU faults racing
- CPU faults on multiple threads racing
- CPU faults on multiple process racing
- GPU faults of memory not faulted in by CPU
- Partial unmap of allocations
- Attempting to unmap system allocations when GPU has mappings
- Eviction of both system allocations and BOs
- Forking child processes and reading data from VRAM
- mremap data in VRAM
- Protection changes
- Multiple faults per execbufRunning on LNL, BMG, PVC 1 tile, and PVC 2 tile.
v2:
- Rebase
- Fix memory allocation to not interfear with malloc (Thomas)
v3:
- Fix memory leak (Francois)
- Break out uAPI into own patch (Francois)
- Use mkstemp for sync file (Francois)
- Use mkstemp for file backed data (Francois)
- Drop i argument fromREAD_VALUE(Francois)
- Fix test description (Francois)
- Add comment tocheck_all_pages_process(Francois)
- Preferigt_infoover printf (Francois)
- Fix types in messages (Francois)
- Prefer odd macro (Francois)
Quoted 1 line
Signed-off-by Matthew Brost <[email protected]>
Earlier comments [1] are addressed, thanks.
[1] https://patchwork.freedesktop.org/patch/648841/?series=137545&rev=4#comment_1189490
Francois
Quoted 1,913 lines
--- lib/xe/xe_ioctl.c | 12 + lib/xe/xe_ioctl.h | 1 + tests/intel/xe_exec_system_allocator.c | 1849 ++++++++++++++++++++++++ tests/meson.build | 1 + 4 files changed, 1863 insertions(+) create mode 100644 tests/intel/xe_exec_system_allocator.clib/xe/xe_ioctl.c +12
-0@@ -440,6 +440,18 @@ void *xe_bo_map(int fd, uint32_t bo, size_t size)return __xe_bo_map (fd, bo, size,PROT_WRITE ); } +void *xe_bo_map_fixed (int fd,uint32_t bo,size_t size,uint64_t addr) +{ +uint64_t mmo; +void *map; + + mmo =xe_bo_mmap_offset (fd, bo); + map =mmap ((void *)addr, size,PROT_WRITE ,MAP_SHARED |MAP_FIXED , fd, mmo); +igt_assert (map !=MAP_FAILED ); + +return map; +} +void *xe_bo_mmap_ext (int fd,uint32_t bo,size_t size,int prot) {return __xe_bo_map (fd, bo, size, prot);lib/xe/xe_ioctl.h +1
-0@@ -86,6 +86,7 @@ uint32_t xe_exec_queue_create_class(int fd, uint32_t vm, uint16_t class);void xe_exec_queue_destroy (int fd,uint32_t exec_queue);uint64_t xe_bo_mmap_offset (int fd,uint32_t bo);void *xe_bo_map (int fd,uint32_t bo,size_t size); +void *xe_bo_map_fixed (int fd,uint32_t bo,size_t size,long unsigned int addr);void *xe_bo_mmap_ext (int fd,uint32_t bo,size_t size,int prot);int __xe_exec (int fd,struct drm_xe_exec *exec);void xe_exec (int fd,struct drm_xe_exec *exec);tests/intel/xe_exec_system_allocator.c new file +1849
-0@@ -0,0 +1,1849 @@ +// SPDX-License-Identifier: MIT +/* +* Copyright © 2024 Intel Corporation +*/ + +/** +* TEST: Basic tests for execbuf functionality using system allocator +* Category: Core +* Mega feature: USM +* Sub-category: System allocator +* Functionality: fault mode, system allocator +* GPU: LNL, BMG, PVC +*/ + +#include <fcntl.h> +#include <linux/mman.h> +#include <time.h> + +#include "igt.h" +#include "lib/igt_syncobj.h" +#include "lib/intel_reg.h" +#include "xe_drm.h" + +#include "xe/xe_ioctl.h" +#include "xe/xe_query.h" +#include <string.h> + +#define USER_FENCE_VALUE 0xdeadbeefdeadbeefull +#define QUARTER_SEC (NSEC_PER_SEC / 4) +#define FIVE_SEC (5LL * NSEC_PER_SEC) + +struct batch_data { +uint32_t batch[16 ]; +uint64_t pad; +uint32_t data; +uint32_t expected_data; +}; + +#define WRITE_VALUE (data__, i__) ({ \ + if (!(data__)->expected_data) \ + (data__)->expected_data = rand() << 12 | (i__); \ + (data__)->expected_data; \ +}) +#define READ_VALUE (data__) ((data__)->expected_data) + +static void __write_dword (uint32_t *batch,uint64_t sdi_addr,uint32_t wdata, +int *idx) +{ + batch[(*idx)++] =MI_STORE_DWORD_IMM_GEN4 ; + batch[(*idx)++] = sdi_addr; + batch[(*idx)++] = sdi_addr >>32 ; + batch[(*idx)++] = wdata; +} + +static void write_dword (uint32_t *batch,uint64_t sdi_addr,uint32_t wdata, +int *idx) +{ +__write_dword (batch, sdi_addr, wdata, idx); + batch[(*idx)++] =MI_BATCH_BUFFER_END ; +} + +static void check_all_pages (void *ptr,uint64_t alloc_size,uint64_t stride, +pthread_barrier_t *barrier) +{ +int i, n_writes = alloc_size / stride; + +for (i =0 ; i < n_writes; ++i) { +struct batch_data *data = ptr + i * stride; + +igt_assert_eq (data->data,READ_VALUE (data)); + +if (barrier) +pthread_barrier_wait (barrier); + } +} + +static char sync_file[] ="/tmp/xe_exec_system_allocator_syncXXXXXX" ; +static int sync_fd; + +static void open_sync_file (void ) +{ + sync_fd =mkstemp (sync_file); +} + +static void close_sync_file (void ) +{ +close (sync_fd); +} + +struct process_data { +pthread_mutex_t mutex; +pthread_cond_t cond; +pthread_barrier_t barrier; +bool go; +}; + +static void wait_pdata (struct process_data *pdata) +{ +pthread_mutex_lock (&pdata->mutex); +while (!pdata->go) +pthread_cond_wait (&pdata->cond, &pdata->mutex); +pthread_mutex_unlock (&pdata->mutex); +} + +static void init_pdata (struct process_data *pdata,int n_engine) +{ +pthread_mutexattr_t mutex_attr; +pthread_condattr_t cond_attr; +pthread_barrierattr_t barrier_attr; + +pthread_mutexattr_init (&mutex_attr); +pthread_mutexattr_setpshared (&mutex_attr,PTHREAD_PROCESS_SHARED ); +pthread_mutex_init (&pdata->mutex, &mutex_attr); + +pthread_condattr_init (&cond_attr); +pthread_condattr_setpshared (&cond_attr,PTHREAD_PROCESS_SHARED ); +pthread_cond_init (&pdata->cond, &cond_attr); + +pthread_barrierattr_init (&barrier_attr); +pthread_barrierattr_setpshared (&barrier_attr,PTHREAD_PROCESS_SHARED ); +pthread_barrier_init (&pdata->barrier, &barrier_attr, n_engine); + + pdata->go = false; +} + +static void signal_pdata (struct process_data *pdata) +{ +pthread_mutex_lock (&pdata->mutex); + pdata->go = true; +pthread_cond_broadcast (&pdata->cond); +pthread_mutex_unlock (&pdata->mutex); +} + +/* many_alloc flags */ +#define MIX_BO_ALLOC (0x1 << 0) +#define BENCHMARK (0x1 << 1) +#define CPU_FAULT_THREADS (0x1 << 2) +#define CPU_FAULT_PROCESS (0x1 << 3) +#define CPU_FAULT_SAME_PAGE (0x1 << 4) + +static void process_check (void *ptr,uint64_t alloc_size,uint64_t stride, +unsigned int flags) +{ +struct process_data *pdata; +int map_fd; + + map_fd =open (sync_file,O_RDWR ,0x666 ); + pdata =mmap (NULL ,sizeof (*pdata),PROT_READ | +PROT_WRITE ,MAP_SHARED , map_fd,0 ); +wait_pdata (pdata); + +if (flags &CPU_FAULT_SAME_PAGE ) +check_all_pages (ptr, alloc_size, stride, &pdata->barrier); +else +check_all_pages (ptr, alloc_size, stride,NULL ); + +close (map_fd); +munmap (pdata,sizeof (*pdata)); +} + +/* +* Partition checking of results in chunks which causes multiple processes to +* fault same VRAM allocation in parallel. +*/ +static void +check_all_pages_process (void *ptr,uint64_t alloc_size,uint64_t stride, +int n_process,unsigned int flags) +{ +struct process_data *pdata; +int map_fd, i; + + map_fd =open (sync_file,O_RDWR |O_CREAT ,0x666 ); +posix_fallocate (map_fd,0 ,sizeof (*pdata)); + pdata =mmap (NULL ,sizeof (*pdata),PROT_READ | +PROT_WRITE ,MAP_SHARED , map_fd,0 ); + +init_pdata (pdata, n_process); + +for (i =0 ; i < n_process; ++i) { + igt_fork(child, 1) +if (flags &CPU_FAULT_SAME_PAGE ) +process_check (ptr, alloc_size, stride, flags); +else +process_check (ptr + stride * i, alloc_size, + stride * n_process, flags); + } + +signal_pdata (pdata); +igt_waitchildren (); + +close (map_fd); +munmap (pdata,sizeof (*pdata)); +} + +struct thread_check_data { +pthread_t thread; +pthread_mutex_t *mutex; +pthread_cond_t *cond; +pthread_barrier_t *barrier; +void *ptr; +uint64_t alloc_size; +uint64_t stride; +bool *go; +}; + +static void *thread_check (void *data) +{ +struct thread_check_data *t = data; + +pthread_mutex_lock (t->mutex); +while (!*t->go) +pthread_cond_wait (t->cond, t->mutex); +pthread_mutex_unlock (t->mutex); + +check_all_pages (t->ptr, t->alloc_size, t->stride, t->barrier); + +return NULL ; +} + +/* +* Partition checking of results in chunks which causes multiple threads to +* fault same VRAM allocation in parallel. +*/ +static void +check_all_pages_threads (void *ptr,uint64_t alloc_size,uint64_t stride, +int n_threads,unsigned int flags) +{ +struct thread_check_data *threads_check_data; +pthread_mutex_t mutex; +pthread_cond_t cond; +pthread_barrier_t barrier; +int i; +bool go = false; + + threads_check_data =calloc (n_threads,sizeof (*threads_check_data)); +igt_assert (threads_check_data); + +pthread_mutex_init (&mutex,0 ); +pthread_cond_init (&cond,0 ); +pthread_barrier_init (&barrier,0 , n_threads); + +for (i =0 ; i < n_threads; ++i) { + threads_check_data[i].mutex = &mutex; + threads_check_data[i].cond = &cond; +if (flags &CPU_FAULT_SAME_PAGE ) { + threads_check_data[i].barrier = &barrier; + threads_check_data[i].ptr = ptr; + threads_check_data[i].alloc_size = alloc_size; + threads_check_data[i].stride = stride; + }else { + threads_check_data[i].barrier =NULL ; + threads_check_data[i].ptr = ptr + stride * i; + threads_check_data[i].alloc_size = alloc_size; + threads_check_data[i].stride = n_threads * stride; + } + threads_check_data[i].go = &go; + +pthread_create (&threads_check_data[i].thread,0 , thread_check, + &threads_check_data[i]); + } + +pthread_mutex_lock (&mutex); + go = true; +pthread_cond_broadcast (&cond); +pthread_mutex_unlock (&mutex); + +for (i =0 ; i < n_threads; ++i) +pthread_join (threads_check_data[i].thread,NULL ); +free (threads_check_data); +} + +static void touch_all_pages (int fd,uint32_t exec_queue,void *ptr, +uint64_t alloc_size,uint64_t stride, +struct timespec *tv,uint64_t *submit) +{ +struct drm_xe_sync sync[1 ] = { + { .type =DRM_XE_SYNC_TYPE_USER_FENCE , + .flags =DRM_XE_SYNC_FLAG_SIGNAL , + .timeline_value =USER_FENCE_VALUE }, + }; +struct drm_xe_exec exec = { + .num_batch_buffer =1 , + .num_syncs =0 , + .exec_queue_id = exec_queue, + .syncs =to_user_pointer (&sync), + }; +uint64_t addr =to_user_pointer (ptr); +int i, ret, n_writes = alloc_size / stride; +u64 *exec_ufence =NULL ; +int64_t timeout =FIVE_SEC ; + + exec_ufence =mmap (NULL ,SZ_4K ,PROT_READ | +PROT_WRITE ,MAP_SHARED | +MAP_ANONYMOUS ,-1 ,0 ); +igt_assert (exec_ufence !=MAP_FAILED ); +memset (exec_ufence,0 ,SZ_4K ); + sync[0 ].addr =to_user_pointer (exec_ufence); + +for (i =0 ; i < n_writes; ++i, addr += stride) { +struct batch_data *data = ptr + i * stride; +uint64_t sdi_offset = (char *)&data->data - (char *)data; +uint64_t sdi_addr = addr + sdi_offset; +int b =0 ; + +write_dword (data->batch, sdi_addr,WRITE_VALUE (data, i), &b); +igt_assert (b <=ARRAY_SIZE (data->batch)); + } + +igt_nsec_elapsed (tv); + *submit =igt_nsec_elapsed (tv); + + addr =to_user_pointer (ptr); +for (i =0 ; i < n_writes; ++i, addr += stride) { +struct batch_data *data = ptr + i * stride; +uint64_t batch_offset = (char *)&data->batch - (char *)data; +uint64_t batch_addr = addr + batch_offset; + + exec.address = batch_addr; +if (i +1 == n_writes) + exec.num_syncs =1 ; +xe_exec (fd, &exec); + } + + ret =__xe_wait_ufence (fd, exec_ufence,USER_FENCE_VALUE , exec_queue, + &timeout); +if (ret) { +igt_info ("FAIL EXEC_UFENCE_ADDR: 0x%016llx\n" , sync[0 ].addr); +igt_info ("FAIL EXEC_UFENCE: EXPECTED=0x%016llx, ACTUAL=0x%016lx\n" , +USER_FENCE_VALUE , exec_ufence[0 ]); + + addr =to_user_pointer (ptr); +for (i =0 ; i < n_writes; ++i, addr += stride) { +struct batch_data *data = ptr + i * stride; +uint64_t batch_offset = (char *)&data->batch - (char *)data; +uint64_t batch_addr = addr + batch_offset; +uint64_t sdi_offset = (char *)&data->data - (char *)data; +uint64_t sdi_addr = addr + sdi_offset; + +igt_info ("FAIL BATCH_ADDR: 0x%016lx\n" , batch_addr); +igt_info ("FAIL SDI_ADDR: 0x%016lx\n" , sdi_addr); +igt_info ("FAIL SDI_ADDR (in batch): 0x%016lx\n" , + (((u64 )data->batch[2 ]) <<32 ) | data->batch[1 ]); +igt_info ("FAIL DATA: EXPECTED=0x%08x, ACTUAL=0x%08x\n" , + data->expected_data, data->data); + } +igt_assert_eq (ret,0 ); + } +munmap (exec_ufence,SZ_4K ); +} + +static int va_bits; + +#define bind_system_allocator (__sync, __num_sync) \ + __xe_vm_bind_assert(fd, vm, 0, \ + 0, 0, 0, 0x1ull << va_bits, \ + DRM_XE_VM_BIND_OP_MAP, \ + DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR, \ + (__sync), (__num_sync), 0, 0) + +#define unbind_system_allocator () \ + __xe_vm_bind(fd, vm, 0, 0, 0, 0, 0x1ull << va_bits, \ + DRM_XE_VM_BIND_OP_UNMAP, 0, \ + NULL, 0, 0, 0, 0) + +#define odd (__i) (__i & 1) + +struct aligned_alloc_type { +void *__ptr; +void *ptr; +size_t __size; +size_t size; +}; + +static struct aligned_alloc_type __aligned_alloc (size_t alignment,size_t size) +{ +struct aligned_alloc_type aligned_alloc_type; + + aligned_alloc_type.__ptr =mmap (NULL , alignment + size,PROT_NONE ,MAP_PRIVATE | +MAP_ANONYMOUS ,-1 ,0 ); +igt_assert (aligned_alloc_type.__ptr !=MAP_FAILED ); + + aligned_alloc_type.ptr = (void *)ALIGN ((uint64_t )aligned_alloc_type.__ptr, alignment); + aligned_alloc_type.size = size; + aligned_alloc_type.__size = size + alignment; + +return aligned_alloc_type; +} + +static void __aligned_free (struct aligned_alloc_type *aligned_alloc_type) +{ +munmap (aligned_alloc_type->__ptr, aligned_alloc_type->__size); +} + +static void __aligned_partial_free (struct aligned_alloc_type *aligned_alloc_type) +{ +size_t begin_size = (size_t )(aligned_alloc_type->ptr - aligned_alloc_type->__ptr); + +if (begin_size) +munmap (aligned_alloc_type->__ptr, begin_size); +if (aligned_alloc_type->__size - aligned_alloc_type->size - begin_size) +munmap (aligned_alloc_type->ptr + aligned_alloc_type->size, + aligned_alloc_type->__size - aligned_alloc_type->size - begin_size); +} + +/** +* SUBTEST: unaligned-alloc +* Description: allocate unaligned sizes of memory +* Test category: functionality test +* +* SUBTEST: fault-benchmark +* Description: Benchmark how long GPU / CPU take +* Test category: performance test +* +* SUBTEST: fault-threads-benchmark +* Description: Benchmark how long GPU / CPU take, reading results with multiple threads +* Test category: performance and functionality test +* +* SUBTEST: fault-threads-same-page-benchmark +* Description: Benchmark how long GPU / CPU take, reading results with multiple threads, hammer same page +* Test category: performance and functionality test +* +* SUBTEST: fault-process-benchmark +* Description: Benchmark how long GPU / CPU take, reading results with multiple process +* Test category: performance and functionality test +* +* SUBTEST: fault-process-same-page-benchmark +* Description: Benchmark how long GPU / CPU take, reading results with multiple process, hammer same page +* Test category: performance and functionality test +* +* SUBTEST: evict-malloc +* Description: trigger eviction of VRAM allocated via malloc +* Test category: functionality test +* +* SUBTEST: evict-malloc-mix-bo +* Description: trigger eviction of VRAM allocated via malloc and BO create +* Test category: functionality test +* +* SUBTEST: processes-evict-malloc +* Description: multi-process trigger eviction of VRAM allocated via malloc +* Test category: stress test +* +* SUBTEST: processes-evict-malloc-mix-bo +* Description: multi-process trigger eviction of VRAM allocated via malloc and BO create +* Test category: stress test +*/ + +static void +many_allocs (int fd,struct drm_xe_engine_class_instance *eci, +uint64_t total_alloc,uint64_t alloc_size,uint64_t stride, +pthread_barrier_t *barrier,unsigned int flags) +{ +uint32_t vm, exec_queue; +int num_allocs = flags &BENCHMARK ?1 : + (9 * (total_alloc / alloc_size)) /8 ; +struct aligned_alloc_type *allocs; +uint32_t *bos =NULL ; +struct timespec tv = {}; +uint64_t submit, read, elapsed; +int i; + + vm =xe_vm_create (fd,DRM_XE_VM_CREATE_FLAG_LR_MODE | +DRM_XE_VM_CREATE_FLAG_FAULT_MODE ,0 ); + exec_queue =xe_exec_queue_create (fd, vm, eci,0 ); + +bind_system_allocator (NULL ,0 ); + + allocs =malloc (sizeof (*allocs) * num_allocs); +igt_assert (allocs); +memset (allocs,0 ,sizeof (*allocs) * num_allocs); + +if (flags &MIX_BO_ALLOC ) { + bos =malloc (sizeof (*bos) * num_allocs); +igt_assert (bos); +memset (bos,0 ,sizeof (*bos) * num_allocs); + } + +for (i =0 ; i < num_allocs; ++i) { +struct aligned_alloc_type alloc; + +if (flags &MIX_BO_ALLOC &&odd (i)) { +uint32_t bo_flags = +DRM_XE_GEM_CREATE_FLAG_NEEDS_VISIBLE_VRAM ; + + alloc =__aligned_alloc (SZ_2M , alloc_size); +igt_assert (alloc.ptr); + + bos[i] =xe_bo_create (fd, vm, alloc_size, +vram_if_possible (fd, eci->gt_id), + bo_flags); + alloc.ptr =xe_bo_map_fixed (fd, bos[i], alloc_size, +to_user_pointer (alloc.ptr)); +xe_vm_bind_async (fd, vm,0 , bos[i],0 , +to_user_pointer (alloc.ptr), + alloc_size,0 ,0 ); + }else { + alloc.ptr =aligned_alloc (SZ_2M , alloc_size); +igt_assert (alloc.ptr); + } + allocs[i] = alloc; + +touch_all_pages (fd, exec_queue, allocs[i].ptr, alloc_size, stride, + &tv, &submit); + } + +if (barrier) +pthread_barrier_wait (barrier); + +for (i =0 ; i < num_allocs; ++i) { +if (flags &BENCHMARK ) + read =igt_nsec_elapsed (&tv); +#define NUM_CHECK_THREADS 8 +if (flags &CPU_FAULT_PROCESS ) +check_all_pages_process (allocs[i].ptr, alloc_size, stride, +NUM_CHECK_THREADS , flags); +else if (flags &CPU_FAULT_THREADS ) +check_all_pages_threads (allocs[i].ptr, alloc_size, stride, +NUM_CHECK_THREADS , flags); +else +check_all_pages (allocs[i].ptr, alloc_size, stride,NULL ); +if (flags &BENCHMARK ) { + elapsed =igt_nsec_elapsed (&tv); +igt_info ("Execution took %.3fms (submit %.1fus, read %.1fus, total %.1fus, read_total %.1fus)\n" , +1e-6 * elapsed,1e-3 * submit,1e-3 * read, +1e-3 * (elapsed - submit), +1e-3 * (elapsed - read)); + } +if (bos && bos[i]) { +__aligned_free (allocs + i); +gem_close (fd, bos[i]); + }else { +free (allocs[i].ptr); + } + } +if (bos) +free (bos); +free (allocs); +xe_exec_queue_destroy (fd, exec_queue); +xe_vm_destroy (fd, vm); +} + +static void process_evict (struct drm_xe_engine_class_instance *hwe, +uint64_t total_alloc,uint64_t alloc_size, +uint64_t stride,unsigned int flags) +{ +struct process_data *pdata; +int map_fd; +int fd; + + map_fd =open (sync_file,O_RDWR ,0x666 ); + pdata =mmap (NULL ,sizeof (*pdata),PROT_READ | +PROT_WRITE ,MAP_SHARED , map_fd,0 ); +wait_pdata (pdata); + + fd =drm_open_driver (DRIVER_XE ); +many_allocs (fd, hwe, total_alloc, alloc_size, stride, &pdata->barrier, + flags); +drm_close_driver (fd); + +close (map_fd); +munmap (pdata,sizeof (*pdata)); +} + +static void +processes_evict (int fd,uint64_t alloc_size,uint64_t stride, +unsigned int flags) +{ +struct drm_xe_engine_class_instance *hwe; +struct process_data *pdata; +int n_engine_gt[2 ] = {0 ,0 }, n_engine =0 ; +int map_fd; + + map_fd =open (sync_file,O_RDWR |O_CREAT ,0x666 ); +posix_fallocate (map_fd,0 ,sizeof (*pdata)); + pdata =mmap (NULL ,sizeof (*pdata),PROT_READ | +PROT_WRITE ,MAP_SHARED , map_fd,0 ); + +xe_for_each_engine (fd, hwe) { +igt_assert (hwe->gt_id <2 ); + n_engine_gt[hwe->gt_id]++; + n_engine++; + } + +init_pdata (pdata, n_engine); + +xe_for_each_engine (fd, hwe) { + igt_fork(child, 1) +process_evict (hwe, +xe_visible_vram_size (fd, hwe->gt_id) / + n_engine_gt[hwe->gt_id], alloc_size, + stride, flags); + } + +signal_pdata (pdata); +igt_waitchildren (); + +close (map_fd); +munmap (pdata,sizeof (*pdata)); +} + +#define CPU_FAULT (0x1 << 0) +#define REMAP (0x1 << 1) +#define MIDDLE (0x1 << 2) + +/** +* SUBTEST: partial-munmap-cpu-fault +* Description: munmap partially with cpu access in between +* Test category: functionality test +* +* SUBTEST: partial-munmap-no-cpu-fault +* Description: munmap partially with no cpu access in between +* Test category: functionality test +* +* SUBTEST: partial-remap-cpu-fault +* Description: remap partially with cpu access in between +* Test category: functionality test +* +* SUBTEST: partial-remap-no-cpu-fault +* Description: remap partially with no cpu access in between +* Test category: functionality test +* +* SUBTEST: partial-middle-munmap-cpu-fault +* Description: munmap middle with cpu access in between +* Test category: functionality test +* +* SUBTEST: partial-middle-munmap-no-cpu-fault +* Description: munmap middle with no cpu access in between +* Test category: functionality test +* +* SUBTEST: partial-middle-remap-cpu-fault +* Description: remap middle with cpu access in between +* Test category: functionality test +* +* SUBTEST: partial-middle-remap-no-cpu-fault +* Description: remap middle with no cpu access in between +* Test category: functionality test +*/ + +static void +partial (int fd,struct drm_xe_engine_class_instance *eci,unsigned int flags) +{ +struct drm_xe_sync sync[1 ] = { + { .type =DRM_XE_SYNC_TYPE_USER_FENCE , .flags =DRM_XE_SYNC_FLAG_SIGNAL , + .timeline_value =USER_FENCE_VALUE }, + }; +struct drm_xe_exec exec = { + .num_batch_buffer =1 , + .num_syncs =1 , + .syncs =to_user_pointer (sync), + }; +struct { +uint32_t batch[16 ]; +uint64_t pad; +uint64_t vm_sync; +uint64_t exec_sync; +uint32_t data; +uint32_t expected_data; + } *data; +size_t bo_size =SZ_2M , unmap_offset =0 ; +uint32_t vm, exec_queue; +u64 *exec_ufence =NULL ; +int i; +void *old, *new =NULL ; +struct aligned_alloc_type alloc; + +if (flags &MIDDLE ) + unmap_offset = bo_size /4 ; + + vm =xe_vm_create (fd,DRM_XE_VM_CREATE_FLAG_LR_MODE | +DRM_XE_VM_CREATE_FLAG_FAULT_MODE ,0 ); + + alloc =__aligned_alloc (bo_size, bo_size); +igt_assert (alloc.ptr); + + data =mmap (alloc.ptr, bo_size,PROT_READ |PROT_WRITE , +MAP_PRIVATE |MAP_ANONYMOUS |MAP_FIXED ,-1 ,0 ); +igt_assert (data !=MAP_FAILED ); +memset (data,0 , bo_size); + old = data; + + exec_queue =xe_exec_queue_create (fd, vm, eci,0 ); + + sync[0 ].addr =to_user_pointer (&data[0 ].vm_sync); +bind_system_allocator (sync,1 ); +xe_wait_ufence (fd, &data[0 ].vm_sync,USER_FENCE_VALUE ,0 ,FIVE_SEC ); + data[0 ].vm_sync =0 ; + + exec_ufence =mmap (NULL ,SZ_4K ,PROT_READ | +PROT_WRITE ,MAP_SHARED | +MAP_ANONYMOUS ,-1 ,0 ); +igt_assert (exec_ufence !=MAP_FAILED ); +memset (exec_ufence,0 ,SZ_4K ); + +for (i =0 ; i <2 ; i++) { +uint64_t addr =to_user_pointer (data); +uint64_t sdi_offset = (char *)&data[i].data - (char *)data; +uint64_t sdi_addr = addr + sdi_offset; +int b =0 ; + +write_dword (data[i].batch, sdi_addr,WRITE_VALUE (&data[i], i), &b); +igt_assert (b <=ARRAY_SIZE (data[i].batch)); + +if (!i) + data = old + unmap_offset + bo_size /2 ; + } + + data = old; + exec.exec_queue_id = exec_queue; + +for (i =0 ; i <2 ; i++) { +uint64_t addr =to_user_pointer (data); +uint64_t batch_offset = (char *)&data[i].batch - (char *)data; +uint64_t batch_addr = addr + batch_offset; + + sync[0 ].addr = new ?to_user_pointer (new) : +to_user_pointer (exec_ufence); + exec.address = batch_addr; +xe_exec (fd, &exec); + +xe_wait_ufence (fd, new ?: exec_ufence,USER_FENCE_VALUE , + exec_queue,FIVE_SEC ); +if (i || (flags &CPU_FAULT )) +igt_assert_eq (data[i].data,READ_VALUE (&data[i])); + exec_ufence[0 ] =0 ; + +if (!i) { + data = old + unmap_offset + bo_size /2 ; +munmap (old + unmap_offset, bo_size /2 ); +if (flags &REMAP ) { + new =mmap (old + unmap_offset, bo_size /2 , +PROT_READ |PROT_WRITE , +MAP_PRIVATE |MAP_ANONYMOUS |MAP_FIXED | +MAP_LOCKED ,-1 ,0 ); +igt_assert (new !=MAP_FAILED ); + } + } + } + +xe_exec_queue_destroy (fd, exec_queue); +munmap (exec_ufence,SZ_4K ); +__aligned_free (&alloc); +if (new) +munmap (new, bo_size /2 ); +xe_vm_destroy (fd, vm); +} + +#define MAX_N_EXEC_QUEUES 16 + +#define MMAP (0x1 << 0) +#define NEW (0x1 << 1) +#define BO_UNMAP (0x1 << 2) +#define FREE (0x1 << 3) +#define BUSY (0x1 << 4) +#define BO_MAP (0x1 << 5) +#define RACE (0x1 << 6) +#define SKIP_MEMSET (0x1 << 7) +#define FAULT (0x1 << 8) +#define FILE_BACKED (0x1 << 9) +#define LOCK (0x1 << 10) +#define MMAP_SHARED (0x1 << 11) +#define HUGE_PAGE (0x1 << 12) +#define SHARED_ALLOC (0x1 << 13) +#define FORK_READ (0x1 << 14) +#define FORK_READ_AFTER (0x1 << 15) +#define MREMAP (0x1 << 16) +#define DONTUNMAP (0x1 << 17) +#define READ_ONLY_REMAP (0x1 << 18) +#define SYNC_EXEC (0x1 << 19) +#define EVERY_OTHER_CHECK (0x1 << 20) +#define MULTI_FAULT (0x1 << 21) + +#define N_MULTI_FAULT 4 + +/** +* SUBTEST: once-%s +* Description: Run %arg[1] system allocator test only once +* Test category: functionality test +* +* SUBTEST: once-large-%s +* Description: Run %arg[1] system allocator test only once with large allocation +* Test category: functionality test +* +* SUBTEST: twice-%s +* Description: Run %arg[1] system allocator test twice +* Test category: functionality test +* +* SUBTEST: twice-large-%s +* Description: Run %arg[1] system allocator test twice with large allocation +* Test category: functionality test +* +* SUBTEST: many-%s +* Description: Run %arg[1] system allocator test many times +* Test category: stress test +* +* SUBTEST: many-stride-%s +* Description: Run %arg[1] system allocator test many times with a stride on each exec +* Test category: stress test +* +* SUBTEST: many-execqueues-%s +* Description: Run %arg[1] system allocator test on many exec_queues +* Test category: stress test +* +* SUBTEST: many-large-%s +* Description: Run %arg[1] system allocator test many times with large allocations +* Test category: stress test +* +* SUBTEST: many-large-execqueues-%s +* Description: Run %arg[1] system allocator test on many exec_queues with large allocations +* +* SUBTEST: threads-many-%s +* Description: Run %arg[1] system allocator threaded test many times +* Test category: stress test +* +* SUBTEST: threads-many-stride-%s +* Description: Run %arg[1] system allocator threaded test many times with a stride on each exec +* Test category: stress test +* +* SUBTEST: threads-many-execqueues-%s +* Description: Run %arg[1] system allocator threaded test on many exec_queues +* Test category: stress test +* +* SUBTEST: threads-many-large-%s +* Description: Run %arg[1] system allocator threaded test many times with large allocations +* Test category: stress test +* +* SUBTEST: threads-many-large-execqueues-%s +* Description: Run %arg[1] system allocator threaded test on many exec_queues with large allocations +* +* SUBTEST: threads-shared-vm-many-%s +* Description: Run %arg[1] system allocator threaded, shared vm test many times +* Test category: stress test +* +* SUBTEST: threads-shared-vm-many-stride-%s +* Description: Run %arg[1] system allocator threaded, shared vm test many times with a stride on each exec +* Test category: stress test +* +* SUBTEST: threads-shared-vm-many-execqueues-%s +* Description: Run %arg[1] system allocator threaded, shared vm test on many exec_queues +* Test category: stress test +* +* SUBTEST: threads-shared-vm-many-large-%s +* Description: Run %arg[1] system allocator threaded, shared vm test many times with large allocations +* Test category: stress test +* +* SUBTEST: threads-shared-vm-many-large-execqueues-%s +* Description: Run %arg[1] system allocator threaded, shared vm test on many exec_queues with large allocations +* Test category: stress test +* +* SUBTEST: process-many-%s +* Description: Run %arg[1] system allocator multi-process test many times +* Test category: stress test +* +* SUBTEST: process-many-stride-%s +* Description: Run %arg[1] system allocator multi-process test many times with a stride on each exec +* Test category: stress test +* +* SUBTEST: process-many-execqueues-%s +* Description: Run %arg[1] system allocator multi-process test on many exec_queues +* Test category: stress test +* +* SUBTEST: process-many-large-%s +* Description: Run %arg[1] system allocator multi-process test many times with large allocations +* Test category: stress test +* +* SUBTEST: process-many-large-execqueues-%s +* Description: Run %arg[1] system allocator multi-process test on many exec_queues with large allocations +* +* SUBTEST: fault +* Description: use a bad system allocator address resulting in a fault +* Test category: bad input +* +* arg[1]: +* +* @malloc: malloc single buffer for all execs, issue a command which will trigger multiple faults +* @malloc-multi-fault: malloc single buffer for all execs +* @malloc-fork-read: malloc single buffer for all execs, fork a process to read test output +* @malloc-fork-read-after: malloc single buffer for all execs, fork a process to read test output, check again after fork returns in parent +* @malloc-mlock: malloc and mlock single buffer for all execs +* @malloc-race: malloc single buffer for all execs with race between cpu and gpu access +* @malloc-bo-unmap: malloc single buffer for all execs, bind and unbind a BO to same address before execs +* @malloc-busy: malloc single buffer for all execs, try to unbind while buffer valid +* @mmap: mmap single buffer for all execs +* @mmap-remap: mmap and mremap a buffer for all execs +* @mmap-remap-dontunmap: mmap and mremap a buffer with dontunmap flag for all execs +* @mmap-remap-ro: mmap and mremap a read-only buffer for all execs +* @mmap-remap-ro-dontunmap: mmap and mremap a read-only buffer with dontunmap flag for all execs +* @mmap-remap-eocheck: mmap and mremap a buffer for all execs, check data every other loop iteration +* @mmap-remap-dontunmap-eocheck: mmap and mremap a buffer with dontunmap flag for all execs, check data every other loop iteration +* @mmap-remap-ro-eocheck: mmap and mremap a read-only buffer for all execs, check data every other loop iteration +* @mmap-remap-ro-dontunmap-eocheck: mmap and mremap a read-only buffer with dontunmap flag for all execs, check data every other loop iteration +* @mmap-huge: mmap huge page single buffer for all execs +* @mmap-shared: mmap shared single buffer for all execs +* @mmap-shared-remap: mmap shared and mremap a buffer for all execs +* @mmap-shared-remap-dontunmap: mmap shared and mremap a buffer with dontunmap flag for all execs +* @mmap-shared-remap-eocheck: mmap shared and mremap a buffer for all execs, check data every other loop iteration +* @mmap-shared-remap-dontunmap-eocheck: mmap shared and mremap a buffer with dontunmap flag for all execs, check data every other loop iteration +* @mmap-mlock: mmap and mlock single buffer for all execs +* @mmap-file: mmap single buffer, with file backing, for all execs +* @mmap-file-mlock: mmap and mlock single buffer, with file backing, for all execs +* @mmap-race: mmap single buffer for all execs with race between cpu and gpu access +* @free: malloc and free buffer for each exec +* @free-race: malloc and free buffer for each exec with race between cpu and gpu access +* @new: malloc a new buffer for each exec +* @new-race: malloc a new buffer for each exec with race between cpu and gpu access +* @new-bo-map: malloc a new buffer or map BO for each exec +* @new-busy: malloc a new buffer for each exec, try to unbind while buffers valid +* @mmap-free: mmap and free buffer for each exec +* @mmap-free-huge: mmap huge page and free buffer for each exec +* @mmap-free-race: mmap and free buffer for each exec with race between cpu and gpu access +* @mmap-new: mmap a new buffer for each exec +* @mmap-new-huge: mmap huge page a new buffer for each exec +* @mmap-new-race: mmap a new buffer for each exec with race between cpu and gpu access +* @malloc-nomemset: malloc single buffer for all execs, skip memset of buffers +* @malloc-mlock-nomemset: malloc and mlock single buffer for all execs, skip memset of buffers +* @malloc-race-nomemset: malloc single buffer for all execs with race between cpu and gpu access, skip memset of buffers +* @malloc-bo-unmap-nomemset: malloc single buffer for all execs, bind and unbind a BO to same address before execs, skip memset of buffers +* @malloc-busy-nomemset: malloc single buffer for all execs, try to unbind while buffer valid, skip memset of buffers +* @mmap-nomemset: mmap single buffer for all execs, skip memset of buffers +* @mmap-huge-nomemset: mmap huge page single buffer for all execs, skip memset of buffers +* @mmap-shared-nomemset: mmap shared single buffer for all execs, skip memset of buffers +* @mmap-mlock-nomemset: mmap and mlock single buffer for all execs, skip memset of buffers +* @mmap-file-nomemset: mmap single buffer, with file backing, for all execs, skip memset of buffers +* @mmap-file-mlock-nomemset: mmap and mlock single buffer, with file backing, for all execs, skip memset of buffers +* @mmap-race-nomemset: mmap single buffer for all execs with race between cpu and gpu access, skip memset of buffers +* @free-nomemset: malloc and free buffer for each exec, skip memset of buffers +* @free-race-nomemset: malloc and free buffer for each exec with race between cpu and gpu access, skip memset of buffers +* @new-nomemset: malloc a new buffer for each exec, skip memset of buffers +* @new-race-nomemset: malloc a new buffer for each exec with race between cpu and gpu access, skip memset of buffers +* @new-bo-map-nomemset: malloc a new buffer or map BO for each exec, skip memset of buffers +* @new-busy-nomemset: malloc a new buffer for each exec, try to unbind while buffers valid, skip memset of buffers +* @mmap-free-nomemset: mmap and free buffer for each exec, skip memset of buffers +* @mmap-free-huge-nomemset: mmap huge page and free buffer for each exec, skip memset of buffers +* @mmap-free-race-nomemset: mmap and free buffer for each exec with race between cpu and gpu access, skip memset of buffers +* @mmap-new-nomemset: mmap a new buffer for each exec, skip memset of buffers +* @mmap-new-huge-nomemset: mmap huge page new buffer for each exec, skip memset of buffers +* @mmap-new-race-nomemset: mmap a new buffer for each exec with race between cpu and gpu access, skip memset of buffers +* +* SUBTEST: threads-shared-vm-shared-alloc-many-stride-malloc +* Description: Create multiple threads with a shared VM triggering faults on different hardware engines to same addresses +* Test category: stress test +* +* SUBTEST: threads-shared-vm-shared-alloc-many-stride-malloc-race +* Description: Create multiple threads with a shared VM triggering faults on different hardware engines to same addresses, racing between CPU and GPU access +* Test category: stress test +* +* SUBTEST: threads-shared-alloc-many-stride-malloc +* Description: Create multiple threads with a faults on different hardware engines to same addresses +* Test category: stress test +* +* SUBTEST: threads-shared-alloc-many-stride-malloc-sync +* Description: Create multiple threads with a faults on different hardware engines to same addresses, syncing on each exec +* Test category: stress test +* +* SUBTEST: threads-shared-alloc-many-stride-malloc-race +* Description: Create multiple threads with a faults on different hardware engines to same addresses, racing between CPU and GPU access +* Test category: stress test +*/ + +struct test_exec_data { +uint32_t batch[32 ]; +uint64_t pad; +uint64_t vm_sync; +uint64_t exec_sync; +uint32_t data; +uint32_t expected_data; +}; + +static void +test_exec (int fd,struct drm_xe_engine_class_instance *eci, +int n_exec_queues,int n_execs,size_t bo_size, +size_t stride,uint32_t vm,void *alloc,pthread_barrier_t *barrier, +unsigned int flags) +{ +uint64_t addr; +struct drm_xe_sync sync[1 ] = { + { .type =DRM_XE_SYNC_TYPE_USER_FENCE , .flags =DRM_XE_SYNC_FLAG_SIGNAL , + .timeline_value =USER_FENCE_VALUE }, + }; +struct drm_xe_exec exec = { + .num_batch_buffer =1 , + .num_syncs =1 , + .syncs =to_user_pointer (sync), + }; +uint32_t exec_queues[MAX_N_EXEC_QUEUES ]; +struct test_exec_data *data, *next_data =NULL ; +uint32_t bo_flags; +uint32_t bo =0 ; +void **pending_free; +u64 *exec_ufence =NULL ; +int i, j, b, file_fd =-1 , prev_idx; +bool free_vm = false; +size_t aligned_size = bo_size ?:xe_get_default_alignment (fd); +size_t orig_size = bo_size; +struct aligned_alloc_type aligned_alloc_type; + +if (flags &MULTI_FAULT ) { +if (!bo_size) +return ; + + bo_size *=N_MULTI_FAULT ; + } + +if (flags &SHARED_ALLOC ) +return ; + +if (flags &EVERY_OTHER_CHECK &&odd (n_execs)) +return ; + +if (flags &EVERY_OTHER_CHECK ) +igt_assert (flags &MREMAP ); + +igt_assert (n_exec_queues <=MAX_N_EXEC_QUEUES ); + +if (flags &NEW && !(flags &FREE )) { + pending_free =malloc (sizeof (*pending_free) * n_execs); +igt_assert (pending_free); +memset (pending_free,0 ,sizeof (*pending_free) * n_execs); + } + +if (!vm) { + vm =xe_vm_create (fd,DRM_XE_VM_CREATE_FLAG_LR_MODE | +DRM_XE_VM_CREATE_FLAG_FAULT_MODE ,0 ); + free_vm = true; + } +if (!bo_size) { +if (!stride) { + bo_size =sizeof (*data) * n_execs; + bo_size =xe_bb_size (fd, bo_size); + }else { + bo_size = stride * n_execs *sizeof (*data); + bo_size =xe_bb_size (fd, bo_size); + } + } +if (flags &HUGE_PAGE ) { + aligned_size =ALIGN (aligned_size,SZ_2M ); + bo_size =ALIGN (bo_size,SZ_2M ); + } + +if (alloc) { + data = alloc; + }else { +if (flags &MMAP ) { +int mmap_flags =MAP_FIXED ; + + aligned_alloc_type =__aligned_alloc (aligned_size, bo_size); + data = aligned_alloc_type.ptr; +igt_assert (data); +__aligned_partial_free (&aligned_alloc_type); + +if (flags &MMAP_SHARED ) + mmap_flags |=MAP_SHARED ; +else + mmap_flags |=MAP_PRIVATE ; + +if (flags &HUGE_PAGE ) + mmap_flags |=MAP_HUGETLB |MAP_HUGE_2MB ; + +if (flags &FILE_BACKED ) { +char name[] ="/tmp/xe_exec_system_allocator_datXXXXXX" ; + +igt_assert (!(flags &NEW )); + + file_fd =mkstemp (name); +posix_fallocate (file_fd,0 , bo_size); + }else { + mmap_flags |=MAP_ANONYMOUS ; + } + + data =mmap (data, bo_size,PROT_READ | +PROT_WRITE , mmap_flags, file_fd,0 ); +igt_assert (data !=MAP_FAILED ); + }else { + data =aligned_alloc (aligned_size, bo_size); +igt_assert (data); + } +if (!(flags &SKIP_MEMSET )) +memset (data,0 , bo_size); +if (flags &LOCK ) { +igt_assert (!(flags &NEW )); +mlock (data, bo_size); + } + } + +for (i =0 ; i < n_exec_queues; i++) + exec_queues[i] =xe_exec_queue_create (fd, vm, eci,0 ); + + sync[0 ].addr =to_user_pointer (&data[0 ].vm_sync); +if (free_vm) { +bind_system_allocator (sync,1 ); +xe_wait_ufence (fd, &data[0 ].vm_sync,USER_FENCE_VALUE ,0 ,FIVE_SEC ); + } + data[0 ].vm_sync =0 ; + + addr =to_user_pointer (data); + +if (flags &BO_UNMAP ) { + bo_flags =DRM_XE_GEM_CREATE_FLAG_NEEDS_VISIBLE_VRAM ; + bo =xe_bo_create (fd, vm, bo_size, +vram_if_possible (fd, eci->gt_id), bo_flags); +xe_vm_bind_async (fd, vm,0 , bo,0 , addr, bo_size,0 ,0 ); + +__xe_vm_bind_assert (fd, vm,0 , +0 ,0 , addr, bo_size, +DRM_XE_VM_BIND_OP_MAP , +DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR , sync, +1 ,0 ,0 ); +xe_wait_ufence (fd, &data[0 ].vm_sync,USER_FENCE_VALUE ,0 , +FIVE_SEC ); + data[0 ].vm_sync =0 ; +gem_close (fd, bo); + bo =0 ; + } + +if (!(flags &RACE )) { + exec_ufence =mmap (NULL ,SZ_4K ,PROT_READ | +PROT_WRITE ,MAP_SHARED | +MAP_ANONYMOUS ,-1 ,0 ); +igt_assert (exec_ufence !=MAP_FAILED ); +memset (exec_ufence,0 ,SZ_4K ); + } + +for (i =0 ; i < n_execs; i++) { +int idx = !stride ? i : i * stride, next_idx = !stride + ? (i +1 ) : (i +1 ) * stride; +uint64_t batch_offset = (char *)&data[idx].batch - (char *)data; +uint64_t batch_addr = addr + batch_offset; +uint64_t sdi_offset = (char *)&data[idx].data - (char *)data; +uint64_t sdi_addr = addr + sdi_offset; +int e = i % n_exec_queues, err; +bool fault_inject = (FAULT & flags) && i == n_execs /2 ; +bool fault_injected = (FAULT & flags) && i > n_execs; + +if (barrier) +pthread_barrier_wait (barrier); + +if (flags &MULTI_FAULT ) { + b =0 ; +for (j =0 ; j <N_MULTI_FAULT -1 ; ++j) +__write_dword (data[idx].batch, + sdi_addr + j * orig_size, +WRITE_VALUE (&data[idx], idx), &b); +write_dword (data[idx].batch, sdi_addr + j * orig_size, +WRITE_VALUE (&data[idx], idx), &b); +igt_assert (b <=ARRAY_SIZE (data[idx].batch)); + }else if (!(flags &EVERY_OTHER_CHECK )) { + b =0 ; +write_dword (data[idx].batch, sdi_addr, +WRITE_VALUE (&data[idx], idx), &b); +igt_assert (b <=ARRAY_SIZE (data[idx].batch)); + }else if (flags &EVERY_OTHER_CHECK && !odd (i)) { + b =0 ; +write_dword (data[idx].batch, sdi_addr, +WRITE_VALUE (&data[idx], idx), &b); +igt_assert (b <=ARRAY_SIZE (data[idx].batch)); + + aligned_alloc_type =__aligned_alloc (aligned_size, bo_size); + next_data = aligned_alloc_type.ptr; +igt_assert (next_data); +__aligned_partial_free (&aligned_alloc_type); + + b =0 ; +write_dword (data[next_idx].batch, +to_user_pointer (next_data) + + (char *)&data[next_idx].data - (char *)data, +WRITE_VALUE (&data[next_idx], next_idx), &b); +igt_assert (b <=ARRAY_SIZE (data[next_idx].batch)); + } + +if (!exec_ufence) + data[idx].exec_sync =0 ; + + sync[0 ].addr = exec_ufence ?to_user_pointer (exec_ufence) : + addr + (char *)&data[idx].exec_sync - (char *)data; + + exec.exec_queue_id = exec_queues[e]; +if (fault_inject) + exec.address = batch_addr *2 ; +else + exec.address = batch_addr; + +if (fault_injected) { + err =__xe_exec (fd, &exec); +igt_assert (err == -ENOENT ); + }else { +xe_exec (fd, &exec); + } + +if (barrier) +pthread_barrier_wait (barrier); + +if (fault_inject || fault_injected) { +int64_t timeout =QUARTER_SEC ; + + err =__xe_wait_ufence (fd, exec_ufence ? exec_ufence : + &data[idx].exec_sync, +USER_FENCE_VALUE , + exec_queues[e], &timeout); +igt_assert (err == -ETIME || err == -EIO ); + }else { +xe_wait_ufence (fd, exec_ufence ? exec_ufence : + &data[idx].exec_sync,USER_FENCE_VALUE , + exec_queues[e],FIVE_SEC ); +if (flags &LOCK && !i) +munlock (data, bo_size); + +if (flags &MREMAP ) { +void *old = data; +int remap_flags =MREMAP_MAYMOVE |MREMAP_FIXED ; + +if (flags &DONTUNMAP ) + remap_flags |=MREMAP_DONTUNMAP ; + +if (flags &READ_ONLY_REMAP ) +igt_assert (!mprotect (old, bo_size, +PROT_READ )); + +if (!next_data) { + aligned_alloc_type =__aligned_alloc (aligned_size, + bo_size); + data = aligned_alloc_type.ptr; +__aligned_partial_free (&aligned_alloc_type); + }else { + data = next_data; + } + next_data =NULL ; +igt_assert (data); + + data =mremap (old, bo_size, bo_size, + remap_flags, data); +igt_assert (data !=MAP_FAILED ); + +if (flags &READ_ONLY_REMAP ) +igt_assert (!mprotect (data, bo_size, +PROT_READ | +PROT_WRITE )); + + addr =to_user_pointer (data); +if (flags &DONTUNMAP ) +munmap (old, bo_size); + } + +if (!(flags &EVERY_OTHER_CHECK ) ||odd (i)) { +if (flags &FORK_READ ) { + igt_fork(child, 1) +igt_assert_eq (data[idx].data, +READ_VALUE (&data[idx])); +if (!(flags &FORK_READ_AFTER )) +igt_assert_eq (data[idx].data, +READ_VALUE (&data[idx])); +igt_waitchildren (); +if (flags &FORK_READ_AFTER ) +igt_assert_eq (data[idx].data, +READ_VALUE (&data[idx])); + }else { +igt_assert_eq (data[idx].data, +READ_VALUE (&data[idx])); + +if (flags &MULTI_FAULT ) { +for (j =1 ; j <N_MULTI_FAULT ; ++j) { +struct test_exec_data *__data = + ((void *)data) + j * orig_size; + +igt_assert_eq (__data[idx].data, +READ_VALUE (&data[idx])); + } + } + } +if (flags &EVERY_OTHER_CHECK ) +igt_assert_eq (data[prev_idx].data, +READ_VALUE (&data[prev_idx])); + } + } + +if (exec_ufence) + exec_ufence[0 ] =0 ; + +if (bo) { +__xe_vm_bind_assert (fd, vm,0 , +0 ,0 , addr, bo_size, +DRM_XE_VM_BIND_OP_MAP , +DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR , +NULL ,0 ,0 ,0 ); +munmap (data, bo_size); +gem_close (fd, bo); + } + +if (flags &NEW ) { +if (flags &MMAP ) { +if (flags &FREE ) +munmap (data, bo_size); +else + pending_free[i] = data; + data =mmap (NULL , bo_size,PROT_READ | +PROT_WRITE ,MAP_SHARED | +MAP_ANONYMOUS ,-1 ,0 ); +igt_assert (data !=MAP_FAILED ); + }else if (flags &BO_MAP &&odd (i)) { +if (!bo) { +if (flags &FREE ) +free (data); +else + pending_free[i] = data; + } + + aligned_alloc_type =__aligned_alloc (aligned_size, bo_size); + data = aligned_alloc_type.ptr; +igt_assert (data); +__aligned_partial_free (&aligned_alloc_type); + + bo_flags =DRM_XE_GEM_CREATE_FLAG_NEEDS_VISIBLE_VRAM ; + bo =xe_bo_create (fd, vm, bo_size, +vram_if_possible (fd, eci->gt_id), + bo_flags); + data =xe_bo_map_fixed (fd, bo, bo_size, +to_user_pointer (data)); + +xe_vm_bind_async (fd, vm,0 , bo,0 , +to_user_pointer (data), + bo_size,0 ,0 ); + }else { +if (!bo) { +if (flags &FREE ) +free (data); +else + pending_free[i] = data; + } + bo =0 ; + data =aligned_alloc (aligned_size, bo_size); +igt_assert (data); + } + addr =to_user_pointer (data); +if (!(flags &SKIP_MEMSET )) +memset (data,0 , bo_size); + } + + prev_idx = idx; + } + +if (bo) { +__xe_vm_bind_assert (fd, vm,0 , +0 ,0 , addr, bo_size, +DRM_XE_VM_BIND_OP_MAP , +DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR , +NULL ,0 ,0 ,0 ); +munmap (data, bo_size); + data =NULL ; +gem_close (fd, bo); + } + +if (flags &BUSY ) +igt_assert_eq (unbind_system_allocator (), -EBUSY ); + +for (i =0 ; i < n_exec_queues; i++) +xe_exec_queue_destroy (fd, exec_queues[i]); + +if (exec_ufence) +munmap (exec_ufence,SZ_4K ); + +if (flags &LOCK ) +munlock (data, bo_size); + +if (file_fd !=-1 ) +close (file_fd); + +if (flags &NEW && !(flags &FREE )) { +for (i =0 ; i < n_execs; i++) { +if (!pending_free[i]) +continue ; + +if (flags &MMAP ) +munmap (pending_free[i], bo_size); +else +free (pending_free[i]); + } +free (pending_free); + } +if (data) { +if (flags &MMAP ) +munmap (data, bo_size); +else if (!alloc) +free (data); + } +if (free_vm) +xe_vm_destroy (fd, vm); +} + +struct thread_data { +pthread_t thread; +pthread_mutex_t *mutex; +pthread_cond_t *cond; +pthread_barrier_t *barrier; +int fd; +struct drm_xe_engine_class_instance *eci; +int n_exec_queues; +int n_execs; +size_t bo_size; +size_t stride; +uint32_t vm; +unsigned int flags; +void *alloc; +bool *go; +}; + +static void *thread (void *data) +{ +struct thread_data *t = data; + +pthread_mutex_lock (t->mutex); +while (!*t->go) +pthread_cond_wait (t->cond, t->mutex); +pthread_mutex_unlock (t->mutex); + +test_exec (t->fd, t->eci, t->n_exec_queues, t->n_execs, + t->bo_size, t->stride, t->vm, t->alloc, t->barrier, + t->flags); + +return NULL ; +} + +static void +threads (int fd,int n_exec_queues,int n_execs,size_t bo_size, +size_t stride,unsigned int flags,bool shared_vm) +{ +struct drm_xe_engine_class_instance *hwe; +struct thread_data *threads_data; +int n_engines =0 , i =0 ; +pthread_mutex_t mutex; +pthread_cond_t cond; +pthread_barrier_t barrier; +uint32_t vm =0 ; +bool go = false; +void *alloc =NULL ; + +if ((FILE_BACKED |FORK_READ ) & flags) +return ; + +xe_for_each_engine (fd, hwe) + ++n_engines; + +if (shared_vm) { + vm =xe_vm_create (fd,DRM_XE_VM_CREATE_FLAG_LR_MODE | +DRM_XE_VM_CREATE_FLAG_FAULT_MODE ,0 ); +bind_system_allocator (NULL ,0 ); + } + +if (flags &SHARED_ALLOC ) { +uint64_t alloc_size; + +igt_assert (stride); + + alloc_size =sizeof (struct test_exec_data ) * stride * + n_execs * n_engines; + alloc_size =xe_bb_size (fd, alloc_size); + alloc =aligned_alloc (SZ_2M , alloc_size); +igt_assert (alloc); + +memset (alloc,0 , alloc_size); + flags &= ~SHARED_ALLOC ; + } + + threads_data =calloc (n_engines,sizeof (*threads_data)); +igt_assert (threads_data); + +pthread_mutex_init (&mutex,0 ); +pthread_cond_init (&cond,0 ); +pthread_barrier_init (&barrier,0 , n_engines); + +xe_for_each_engine (fd, hwe) { + threads_data[i].mutex = &mutex; + threads_data[i].cond = &cond; + threads_data[i].barrier = (flags &SYNC_EXEC ) ? &barrier :NULL ; + threads_data[i].fd = fd; + threads_data[i].eci = hwe; + threads_data[i].n_exec_queues = n_exec_queues; + threads_data[i].n_execs = n_execs; + threads_data[i].bo_size = bo_size; + threads_data[i].stride = stride; + threads_data[i].vm = vm; + threads_data[i].flags = flags; + threads_data[i].alloc = alloc ? alloc + i * +sizeof (struct test_exec_data ) :NULL ; + threads_data[i].go = &go; +pthread_create (&threads_data[i].thread,0 , thread, + &threads_data[i]); + ++i; + } + +pthread_mutex_lock (&mutex); + go = true; +pthread_cond_broadcast (&cond); +pthread_mutex_unlock (&mutex); + +for (i =0 ; i < n_engines; ++i) +pthread_join (threads_data[i].thread,NULL ); + +if (shared_vm) { +int ret; + +if (flags &MMAP ) { +int tries =300 ; + +while (tries && (ret =unbind_system_allocator ()) == -EBUSY ) { +sleep (.01 ); + --tries; + } +igt_assert_eq (ret,0 ); + } +xe_vm_destroy (fd, vm); +if (alloc) +free (alloc); + } +free (threads_data); +} + +static void process (struct drm_xe_engine_class_instance *hwe,int n_exec_queues, +int n_execs,size_t bo_size,size_t stride, +unsigned int flags) +{ +struct process_data *pdata; +int map_fd; +int fd; + + map_fd =open (sync_file,O_RDWR ,0x666 ); + pdata =mmap (NULL ,sizeof (*pdata),PROT_READ | +PROT_WRITE ,MAP_SHARED , map_fd,0 ); +wait_pdata (pdata); + + fd =drm_open_driver (DRIVER_XE ); +test_exec (fd, hwe, n_exec_queues, n_execs, + bo_size, stride,0 ,NULL ,NULL , flags); +drm_close_driver (fd); + +close (map_fd); +munmap (pdata,sizeof (*pdata)); +} + +static void +processes (int fd,int n_exec_queues,int n_execs,size_t bo_size, +size_t stride,unsigned int flags) +{ +struct drm_xe_engine_class_instance *hwe; +struct process_data *pdata; +int map_fd; + +if (flags &FORK_READ ) +return ; + + map_fd =open (sync_file,O_RDWR |O_CREAT ,0x666 ); +posix_fallocate (map_fd,0 ,sizeof (*pdata)); + pdata =mmap (NULL ,sizeof (*pdata),PROT_READ | +PROT_WRITE ,MAP_SHARED , map_fd,0 ); + +init_pdata (pdata,0 ); + +xe_for_each_engine (fd, hwe) { + igt_fork(child, 1) +process (hwe, n_exec_queues, n_execs, bo_size, + stride, flags); + } + +signal_pdata (pdata); +igt_waitchildren (); + +close (map_fd); +munmap (pdata,sizeof (*pdata)); +} + +struct section { +const char *name; +unsigned int flags; +}; + +igt_main +{ +struct drm_xe_engine_class_instance *hwe; +const struct section sections[] = { + {"malloc" ,0 }, + {"malloc-multi-fault" ,MULTI_FAULT }, + {"malloc-fork-read" ,FORK_READ }, + {"malloc-fork-read-after" ,FORK_READ |FORK_READ_AFTER }, + {"malloc-mlock" ,LOCK }, + {"malloc-race" ,RACE }, + {"malloc-busy" ,BUSY }, + {"malloc-bo-unmap" ,BO_UNMAP }, + {"mmap" ,MMAP }, + {"mmap-remap" ,MMAP |MREMAP }, + {"mmap-remap-dontunmap" ,MMAP |MREMAP |DONTUNMAP }, + {"mmap-remap-ro" ,MMAP |MREMAP |READ_ONLY_REMAP }, + {"mmap-remap-ro-dontunmap" ,MMAP |MREMAP |DONTUNMAP | +READ_ONLY_REMAP }, + {"mmap-remap-eocheck" ,MMAP |MREMAP |EVERY_OTHER_CHECK }, + {"mmap-remap-dontunmap-eocheck" ,MMAP |MREMAP |DONTUNMAP | +EVERY_OTHER_CHECK }, + {"mmap-remap-ro-eocheck" ,MMAP |MREMAP |READ_ONLY_REMAP | +EVERY_OTHER_CHECK }, + {"mmap-remap-ro-dontunmap-eocheck" ,MMAP |MREMAP |DONTUNMAP | +READ_ONLY_REMAP |EVERY_OTHER_CHECK }, + {"mmap-huge" ,MMAP |HUGE_PAGE }, + {"mmap-shared" ,MMAP |LOCK |MMAP_SHARED }, + {"mmap-shared-remap" ,MMAP |LOCK |MMAP_SHARED |MREMAP }, + {"mmap-shared-remap-dontunmap" ,MMAP |LOCK |MMAP_SHARED | +MREMAP |DONTUNMAP }, + {"mmap-shared-remap-eocheck" ,MMAP |LOCK |MMAP_SHARED | +MREMAP |EVERY_OTHER_CHECK }, + {"mmap-shared-remap-dontunmap-eocheck" ,MMAP |LOCK | +MMAP_SHARED |MREMAP |DONTUNMAP |EVERY_OTHER_CHECK }, + {"mmap-mlock" ,MMAP |LOCK }, + {"mmap-file" ,MMAP |FILE_BACKED }, + {"mmap-file-mlock" ,MMAP |LOCK |FILE_BACKED }, + {"mmap-race" ,MMAP |RACE }, + {"free" ,NEW |FREE }, + {"free-race" ,NEW |FREE |RACE }, + {"new" ,NEW }, + {"new-race" ,NEW |RACE }, + {"new-bo-map" ,NEW |BO_MAP }, + {"new-busy" ,NEW |BUSY }, + {"mmap-free" ,MMAP |NEW |FREE }, + {"mmap-free-huge" ,MMAP |NEW |FREE |HUGE_PAGE }, + {"mmap-free-race" ,MMAP |NEW |FREE |RACE }, + {"mmap-new" ,MMAP |NEW }, + {"mmap-new-huge" ,MMAP |NEW |HUGE_PAGE }, + {"mmap-new-race" ,MMAP |NEW |RACE }, + {"malloc-nomemset" ,SKIP_MEMSET }, + {"malloc-mlock-nomemset" ,SKIP_MEMSET |LOCK }, + {"malloc-race-nomemset" ,SKIP_MEMSET |RACE }, + {"malloc-busy-nomemset" ,SKIP_MEMSET |BUSY }, + {"malloc-bo-unmap-nomemset" ,SKIP_MEMSET |BO_UNMAP }, + {"mmap-nomemset" ,SKIP_MEMSET |MMAP }, + {"mmap-huge-nomemset" ,SKIP_MEMSET |MMAP |HUGE_PAGE }, + {"mmap-shared-nomemset" ,SKIP_MEMSET |MMAP |MMAP_SHARED }, + {"mmap-mlock-nomemset" ,SKIP_MEMSET |MMAP |LOCK }, + {"mmap-file-nomemset" ,SKIP_MEMSET |MMAP |FILE_BACKED }, + {"mmap-file-mlock-nomemset" ,SKIP_MEMSET |MMAP |LOCK |FILE_BACKED }, + {"mmap-race-nomemset" ,SKIP_MEMSET |MMAP |RACE }, + {"free-nomemset" ,SKIP_MEMSET |NEW |FREE }, + {"free-race-nomemset" ,SKIP_MEMSET |NEW |FREE |RACE }, + {"new-nomemset" ,SKIP_MEMSET |NEW }, + {"new-race-nomemset" ,SKIP_MEMSET |NEW |RACE }, + {"new-bo-map-nomemset" ,SKIP_MEMSET |NEW |BO_MAP }, + {"new-busy-nomemset" ,SKIP_MEMSET |NEW |BUSY }, + {"mmap-free-nomemset" ,SKIP_MEMSET |MMAP |NEW |FREE }, + {"mmap-free-huge-nomemset" ,SKIP_MEMSET |MMAP |NEW |FREE |HUGE_PAGE }, + {"mmap-free-race-nomemset" ,SKIP_MEMSET |MMAP |NEW |FREE |RACE }, + {"mmap-new-nomemset" ,SKIP_MEMSET |MMAP |NEW }, + {"mmap-new-huge-nomemset" ,SKIP_MEMSET |MMAP |NEW |HUGE_PAGE }, + {"mmap-new-race-nomemset" ,SKIP_MEMSET |MMAP |NEW |RACE }, + {NULL }, + }; +const struct section psections[] = { + {"munmap-cpu-fault" ,CPU_FAULT }, + {"munmap-no-cpu-fault" ,0 }, + {"remap-cpu-fault" ,CPU_FAULT |REMAP }, + {"remap-no-cpu-fault" ,REMAP }, + {"middle-munmap-cpu-fault" ,MIDDLE |CPU_FAULT }, + {"middle-munmap-no-cpu-fault" ,MIDDLE }, + {"middle-remap-cpu-fault" ,MIDDLE |CPU_FAULT |REMAP }, + {"middle-remap-no-cpu-fault" ,MIDDLE |REMAP }, + {NULL }, + }; +const struct section esections[] = { + {"malloc" ,0 }, + {"malloc-mix-bo" ,MIX_BO_ALLOC }, + {NULL }, + }; +int fd; + + igt_fixture { +struct xe_device *xe; + + fd =drm_open_driver (DRIVER_XE ); +igt_require (!xe_supports_faults (fd)); + + xe =xe_device_get (fd); + va_bits = xe->va_bits; +open_sync_file (); + } + +for (const struct section *s = sections; s->name; s++) { + igt_subtest_f("once-%s", s->name) +xe_for_each_engine (fd, hwe) +test_exec (fd, hwe,1 ,1 ,0 ,0 ,0 ,NULL , +NULL , s->flags); + + igt_subtest_f("once-large-%s", s->name) +xe_for_each_engine (fd, hwe) +test_exec (fd, hwe,1 ,1 ,SZ_2M ,0 ,0 ,NULL , +NULL , s->flags); + + igt_subtest_f("twice-%s", s->name) +xe_for_each_engine (fd, hwe) +test_exec (fd, hwe,1 ,2 ,0 ,0 ,0 ,NULL , +NULL , s->flags); + + igt_subtest_f("twice-large-%s", s->name) +xe_for_each_engine (fd, hwe) +test_exec (fd, hwe,1 ,2 ,SZ_2M ,0 ,0 ,NULL , +NULL , s->flags); + + igt_subtest_f("many-%s", s->name) +xe_for_each_engine (fd, hwe) +test_exec (fd, hwe,1 ,128 ,0 ,0 ,0 ,NULL , +NULL , s->flags); + + igt_subtest_f("many-stride-%s", s->name) +xe_for_each_engine (fd, hwe) +test_exec (fd, hwe,1 ,128 ,0 ,256 ,0 ,NULL , +NULL , s->flags); + + igt_subtest_f("many-execqueues-%s", s->name) +xe_for_each_engine (fd, hwe) +test_exec (fd, hwe,16 ,128 ,0 ,0 ,0 ,NULL , +NULL , s->flags); + + igt_subtest_f("many-large-%s", s->name) +xe_for_each_engine (fd, hwe) +test_exec (fd, hwe,1 ,128 ,SZ_2M ,0 ,0 ,NULL , +NULL , s->flags); + + igt_subtest_f("many-large-execqueues-%s", s->name) +xe_for_each_engine (fd, hwe) +test_exec (fd, hwe,16 ,128 ,SZ_2M ,0 ,0 ,NULL , +NULL , s->flags); + + igt_subtest_f("threads-many-%s", s->name) +threads (fd,1 ,128 ,0 ,0 , s->flags, false); + + igt_subtest_f("threads-many-stride-%s", s->name) +threads (fd,1 ,128 ,0 ,256 , s->flags, false); + + igt_subtest_f("threads-many-execqueues-%s", s->name) +threads (fd,16 ,128 ,0 ,0 , s->flags, false); + + igt_subtest_f("threads-many-large-%s", s->name) +threads (fd,1 ,128 ,SZ_2M ,0 , s->flags, false); + + igt_subtest_f("threads-many-large-execqueues-%s", s->name) +threads (fd,16 ,128 ,SZ_2M ,0 , s->flags, false); + + igt_subtest_f("threads-shared-vm-many-%s", s->name) +threads (fd,1 ,128 ,0 ,0 , s->flags, true); + + igt_subtest_f("threads-shared-vm-many-stride-%s", s->name) +threads (fd,1 ,128 ,0 ,256 , s->flags, true); + + igt_subtest_f("threads-shared-vm-many-execqueues-%s", s->name) +threads (fd,16 ,128 ,0 ,0 , s->flags, true); + + igt_subtest_f("threads-shared-vm-many-large-%s", s->name) +threads (fd,1 ,128 ,SZ_2M ,0 , s->flags, true); + + igt_subtest_f("threads-shared-vm-many-large-execqueues-%s", s->name) +threads (fd,16 ,128 ,SZ_2M ,0 , s->flags, true); + + igt_subtest_f("process-many-%s", s->name) +processes (fd,1 ,128 ,0 ,0 , s->flags); + + igt_subtest_f("process-many-stride-%s", s->name) +processes (fd,1 ,128 ,0 ,256 , s->flags); + + igt_subtest_f("process-many-execqueues-%s", s->name) +processes (fd,16 ,128 ,0 ,0 , s->flags); + + igt_subtest_f("process-many-large-%s", s->name) +processes (fd,1 ,128 ,SZ_2M ,0 , s->flags); + + igt_subtest_f("process-many-large-execqueues-%s", s->name) +processes (fd,16 ,128 ,SZ_2M ,0 , s->flags); + } + + igt_subtest("threads-shared-vm-shared-alloc-many-stride-malloc") +threads (fd,1 ,128 ,0 ,256 ,SHARED_ALLOC , true); + + igt_subtest("threads-shared-vm-shared-alloc-many-stride-malloc-race") +threads (fd,1 ,128 ,0 ,256 ,RACE |SHARED_ALLOC , true); + + igt_subtest("threads-shared-alloc-many-stride-malloc") +threads (fd,1 ,128 ,0 ,256 ,SHARED_ALLOC , false); + + igt_subtest("threads-shared-alloc-many-stride-malloc-sync") +threads (fd,1 ,128 ,0 ,256 ,SHARED_ALLOC |SYNC_EXEC , false); + + igt_subtest("threads-shared-alloc-many-stride-malloc-race") +threads (fd,1 ,128 ,0 ,256 ,RACE |SHARED_ALLOC , false); + + igt_subtest_f("fault") +xe_for_each_engine (fd, hwe) +test_exec (fd, hwe,4 ,1 ,SZ_2M ,0 ,0 ,NULL ,NULL , +FAULT ); + +for (const struct section *s = psections; s->name; s++) { + igt_subtest_f("partial-%s", s->name) +xe_for_each_engine (fd, hwe) +partial (fd, hwe, s->flags); + } + + igt_subtest_f("unaligned-alloc") +xe_for_each_engine (fd, hwe) { +many_allocs (fd, hwe, (SZ_1M +SZ_512K ) *8 , +SZ_1M +SZ_512K ,SZ_4K ,NULL ,0 ); +break ; + } + + igt_subtest_f("fault-benchmark") +xe_for_each_engine (fd, hwe) +many_allocs (fd, hwe,SZ_64M ,SZ_64M ,SZ_4K ,NULL , +BENCHMARK ); + + igt_subtest_f("fault-threads-benchmark") +xe_for_each_engine (fd, hwe) +many_allocs (fd, hwe,SZ_64M ,SZ_64M ,SZ_4K ,NULL , +BENCHMARK |CPU_FAULT_THREADS ); + + igt_subtest_f("fault-threads-same-page-benchmark") +xe_for_each_engine (fd, hwe) +many_allocs (fd, hwe,SZ_64M ,SZ_64M ,SZ_4K ,NULL , +BENCHMARK |CPU_FAULT_THREADS | +CPU_FAULT_SAME_PAGE ); + + igt_subtest_f("fault-process-benchmark") +xe_for_each_engine (fd, hwe) +many_allocs (fd, hwe,SZ_64M ,SZ_64M ,SZ_4K ,NULL , +BENCHMARK |CPU_FAULT_PROCESS ); + + igt_subtest_f("fault-process-same-page-benchmark") +xe_for_each_engine (fd, hwe) +many_allocs (fd, hwe,SZ_64M ,SZ_64M ,SZ_4K ,NULL , +BENCHMARK |CPU_FAULT_PROCESS | +CPU_FAULT_SAME_PAGE ); + +for (const struct section *s = esections; s->name; s++) { + igt_subtest_f("evict-%s", s->name) +xe_for_each_engine (fd, hwe) { +many_allocs (fd, hwe, +xe_visible_vram_size (fd, hwe->gt_id), +SZ_8M ,SZ_1M ,NULL , s->flags); +break ; + } + } + +for (const struct section *s = esections; s->name; s++) { + igt_subtest_f("processes-evict-%s", s->name) +processes_evict (fd,SZ_8M ,SZ_1M , s->flags); + } + + igt_fixture { +xe_device_put (fd); +drm_close_driver (fd); +close_sync_file (); + } +}tests/meson.build +1
-0@@ -295,6 +295,7 @@ intel_xe_progs = [ 'xe_exec_reset', 'xe_exec_sip', 'xe_exec_store', + 'xe_exec_system_allocator', 'xe_exec_threads', 'xe_exercise_blt', 'xe_fault_injection',Signature · 1 line
2.34.1